diff --git a/README.md b/README.md index 0c6a91082dfb..862252da3174 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,21 @@ $object = $bucket->object('file_backup.txt'); $object->downloadToFile('/data/file_backup.txt'); ``` +#### Stream Wrapper + +```php +require 'vendor/autoload.php'; + +use Google\Cloud\Storage\StorageClient; + +$storage = new StorageClient([ + 'projectId' => 'my_project' +]); +$storage->registerStreamWrapper(); + +$contents = file_get_contents('gs://my_bucket/file_backup.txt'); +``` + ## Google Cloud Translation (Alpha) - [API Documentation](http://googlecloudplatform.github.io/google-cloud-php/#/docs/latest/translate/translateclient) diff --git a/phpcs-ruleset.xml b/phpcs-ruleset.xml index ed53b7bbb602..d08453666091 100644 --- a/phpcs-ruleset.xml +++ b/phpcs-ruleset.xml @@ -4,5 +4,8 @@ src/*/V[0-9]+ + + src/Storage/StreamWrapper.php + src diff --git a/src/Storage/Bucket.php b/src/Storage/Bucket.php index 89d0af05c7e6..e0ad49cfc50e 100644 --- a/src/Storage/Bucket.php +++ b/src/Storage/Bucket.php @@ -18,6 +18,7 @@ namespace Google\Cloud\Storage; use Google\Cloud\Exception\NotFoundException; +use Google\Cloud\Exception\ServiceException; use Google\Cloud\Storage\Connection\ConnectionInterface; use Google\Cloud\Upload\ResumableUploader; use GuzzleHttp\Psr7; @@ -274,6 +275,67 @@ public function upload($data, array $options = []) * applied using md5 hashing functionality. If true and the * calculated hash does not match that of the upstream server the * upload will be rejected. + * @type string $predefinedAcl Predefined ACL to apply to the object. + * Acceptable values include `"authenticatedRead`", + * `"bucketOwnerFullControl`", `"bucketOwnerRead`", `"private`", + * `"projectPrivate`", and `"publicRead"`. + * @type array $metadata The available options for metadata are outlined + * at the [JSON API docs](https://cloud.google.com/storage/docs/json_api/v1/objects/insert#request-body). + * @type string $encryptionKey A base64 encoded AES-256 customer-supplied + * encryption key. + * @type string $encryptionKeySHA256 Base64 encoded SHA256 hash of the + * customer-supplied encryption key. This value will be calculated + * from the `encryptionKey` on your behalf if not provided, but + * for best performance it is recommended to pass in a cached + * version of the already calculated SHA. + * } + * @return ResumableUploader + * @throws \InvalidArgumentException + */ + public function getResumableUploader($data, array $options = []) + { + if (is_string($data) && !isset($options['name'])) { + throw new \InvalidArgumentException('A name is required when data is of type string.'); + } + + return $this->connection->insertObject( + $this->formatEncryptionHeaders($options) + [ + 'bucket' => $this->identity['bucket'], + 'data' => $data, + 'resumable' => true + ] + ); + } + + /** + * Get a streamable uploader which can provide greater control over the + * upload process. This is useful for generating large files and uploading + * the contents in chunks. + * + * Example: + * ``` + * $uploader = $bucket->getStreamableUploader( + * 'initial contents', + * ['name' => 'data.txt'] + * ); + * + * // finish uploading the item + * $uploader->upload(); + * ``` + * + * @see https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload#resumable Learn more about resumable + * uploads. + * @see https://cloud.google.com/storage/docs/json_api/v1/objects/insert Objects insert API documentation. + * + * @param string|resource|StreamInterface $data The data to be uploaded. + * @param array $options [optional] { + * Configuration options. + * + * @type string $name The name of the destination. + * @type bool $validate Indicates whether or not validation will be + * applied using md5 hashing functionality. If true and the + * calculated hash does not match that of the upstream server the + * upload will be rejected. * @type int $chunkSize If provided the upload will be done in chunks. * The size must be in multiples of 262144 bytes. With chunking * you have increased reliability at the risk of higher overhead. @@ -292,10 +354,10 @@ public function upload($data, array $options = []) * for best performance it is recommended to pass in a cached * version of the already calculated SHA. * } - * @return ResumableUploader + * @return StreamableUploader * @throws \InvalidArgumentException */ - public function getResumableUploader($data, array $options = []) + public function getStreamableUploader($data, array $options = []) { if (is_string($data) && !isset($options['name'])) { throw new \InvalidArgumentException('A name is required when data is of type string.'); @@ -305,7 +367,8 @@ public function getResumableUploader($data, array $options = []) $this->formatEncryptionHeaders($options) + [ 'bucket' => $this->identity['bucket'], 'data' => $data, - 'resumable' => true + 'streamable' => true, + 'validate' => false ] ); } @@ -671,4 +734,34 @@ public function name() { return $this->identity['bucket']; } + + /** + * Returns whether the bucket with the given file prefix is writable. + * Tries to create a temporary file as a resumable upload which will + * not be completed (and cleaned up by GCS). + * + * @param string $file Optional file to try to write. + * @return boolean + * @throws ServiceException + */ + public function isWritable($file = null) + { + $file = $file ?: '__tempfile'; + $uploader = $this->getResumableUploader( + Psr7\stream_for(''), + ['name' => $file] + ); + try { + $uploader->getResumeUri(); + } catch (ServiceException $e) { + // We expect a 403 access denied error if the bucket is not writable + if ($e->getCode() == 403) { + return false; + } + // If not a 403, re-raise the unexpected error + throw $e; + } + + return true; + } } diff --git a/src/Storage/Connection/Rest.php b/src/Storage/Connection/Rest.php index 6e4ea62cea14..7d71d2e3262c 100644 --- a/src/Storage/Connection/Rest.php +++ b/src/Storage/Connection/Rest.php @@ -24,6 +24,7 @@ use Google\Cloud\Upload\AbstractUploader; use Google\Cloud\Upload\MultipartUploader; use Google\Cloud\Upload\ResumableUploader; +use Google\Cloud\Upload\StreamableUploader; use Google\Cloud\UriTrait; use GuzzleHttp\Psr7; use GuzzleHttp\Psr7\Request; @@ -230,10 +231,16 @@ public function downloadObject(array $args = []) public function insertObject(array $args = []) { $args = $this->resolveUploadOptions($args); - $isResumable = $args['resumable']; - $uploadType = $isResumable - ? AbstractUploader::UPLOAD_TYPE_RESUMABLE - : AbstractUploader::UPLOAD_TYPE_MULTIPART; + + $uploadType = AbstractUploader::UPLOAD_TYPE_RESUMABLE; + if ($args['streamable']) { + $uploaderClass = StreamableUploader::class; + } elseif ($args['resumable']) { + $uploaderClass = ResumableUploader::class; + } else { + $uploaderClass = MultipartUploader::class; + $uploadType = AbstractUploader::UPLOAD_TYPE_MULTIPART; + } $uriParams = [ 'bucket' => $args['bucket'], @@ -243,16 +250,7 @@ public function insertObject(array $args = []) ] ]; - if ($isResumable) { - return new ResumableUploader( - $this->requestWrapper, - $args['data'], - $this->expandUri(self::UPLOAD_URI, $uriParams), - $args['uploaderOptions'] - ); - } - - return new MultipartUploader( + return new $uploaderClass( $this->requestWrapper, $args['data'], $this->expandUri(self::UPLOAD_URI, $uriParams), @@ -270,6 +268,7 @@ private function resolveUploadOptions(array $args) 'name' => null, 'validate' => true, 'resumable' => null, + 'streamable' => null, 'predefinedAcl' => null, 'metadata' => [] ]; diff --git a/src/Storage/ReadStream.php b/src/Storage/ReadStream.php new file mode 100644 index 000000000000..33e55094ca4c --- /dev/null +++ b/src/Storage/ReadStream.php @@ -0,0 +1,93 @@ +stream = $stream; + } + + /** + * Return the full size of the buffer. If the underlying stream does + * not report it's size, try to fetch the size from the Content-Length + * response header. + * + * @return int The size of the stream. + */ + public function getSize() + { + return $this->stream->getSize() ?: $this->getSizeFromMetadata(); + } + + /** + * Attempt to fetch the size from the Content-Length response header. + * If we cannot, return 0. + * + * @return int The Size of the stream + */ + private function getSizeFromMetadata() + { + foreach ($this->stream->getMetadata('wrapper_data') as $value) { + if (substr($value, 0, 15) == "Content-Length:") { + return (int) substr($value, 16); + } + } + return 0; + } + + /** + * Read bytes from the underlying buffer, retrying until we have read + * enough bytes or we cannot read any more. We do this because the + * internal C code for filling a buffer does not account for when + * we try to read large chunks from a user-land stream that does not + * return enough bytes. + * + * @param int $length The number of bytes to read. + * @return string Read bytes from the underlying stream. + */ + public function read($length) + { + $data = ''; + do { + $moreData = $this->stream->read($length); + $data .= $moreData; + $readLength = strlen($moreData); + $length -= $readLength; + } while ($length > 0 && $readLength > 0); + + return $data; + } +} diff --git a/src/Storage/StorageClient.php b/src/Storage/StorageClient.php index 8b64cffc8186..b50ff9a35cdb 100644 --- a/src/Storage/StorageClient.php +++ b/src/Storage/StorageClient.php @@ -217,4 +217,25 @@ public function createBucket($name, array $options = []) $response = $this->connection->insertBucket($options + ['name' => $name, 'project' => $this->projectId]); return new Bucket($this->connection, $name, $response); } + + /** + * Registers this StorageClient as the handler for stream reading/writing. + * + * @param string $protocol The name of the protocol to use. **Defaults to** `gs`. + * @throws \RuntimeException + */ + public function registerStreamWrapper($protocol = null) + { + return StreamWrapper::register($this, $protocol); + } + + /** + * Unregisters the SteamWrapper + * + * @param string $protocol The name of the protocol to unregister. **Defaults to** `gs`. + */ + public function unregisterStreamWrapper($protocol = null) + { + StreamWrapper::unregister($protocol); + } } diff --git a/src/Storage/StorageObject.php b/src/Storage/StorageObject.php index bac0f32ce074..ad2da495ad7c 100644 --- a/src/Storage/StorageObject.php +++ b/src/Storage/StorageObject.php @@ -497,12 +497,19 @@ public function rewrite($destination, array $options = []) * @type string $ifSourceMetagenerationNotMatch Makes the operation * conditional on whether the source object's current * metageneration does not match the given value. + * @type string $destinationBucket Will move to this bucket if set. If + * not set, will default to the same bucket. * } * @return StorageObject The renamed object. */ public function rename($name, array $options = []) { - $copiedObject = $this->copy($this->identity['bucket'], [ + $destinationBucket = isset($options['destinationBucket']) + ? $options['destinationBucket'] + : $this->identity['bucket']; + unset($options['destinationBucket']); + + $copiedObject = $this->copy($destinationBucket, [ 'name' => $name ] + $options); diff --git a/src/Storage/StreamWrapper.php b/src/Storage/StreamWrapper.php new file mode 100644 index 000000000000..f5e9e83c8a10 --- /dev/null +++ b/src/Storage/StreamWrapper.php @@ -0,0 +1,674 @@ +stream_close(); + } + + /** + * Register a StreamWrapper for reading and writing to Google Storage + * + * @param StorageClient $client The StorageClient configuration to use. + * @param string $protocol The name of the protocol to use. **Defaults to** + * `gs`. + * @throws \RuntimeException + */ + public static function register(StorageClient $client, $protocol = null) + { + $protocol = $protocol ?: self::DEFAULT_PROTOCOL; + if (!in_array($protocol, stream_get_wrappers())) { + if (!stream_wrapper_register($protocol, StreamWrapper::class, STREAM_IS_URL)) { + throw new \RuntimeException("Failed to register '$protocol://' protocol"); + } + self::$clients[$protocol] = $client; + return true; + } + return false; + } + + /** + * Unregisters the SteamWrapper + * + * @param string $protocol The name of the protocol to unregister. **Defaults + * to** `gs`. + */ + public static function unregister($protocol = null) + { + $protocol = $protocol ?: self::DEFAULT_PROTOCOL; + stream_wrapper_unregister($protocol); + unset(self::$clients[$protocol]); + } + + /** + * Get the default client to use for streams. + * + * @param string $protocol The name of the protocol to get the client for. + * **Defaults to** `gs`. + * @return StorageClient + */ + public static function getClient($protocol = null) + { + $protocol = $protocol ?: self::DEFAULT_PROTOCOL; + return self::$clients[$protocol]; + } + + /** + * Callback handler for when a stream is opened. For reads, we need to + * download the file to see if it can be opened. + * + * @param string $path The path of the resource to open + * @param string $mode The fopen mode. Currently only supports ('r', 'rb', 'rt', 'w', 'wb', 'wt') + * @param int $flags Bitwise options STREAM_USE_PATH|STREAM_REPORT_ERRORS|STREAM_MUST_SEEK + * @param string $openedPath Will be set to the path on success if STREAM_USE_PATH option is set + * @return bool + */ + public function stream_open($path, $mode, $flags, &$openedPath) + { + $client = $this->openPath($path); + + // strip off 'b' or 't' from the mode + $mode = rtrim($mode, 'bt'); + + $options = []; + if ($this->context) { + $contextOptions = stream_context_get_options($this->context); + if (array_key_exists($this->protocol, $contextOptions)) { + $options = $contextOptions[$this->protocol] ?: []; + } + } + + if ($mode == 'w') { + $this->stream = new WriteStream(null, $options); + $this->stream->setUploader( + $this->bucket->getStreamableUploader( + $this->stream, + $options + ['name' => $this->file] + ) + ); + } elseif ($mode == 'r') { + try { + // Lazy read from the source + $options['httpOptions']['stream'] = true; + $this->stream = new ReadStream( + $this->bucket->object($this->file)->downloadAsStream($options) + ); + + // Wrap the response in a caching stream to make it seekable + if (!$this->stream->isSeekable() && ($flags & STREAM_MUST_SEEK)) { + $this->stream = new CachingStream($this->stream); + } + } catch (ServiceException $ex) { + return $this->returnError($ex->getMessage(), $flags); + } + } else { + return $this->returnError('Unknown stream_open mode.', $flags); + } + + if ($flags & STREAM_USE_PATH) { + $openedPath = $path; + } + return true; + } + + /** + * Callback handler for when we try to read a certain number of bytes. + * + * @param int $count The number of bytes to read. + * + * @return string + */ + public function stream_read($count) + { + return $this->stream->read($count); + } + + /** + * Callback handler for when we try to write data to the stream. + * + * @param string $data The data to write + * + * @return int The number of bytes written. + */ + public function stream_write($data) + { + return $this->stream->write($data); + } + + /** + * Callback handler for getting data about the stream. + * + * @return array + */ + public function stream_stat() + { + $mode = $this->stream->isWritable() + ? self::FILE_WRITABLE_MODE + : self::FILE_READABLE_MODE; + return $this->makeStatArray([ + 'mode' => $mode, + 'size' => $this->stream->getSize() + ]); + } + + /** + * Callback handler for checking to see if the stream is at the end of file. + * + * @return bool + */ + public function stream_eof() + { + return $this->stream->eof(); + } + + /** + * Callback handler for trying to close the stream. + */ + public function stream_close() + { + if (isset($this->stream)) { + $this->stream->close(); + } + } + + /** + * Callback handler for trying to seek to a certain location in the stream. + * + * @param int $offset The stream offset to seek to + * @param int $whence Flag for what the offset is relative to. See: + * http://php.net/manual/en/streamwrapper.stream-seek.php + * @return bool + */ + public function stream_seek($offset, $whence = SEEK_SET) + { + if ($this->stream->isSeekable()) { + $this->stream->seek($offset, $whence); + return true; + } + return false; + } + + /** + * Callhack handler for inspecting our current position in the stream + * + * @return int + */ + public function stream_tell() + { + return $this->stream->tell(); + } + + /** + * Callback handler for trying to close an opened directory. + * + * @return bool + */ + public function dir_closedir() + { + return false; + } + + /** + * Callback handler for trying to open a directory. + * + * @param string $path The url directory to open + * @param int $options Whether or not to enforce safe_mode + * @return bool + */ + public function dir_opendir($path, $options) + { + $this->openPath($path); + return $this->dir_rewinddir(); + } + + /** + * Callback handler for reading an entry from a directory handle. + * + * @return string|bool + */ + public function dir_readdir() + { + $object = $this->directoryGenerator->current(); + if ($object) { + $this->directoryGenerator->next(); + return $object->name(); + } else { + return false; + } + } + + /** + * Callback handler for rewind the directory handle. + * + * @return bool + */ + public function dir_rewinddir() + { + try { + $this->directoryGenerator = $this->bucket->objects([ + 'prefix' => $this->file, + 'fields' => 'items/name,nextPageToken' + ]); + } catch (ServiceException $e) { + return false; + } + return true; + } + + /** + * Callback handler for trying to create a directory. If no file path is specified, + * or STREAM_MKDIR_RECURSIVE option is set, then create the bucket if it does not exist. + * + * @param string $path The url directory to create + * @param int $mode The permissions on the directory + * @param int $options Bitwise mask of options. STREAM_MKDIR_RECURSIVE + * @return bool + */ + public function mkdir($path, $mode, $options) + { + $path = $this->makeDirectory($path); + $client = $this->openPath($path); + $predefinedAcl = $this->determineAclFromMode($mode); + + try { + if ($options & STREAM_MKDIR_RECURSIVE || $this->file == '') { + if (!$this->bucket->exists()) { + $client->createBucket($this->bucket->name(), [ + 'predefinedAcl' => $predefinedAcl, + 'predefinedDefaultObjectAcl' => $predefinedAcl + ]); + } + } + + // If the file name is empty, we were trying to create a bucket. In this case, + // don't create the placeholder file. + if ($this->file != '') { + // Fake a directory by creating an empty placeholder file whose name ends in '/' + $this->bucket->upload('', [ + 'name' => $this->file, + 'predefinedAcl' => $predefinedAcl + ]); + } + } catch (ServiceException $e) { + return false; + } + return true; + } + + /** + * Callback handler for trying to move a file or directory. + * + * @param string $from The URL to the current file + * @param string $to The URL of the new file location + * @return bool + */ + public function rename($from, $to) + { + $url = parse_url($to); + $destinationBucket = $url['host']; + $destinationPath = substr($url['path'], 1); + + $this->dir_opendir($from, []); + foreach ($this->directoryGenerator as $file) { + $name = $file->name(); + $newPath = str_replace($this->file, $destinationPath, $name); + + $obj = $this->bucket->object($name); + try { + $obj->rename($newPath, ['destinationBucket' => $destinationBucket]); + } catch (ServiceException $e) { + // If any rename calls fail, abort and return false + return false; + } + } + return true; + } + + /** + * Callback handler for trying to remove a directory or a bucket. If the path is empty + * or '/', the bucket will be deleted. + * + * Note that the STREAM_MKDIR_RECURSIVE flag is ignored because the option cannot + * be set via the `rmdir()` function. + * + * @param string $path The URL directory to remove. If the path is empty or is '/', + * This will attempt to destroy the bucket. + * @param int $options Bitwise mask of options. + * @return bool + */ + public function rmdir($path, $options) + { + $path = $this->makeDirectory($path); + $this->openPath($path); + + try { + if ($this->file == '') { + $this->bucket->delete(); + return true; + } else { + return $this->unlink($path); + } + } catch (ServiceException $e) { + return false; + } + } + + /** + * Callback handler for retrieving the underlaying resource + * + * @param int $castAs STREAM_CAST_FOR_SELECT|STREAM_CAST_AS_STREAM + * @return resource|bool + */ + public function stream_cast($castAs) + { + return false; + } + + /** + * Callback handler for deleting a file + * + * @param string $path The URL of the file to delete + * @return bool + */ + public function unlink($path) + { + $client = $this->openPath($path); + $object = $this->bucket->object($this->file); + + try { + $object->delete(); + return true; + } catch (ServiceException $e) { + return false; + } + } + + /** + * Callback handler for retrieving information about a file + * + * @param string $path The URI to the file + * @param int $flags Bitwise mask of options + * @return array|bool + */ + public function url_stat($path, $flags) + { + $client = $this->openPath($path); + + // if directory + if ($this->isDirectory($this->file)) { + return $this->urlStatDirectory(); + } else { + return $this->urlStatFile(); + } + } + + /** + * Parse the URL and set protocol, filename and bucket. + * + * @param string $path URL to open + * @return StorageClient + */ + private function openPath($path) + { + $url = parse_url($path); + $this->protocol = $url['scheme']; + $this->file = ltrim($url['path'], '/'); + $client = self::getClient($this->protocol); + $this->bucket = $client->bucket($url['host']); + return $client; + } + + /** + * Given a path, ensure that we return a path that looks like a directory + * + * @param string $path + * @return string + */ + private function makeDirectory($path) + { + if (substr($path, -1) == '/') { + return $path; + } else { + return $path . '/'; + } + } + + /** + * Calculate the `url_stat` response for a directory + * + * @return array|bool + */ + private function urlStatDirectory() + { + $stats = []; + // 1. try to look up the directory as a file + try { + $this->object = $this->bucket->object($this->file); + $info = $this->object->info(); + + // equivalent to 40777 and 40444 in octal + $stats['mode'] = $this->bucket->isWritable() + ? self::DIRECTORY_WRITABLE_MODE + : self::DIRECTORY_READABLE_MODE; + $this->statsFromFileInfo($info, $stats); + + return $this->makeStatArray($stats); + } catch (NotFoundException $e) { + } catch (ServiceException $e) { + return false; + } + + // 2. try list files in that directory + try { + $objects = $this->bucket->objects([ + 'prefix' => $this->file, + ]); + + if (!$objects->current()) { + // can't list objects or doesn't exist + return false; + } + } catch (ServiceException $e) { + return false; + } + + // equivalent to 40777 and 40444 in octal + $mode = $this->bucket->isWritable() + ? self::DIRECTORY_WRITABLE_MODE + : self::DIRECTORY_READABLE_MODE; + return $this->makeStatArray([ + 'mode' => $mode + ]); + } + + /** + * Calculate the `url_stat` response for a file + * + * @return array|bool + */ + private function urlStatFile() + { + try { + $this->object = $this->bucket->object($this->file); + $info = $this->object->info(); + } catch (ServiceException $e) { + // couldn't stat file + return false; + } + + // equivalent to 100666 and 100444 in octal + $stats = array( + 'mode' => $this->bucket->isWritable() + ? self::FILE_WRITABLE_MODE + : self::FILE_READABLE_MODE + ); + $this->statsFromFileInfo($info, $stats); + return $this->makeStatArray($stats); + } + + /** + * Given a `StorageObject` info array, extract the available fields into the + * provided `$stats` array. + * + * @param array $info Array provided from a `StorageObject`. + * @param array $stats Array to put the calculated stats into. + */ + private function statsFromFileInfo(array &$info, array &$stats) + { + $stats['size'] = (int) $info['size']; + $stats['mtime'] = strtotime($info['updated']); + $stats['ctime'] = strtotime($info['timeCreated']); + } + + /** + * Return whether we think the provided path is a directory or not + * + * @param string $path + * @return bool + */ + private function isDirectory($path) + { + return substr($path, -1) == '/'; + } + + /** + * Returns the associative array that a `stat()` response expects using the + * provided stats. Defaults the remaining fields to 0. + * + * @param array $stats Sparse stats entries to set. + * @return array + */ + private function makeStatArray($stats) + { + return array_merge( + array_fill_keys([ + 'dev', + 'ino', + 'mode', + 'nlink', + 'uid', + 'gid', + 'rdev', + 'size', + 'atime', + 'mtime', + 'ctime', + 'blksize', + 'blocks' + ], 0), + $stats + ); + } + + /** + * Helper for whether or not to trigger an error or just return false on an error. + * + * @param string $message The PHP error message to emit. + * @param int $flags Bitwise mask of options (STREAM_REPORT_ERRORS) + * @return bool Returns false + */ + private function returnError($message, $flags) + { + if ($flags & STREAM_REPORT_ERRORS) { + trigger_error($message, E_USER_WARNING); + } + return false; + } + + /** + * Helper for determining which predefinedAcl to use given a mode. + * + * @param int $mode Decimal representation of the file system permissions + * @return string + */ + private function determineAclFromMode($mode) + { + if ($mode & 0004) { + // If any user can read, assume it should be publicRead. + return 'publicRead'; + } elseif ($mode & 0040) { + // If any group user can read, assume it should be projectPrivate. + return 'projectPrivate'; + } else { + // Otherwise, assume only the project/bucket owner can use the bucket. + return 'private'; + } + } +} diff --git a/src/Storage/WriteStream.php b/src/Storage/WriteStream.php new file mode 100644 index 000000000000..e3dcd689393c --- /dev/null +++ b/src/Storage/WriteStream.php @@ -0,0 +1,110 @@ +setUploader($uploader); + } + if (array_key_exists('chunkSize', $options)) { + $this->chunkSize = $options['chunkSize']; + } + $this->stream = new BufferStream($this->chunkSize); + } + + /** + * Close the stream. Uploads any remaining data. + */ + public function close() + { + if ($this->uploader && $this->hasWritten) { + $this->uploader->upload(); + $this->uploader = null; + } + } + + /** + * Write to the stream. If we pass the chunkable size, upload the available chunk. + * + * @param string $data Data to write + * @return int The number of bytes written + * @throws \RuntimeException + */ + public function write($data) + { + if (!isset($this->uploader)) { + throw new \RuntimeException("No uploader set."); + } + + // Ensure we have a resume uri here because we need to create the streaming + // upload before we have data (size of 0). + $this->uploader->getResumeUri(); + $this->hasWritten = true; + + if (!$this->stream->write($data)) { + $this->uploader->upload($this->getChunkedWriteSize()); + } + return strlen($data); + } + + /** + * Set the uploader for this class. You may need to set this after initialization + * if the uploader depends on this stream. + * + * @param AbstractUploader $uploader The new uploader to use. + */ + public function setUploader($uploader) + { + $this->uploader = $uploader; + } + + private function getChunkedWriteSize() + { + return (int) floor($this->getSize() / $this->chunkSize) * $this->chunkSize; + } +} diff --git a/src/Upload/ResumableUploader.php b/src/Upload/ResumableUploader.php index dea4dcb44e59..69666282c233 100644 --- a/src/Upload/ResumableUploader.php +++ b/src/Upload/ResumableUploader.php @@ -31,7 +31,7 @@ class ResumableUploader extends AbstractUploader /** * @var int */ - private $rangeStart = 0; + protected $rangeStart = 0; /** * @var string diff --git a/src/Upload/StreamableUploader.php b/src/Upload/StreamableUploader.php new file mode 100644 index 000000000000..70013d84f227 --- /dev/null +++ b/src/Upload/StreamableUploader.php @@ -0,0 +1,85 @@ +getResumeUri(); + + if ($writeSize) { + $rangeEnd = $this->rangeStart + $writeSize - 1; + $data = $this->data->read($writeSize); + } else { + $rangeEnd = '*'; + $data = $this->data; + } + + // do the streaming write + $headers = [ + 'Content-Length' => $writeSize, + 'Content-Type' => $this->contentType, + 'Content-Range' => "bytes {$this->rangeStart}-$rangeEnd/*" + ]; + + $request = new Request( + 'PUT', + $resumeUri, + $headers, + $data + ); + + try { + $response = $this->requestWrapper->send($request, $this->requestOptions); + } catch (ServiceException $ex) { + throw new GoogleException( + "Upload failed. Please use this URI to resume your upload: $resumeUri", + $ex->getCode(), + $ex + ); + } + + // reset the buffer with the remaining contents + $this->rangeStart += $writeSize; + + return json_decode($response->getBody(), true); + } +} diff --git a/tests/system/Storage/StorageTestCase.php b/tests/system/Storage/StorageTestCase.php index 2cc5c72f15a5..c41dbd700c54 100644 --- a/tests/system/Storage/StorageTestCase.php +++ b/tests/system/Storage/StorageTestCase.php @@ -39,7 +39,8 @@ public static function setUpBeforeClass() self::$client = new StorageClient([ 'keyFilePath' => getenv('GOOGLE_CLOUD_PHP_TESTS_KEY_PATH') ]); - self::$bucket = self::$client->createBucket(uniqid(self::TESTING_PREFIX)); + $bucket = getenv('BUCKET') ?: uniqid(self::TESTING_PREFIX); + self::$bucket = self::$client->createBucket($bucket); self::$object = self::$bucket->upload('somedata', ['name' => uniqid(self::TESTING_PREFIX)]); self::$hasSetUp = true; } @@ -62,5 +63,3 @@ public static function tearDownFixtures() } } } - - diff --git a/tests/system/Storage/StreamWrapper/DirectoryTest.php b/tests/system/Storage/StreamWrapper/DirectoryTest.php new file mode 100644 index 000000000000..a7e1883430af --- /dev/null +++ b/tests/system/Storage/StreamWrapper/DirectoryTest.php @@ -0,0 +1,84 @@ +upload('somedata', ['name' => 'some_folder/1.txt']); + self::$bucket->upload('somedata', ['name' => 'some_folder/2.txt']); + self::$bucket->upload('somedata', ['name' => 'some_folder/3.txt']); + } + + public function testMkDir() + { + $dir = self::generateUrl('test_directory'); + $this->assertTrue(mkdir($dir)); + $this->assertTrue(file_exists($dir . '/')); + $this->assertTrue(is_dir($dir . '/')); + } + + public function testRmDir() + { + $dir = self::generateUrl('test_directory/'); + $this->assertTrue(rmdir($dir)); + $this->assertFalse(file_exists($dir . '/')); + } + + public function testMkDirCreatesBucket() + { + $newBucket = uniqid(self::TESTING_PREFIX); + $bucketUrl = "gs://$newBucket/"; + $this->assertTrue(mkdir($bucketUrl, 0700)); + + $bucket = self::$client->bucket($newBucket); + $this->assertTrue($bucket->exists()); + $this->assertTrue(rmdir($bucketUrl)); + } + + public function testListDirectory() + { + $dir = self::generateUrl('some_folder'); + $fd = opendir($dir); + $this->assertEquals('some_folder/1.txt', readdir($fd)); + $this->assertEquals('some_folder/2.txt', readdir($fd)); + rewinddir($fd); + $this->assertEquals('some_folder/1.txt', readdir($fd)); + closedir($fd); + } + + public function testScanDirectory() + { + $dir = self::generateUrl('some_folder'); + $expected = [ + 'some_folder/1.txt', + 'some_folder/2.txt', + 'some_folder/3.txt', + ]; + $this->assertEquals($expected, scandir($dir)); + $this->assertEquals(array_reverse($expected), scandir($dir, SCANDIR_SORT_DESCENDING)); + } +} diff --git a/tests/system/Storage/StreamWrapper/ImageTest.php b/tests/system/Storage/StreamWrapper/ImageTest.php new file mode 100644 index 000000000000..9ea0769db235 --- /dev/null +++ b/tests/system/Storage/StreamWrapper/ImageTest.php @@ -0,0 +1,78 @@ +upload( + $contents, + ['name' => 'exif.jpg'] + ); + $contents = file_get_contents(self::TEST_IMAGE); + self::$bucket->upload( + $contents, + ['name' => 'plain.jpg'] + ); + } + + /** + * @dataProvider imageProvider + */ + public function testGetImageSize($image, $width, $height) + { + $url = self::generateUrl($image); + $size = getimagesize($url); + $this->assertEquals($width, $size[0]); + $this->assertEquals($height, $size[1]); + } + + /** + * @dataProvider imageProvider + */ + public function testGetImageSizeWithInfo($image, $width, $height) + { + $url = self::generateUrl($image); + $info = array(); + $size = getimagesize($url, $info); + $this->assertEquals($width, $size[0]); + $this->assertEquals($height, $size[1]); + $this->assertTrue(count(array_keys($info)) > 1); + } + + public function imageProvider() + { + return [ + ['plain.jpg', 1956, 960], + ['exif.jpg', 3960, 2640], + ]; + } +} diff --git a/tests/system/Storage/StreamWrapper/ReadTest.php b/tests/system/Storage/StreamWrapper/ReadTest.php new file mode 100644 index 000000000000..6f9dbd619aee --- /dev/null +++ b/tests/system/Storage/StreamWrapper/ReadTest.php @@ -0,0 +1,63 @@ +file = self::generateUrl(self::$object->name()); + } + + public function testFread() + { + $fd = fopen($this->file, 'r'); + $expected = 'somedata'; + $this->assertEquals($expected, fread($fd, strlen($expected))); + $this->assertTrue(fclose($fd)); + } + + public function testFileGetContents() + { + $this->assertEquals('somedata', file_get_contents($this->file)); + } + + public function testGetLines() + { + $fd = fopen($this->file, 'r'); + $expected = 'somedata'; + $this->assertEquals($expected, fgets($fd)); + $this->assertTrue(fclose($fd)); + } + + public function testEof() + { + $fd = fopen($this->file, 'r'); + $this->assertFalse(feof($fd)); + fread($fd, 1000); + $this->assertTrue(feof($fd)); + $this->assertTrue(fclose($fd)); + } + +} diff --git a/tests/system/Storage/StreamWrapper/RenameTest.php b/tests/system/Storage/StreamWrapper/RenameTest.php new file mode 100644 index 000000000000..9ea898393fd6 --- /dev/null +++ b/tests/system/Storage/StreamWrapper/RenameTest.php @@ -0,0 +1,54 @@ +upload('somedata', ['name' => self::TEST_FILE]); + } + + public function testRenameFile() + { + $oldFile = self::generateUrl(self::TEST_FILE); + $newFile = self::generateUrl(self::NEW_TEST_FILE); + $this->assertTrue(rename($oldFile, $newFile)); + $this->assertTrue(file_exists($newFile)); + } + + public function testRenameDirectory() + { + $oldFolder = self::generateUrl(dirname(self::TEST_FILE)); + $newFolder = self::generateUrl('new_folder'); + $newFile = $newFolder . '/bar.txt'; + $this->assertTrue(rename($oldFolder, $newFolder)); + $this->assertTrue(file_exists($newFile)); + } + +} diff --git a/tests/system/Storage/StreamWrapper/StreamWrapperTestCase.php b/tests/system/Storage/StreamWrapper/StreamWrapperTestCase.php new file mode 100644 index 000000000000..8928ff7da42b --- /dev/null +++ b/tests/system/Storage/StreamWrapper/StreamWrapperTestCase.php @@ -0,0 +1,46 @@ +registerStreamWrapper(); + } + + public static function tearDownAfterClass() + { + self::$client->unregisterStreamWrapper(); + parent::tearDownAfterClass(); + } + + protected static function generateUrl($file) + { + $bucketName = self::$bucket->name(); + return "gs://$bucketName/$file"; + } + +} diff --git a/tests/system/Storage/StreamWrapper/UrlStatTest.php b/tests/system/Storage/StreamWrapper/UrlStatTest.php new file mode 100644 index 000000000000..aefa2d8bc6c0 --- /dev/null +++ b/tests/system/Storage/StreamWrapper/UrlStatTest.php @@ -0,0 +1,107 @@ +name()); + self::$dirUrl = self::generateUrl('some_folder/'); + mkdir(self::$dirUrl); + } + + public function testUrlStatFile() + { + $stat = stat(self::$fileUrl); + $this->assertEquals(33206, $stat['mode']); + } + + public function testUrlStatDirectory() + { + $stat = stat(self::$dirUrl); + $this->assertEquals(16895, $stat['mode']); + } + + public function testStatOnOpenFileForWrite() + { + $fd = fopen(self::$fileUrl, 'w'); + $stat = fstat($fd); + $this->assertEquals(33206, $stat['mode']); + } + + public function testStatOnOpenFileForRead() + { + $fd = fopen(self::$fileUrl, 'r'); + $stat = fstat($fd); + $this->assertEquals(33060, $stat['mode']); + } + + public function testIsWritable() + { + $this->assertTrue(is_writable(self::$dirUrl)); + $this->assertTrue(is_writable(self::$fileUrl)); + } + + public function testIsReadable() + { + $this->assertTrue(is_readable(self::$dirUrl)); + $this->assertTrue(is_readable(self::$fileUrl)); + } + + public function testFileExists() + { + $this->assertTrue(file_exists(self::$dirUrl)); + $this->assertTrue(file_exists(self::$fileUrl)); + } + + public function testIsLink() + { + $this->assertFalse(is_link(self::$dirUrl)); + $this->assertFalse(is_link(self::$fileUrl)); + } + + public function testIsExecutable() + { + // php returns false for is_executable if the file is a directory + // https://github.com/php/php-src/blob/master/ext/standard/filestat.c#L907 + $this->assertFalse(is_executable(self::$dirUrl)); + $this->assertFalse(is_executable(self::$fileUrl)); + } + + public function testIsFile() + { + $this->assertTrue(is_file(self::$fileUrl)); + $this->assertFalse(is_file(self::$dirUrl)); + } + + public function testIsDir() + { + $this->assertTrue(is_dir(self::$dirUrl)); + $this->assertFalse(is_dir(self::$fileUrl)); + } + +} diff --git a/tests/system/Storage/StreamWrapper/WriteTest.php b/tests/system/Storage/StreamWrapper/WriteTest.php new file mode 100644 index 000000000000..74d3bb45c6f9 --- /dev/null +++ b/tests/system/Storage/StreamWrapper/WriteTest.php @@ -0,0 +1,73 @@ +fileUrl = self::generateUrl('output.txt'); + unlink($this->fileUrl); + } + + public function tearDown() + { + unlink($this->fileUrl); + } + + public function testFilePutContents() + { + $this->assertFalse(file_exists($this->fileUrl)); + + $output = 'This is a test'; + $this->assertEquals(strlen($output), file_put_contents($this->fileUrl, $output)); + + $this->assertTrue(file_exists($this->fileUrl)); + } + + public function testFwrite() + { + $this->assertFalse(file_exists($this->fileUrl)); + + $output = 'This is a test'; + $fd = fopen($this->fileUrl, 'w'); + $this->assertEquals(strlen($output), fwrite($fd, $output)); + $this->assertTrue(fclose($fd)); + + $this->assertTrue(file_exists($this->fileUrl)); + } + + public function testStreamingWrite() + { + $this->assertFalse(file_exists($this->fileUrl)); + + $fp = fopen($this->fileUrl, 'w'); + for($i = 0; $i < 20000; $i++) { + fwrite($fp, "Line Number: $i\n"); + } + fclose($fp); + + $this->assertTrue(file_exists($this->fileUrl)); + } +} diff --git a/tests/unit/Storage/BucketTest.php b/tests/unit/Storage/BucketTest.php index 4e50e91f8b96..0d0bd2558e86 100644 --- a/tests/unit/Storage/BucketTest.php +++ b/tests/unit/Storage/BucketTest.php @@ -18,6 +18,8 @@ namespace Google\Cloud\Tests\Storage; use Google\Cloud\Exception\NotFoundException; +use Google\Cloud\Exception\ServerException; +use Google\Cloud\Exception\ServiceException; use Google\Cloud\Storage\Bucket; use Google\Cloud\Storage\Connection\ConnectionInterface; use Google\Cloud\Storage\StorageObject; @@ -311,4 +313,31 @@ public function testGetsName() $this->assertEquals($name, $bucket->name()); } + + public function testIsWritable() + { + $this->connection->insertObject(Argument::any())->willReturn($this->resumableUploader); + $this->resumableUploader->getResumeUri()->willReturn('http://some-uri/'); + $bucket = new Bucket($this->connection->reveal(), $name = 'bucket'); + $this->assertTrue($bucket->isWritable()); + } + + public function testIsWritableAccessDenied() + { + $this->connection->insertObject(Argument::any())->willReturn($this->resumableUploader); + $this->resumableUploader->getResumeUri()->willThrow(new ServiceException('access denied', 403)); + $bucket = new Bucket($this->connection->reveal(), $name = 'bucket'); + $this->assertFalse($bucket->isWritable()); + } + + /** + * @expectedException \Google\Cloud\Exception\ServerException + */ + public function testIsWritableServerException() + { + $this->connection->insertObject(Argument::any())->willReturn($this->resumableUploader); + $this->resumableUploader->getResumeUri()->willThrow(new ServerException('maintainence')); + $bucket = new Bucket($this->connection->reveal(), $name = 'bucket'); + $bucket->isWritable(); // raises exception + } } diff --git a/tests/unit/Storage/ReadStreamTest.php b/tests/unit/Storage/ReadStreamTest.php new file mode 100644 index 000000000000..0b64fde0d9b3 --- /dev/null +++ b/tests/unit/Storage/ReadStreamTest.php @@ -0,0 +1,71 @@ +prophesize('Psr\Http\Message\StreamInterface'); + $httpStream->getSize()->willReturn(null); + $httpStream->getMetadata('wrapper_data')->willReturn([ + "Foo: bar", + "User-Agent: php", + "Content-Length: 1234", + "Asdf: qwer", + ]); + + $stream = new ReadStream($httpStream->reveal()); + + $this->assertEquals(1234, $stream->getSize()); + } + + public function testReadsFromHeadersWhenGetSizeIsZero() + { + $httpStream = $this->prophesize('Psr\Http\Message\StreamInterface'); + $httpStream->getSize()->willReturn(0); + $httpStream->getMetadata('wrapper_data')->willReturn([ + "Foo: bar", + "User-Agent: php", + "Content-Length: 1234", + "Asdf: qwer", + ]); + + $stream = new ReadStream($httpStream->reveal()); + + $this->assertEquals(1234, $stream->getSize()); + } + + public function testNoContentLengthHeader() + { + $httpStream = $this->prophesize('Psr\Http\Message\StreamInterface'); + $httpStream->getSize()->willReturn(null); + $httpStream->getMetadata('wrapper_data')->willReturn(array()); + + $stream = new ReadStream($httpStream->reveal()); + + $this->assertEquals(0, $stream->getSize()); + } +} diff --git a/tests/unit/Storage/StorageClientTest.php b/tests/unit/Storage/StorageClientTest.php index 97881840d082..96b3f3d7ac7f 100644 --- a/tests/unit/Storage/StorageClientTest.php +++ b/tests/unit/Storage/StorageClientTest.php @@ -18,6 +18,7 @@ namespace Google\Cloud\Tests\Storage; use Google\Cloud\Storage\StorageClient; +use Google\Cloud\Storage\StreamWrapper; use Prophecy\Argument; /** @@ -82,6 +83,14 @@ public function testCreatesBucket() $this->assertInstanceOf('Google\Cloud\Storage\Bucket', $this->client->createBucket('bucket')); } + + public function testRegisteringStreamWrapper() + { + $this->assertTrue($this->client->registerStreamWrapper()); + $this->assertEquals($this->client, StreamWrapper::getClient()); + $this->assertTrue(in_array('gs', stream_get_wrappers())); + $this->client->unregisterStreamWrapper(); + } } class StorageTestClient extends StorageClient diff --git a/tests/unit/Storage/StreamWrapperTest.php b/tests/unit/Storage/StreamWrapperTest.php new file mode 100644 index 000000000000..95a4b3148192 --- /dev/null +++ b/tests/unit/Storage/StreamWrapperTest.php @@ -0,0 +1,426 @@ +client = $this->prophesize(StorageClient::class); + $this->bucket = $this->prophesize(Bucket::class); + $this->client->bucket('my_bucket')->willReturn($this->bucket->reveal()); + + StreamWrapper::register($this->client->reveal()); + } + + public function tearDown() + { + StreamWrapper::unregister(); + + parent::tearDown(); + } + + /** + * @group storageRead + */ + public function testOpeningExistingFile() + { + $this->mockObjectData("existing_file.txt", "some data to read"); + + $fp = fopen('gs://my_bucket/existing_file.txt', 'r'); + $this->assertEquals("some da", fread($fp, 7)); + $this->assertEquals("ta to read", fread($fp, 1000)); + fclose($fp); + } + + /** + * @group storageRead + */ + public function testOpeningNonExistentFileReturnsFalse() + { + $this->mockDownloadException('non-existent/file.txt', \Google\Cloud\Exception\NotFoundException::class); + + $fp = @fopen('gs://my_bucket/non-existent/file.txt', 'r'); + $this->assertFalse($fp); + } + + /** + * @group storageRead + */ + public function testUnknownOpenMode() + { + $fp = @fopen('gs://my_bucket/existing_file.txt', 'a'); + $this->assertFalse($fp); + } + + /** + * @group storageRead + */ + public function testFileGetContents() + { + $this->mockObjectData("file_get_contents.txt", "some data to read"); + + $this->assertEquals('some data to read', file_get_contents('gs://my_bucket/file_get_contents.txt')); + } + + /** + * @group storageRead + */ + public function testReadLines() + { + $this->mockObjectData("some_long_file.txt", "line1.\nline2."); + + $fp = fopen('gs://my_bucket/some_long_file.txt', 'r'); + $this->assertEquals("line1.\n", fgets($fp)); + $this->assertEquals("line2.", fgets($fp)); + fclose($fp); + } + + /** + * @group storageWrite + */ + public function testFileWrite() + { + $uploader = $this->prophesize(StreamableUploader::class); + $uploader->upload()->shouldBeCalled(); + $uploader->getResumeUri()->willReturn('https://resume-uri/'); + $this->bucket->getStreamableUploader("", Argument::type('array'))->willReturn($uploader->reveal()); + + $fp = fopen('gs://my_bucket/output.txt', 'w'); + $this->assertEquals(6, fwrite($fp, "line1.")); + $this->assertEquals(6, fwrite($fp, "line2.")); + fclose($fp); + } + + /** + * @group storageWrite + */ + public function testFilePutContents() + { + $uploader = $this->prophesize(StreamableUploader::class); + $uploader->upload()->shouldBeCalled(); + $uploader->getResumeUri()->willReturn('https://resume-uri/'); + $this->bucket->getStreamableUploader("", Argument::type('array'))->willReturn($uploader->reveal()); + + file_put_contents('gs://my_bucket/file_put_contents.txt', 'Some data.'); + } + + /** + * @group storageSeek + */ + public function testSeekOnWritableStream() + { + $uploader = $this->prophesize(StreamableUploader::class); + $this->bucket->getStreamableUploader("", Argument::type('array'))->willReturn($uploader->reveal()); + + $fp = fopen('gs://my_bucket/output.txt', 'w'); + $this->assertEquals(-1, fseek($fp, 100)); + fclose($fp); + } + + /** + * @group storageSeek + */ + public function testSeekOnReadableStream() + { + $this->mockObjectData("some_long_file.txt", "line1.\nline2."); + $fp = fopen('gs://my_bucket/some_long_file.txt', 'r'); + $this->assertEquals(-1, fseek($fp, 100)); + fclose($fp); + } + + /** + * @group storageInfo + */ + public function testFstat() + { + $this->mockObjectData("some_long_file.txt", "line1.\nline2."); + $fp = fopen('gs://my_bucket/some_long_file.txt', 'r'); + $stat = fstat($fp); + $this->assertEquals(33206, $stat['mode']); + fclose($fp); + } + + /** + * @group storageInfo + */ + public function testStat() + { + $object = $this->prophesize(StorageObject::class); + $object->info()->willReturn([ + 'size' => 1234, + 'updated' => '2017-01-19T19:31:35.833Z', + 'timeCreated' => '2017-01-19T19:31:35.833Z' + ]); + $this->bucket->object('some_long_file.txt')->willReturn($object->reveal()); + $this->bucket->isWritable()->willReturn(true); + + $stat = stat('gs://my_bucket/some_long_file.txt'); + $this->assertEquals(33206, $stat['mode']); + } + + /** + * @group storageInfo + * @expectedException PHPUnit_Framework_Error_Warning + */ + public function testStatOnNonExistentFile() + { + $object = $this->prophesize(StorageObject::class); + $object->info()->willThrow(NotFoundException::class); + $this->bucket->object('non-existent/file.txt')->willReturn($object->reveal()); + + stat('gs://my_bucket/non-existent/file.txt'); + } + + /** + * @group storageDelete + */ + public function testUnlink() + { + $obj = $this->prophesize(StorageObject::class); + $obj->delete()->willReturn(true)->shouldBeCalled(); + $this->bucket->object('some_long_file.txt')->willReturn($obj->reveal()); + $this->assertTrue(unlink('gs://my_bucket/some_long_file.txt')); + } + + /** + * @group storageDelete + */ + public function testUnlinkOnNonExistentFile() + { + $obj = $this->prophesize(StorageObject::class); + $obj->delete()->willThrow(\Google\Cloud\Exception\NotFoundException::class); + $this->bucket->object('some_long_file.txt')->willReturn($obj->reveal()); + $this->assertFalse(unlink('gs://my_bucket/some_long_file.txt')); + } + + /** + * @group storageDirectory + */ + public function testMkdir() + { + $this->bucket->upload('', ['name' => 'foo/bar/', 'predefinedAcl' => 'publicRead'])->shouldBeCalled(); + $this->assertTrue(mkdir('gs://my_bucket/foo/bar')); + } + + /** + * @group storageDirectory + */ + public function testMkdirProjectPrivate() + { + $this->bucket->upload('', ['name' => 'foo/bar/', 'predefinedAcl' => 'projectPrivate'])->shouldBeCalled(); + $this->assertTrue(mkdir('gs://my_bucket/foo/bar', 0740)); + } + + /** + * @group storageDirectory + */ + public function testMkdirPrivate() + { + $this->bucket->upload('', ['name' => 'foo/bar/', 'predefinedAcl' => 'private'])->shouldBeCalled(); + $this->assertTrue(mkdir('gs://my_bucket/foo/bar', 0700)); + } + + /** + * @group storageDirectory + */ + public function testMkdirOnBadDirectory() + { + $this->bucket->upload('', ['name' => 'foo/bar/', 'predefinedAcl' => 'publicRead'])->willThrow(\Google\Cloud\Exception\NotFoundException::class); + $this->assertFalse(mkdir('gs://my_bucket/foo/bar')); + } + + /** + * @group storageDirectory + */ + public function testMkDirCreatesBucket() + { + $this->bucket->exists()->willReturn(false); + $this->bucket->name()->willReturn('my_bucket'); + $this->client->createBucket('my_bucket', [ + 'predefinedAcl' => 'publicRead', + 'predefinedDefaultObjectAcl' => 'publicRead'] + )->willReturn($this->bucket); + $this->bucket->upload('', ['name' => 'foo/bar/', 'predefinedAcl' => 'publicRead'])->shouldBeCalled(); + + $this->assertTrue(mkdir('gs://my_bucket/foo/bar', 0777, STREAM_MKDIR_RECURSIVE)); + } + + /** + * @group storageDirectory + */ + public function testRmdir() + { + $obj = $this->prophesize(StorageObject::class); + $obj->delete()->willReturn(true)->shouldBeCalled(); + $this->bucket->object('foo/bar/')->willReturn($obj->reveal()); + $this->assertTrue(rmdir('gs://my_bucket/foo/bar')); + } + + /** + * @group storageDirectory + */ + public function testRmdirOnBadDirectory() + { + $obj = $this->prophesize(StorageObject::class); + $obj->delete()->willThrow(\Google\Cloud\Exception\NotFoundException::class); + $this->bucket->object('foo/bar/')->willReturn($obj->reveal()); + $this->assertFalse(rmdir('gs://my_bucket/foo/bar')); + } + + /** + * @group storageDirectory + */ + public function testDirectoryListing() + { + $this->mockDirectoryListing('foo/', ['foo/file1.txt', 'foo/file2.txt', 'foo/file3.txt', 'foo/file4.txt']); + $fd = opendir('gs://my_bucket/foo/'); + $this->assertEquals('foo/file1.txt', readdir($fd)); + $this->assertEquals('foo/file2.txt', readdir($fd)); + $this->assertEquals('foo/file3.txt', readdir($fd)); + rewinddir($fd); + $this->assertEquals('foo/file1.txt', readdir($fd)); + closedir($fd); + } + + /** + * @group storageDirectory + */ + public function testDirectoryListingViaScan() + { + $files = ['foo/file1.txt', 'foo/file2.txt', 'foo/file3.txt', 'foo/file4.txt']; + $this->mockDirectoryListing('foo/', $files); + $this->assertEquals($files, scandir('gs://my_bucket/foo/')); + } + + public function testRenameFile() + { + $this->mockDirectoryListing('foo.txt', ['foo.txt']); + $object = $this->prophesize(StorageObject::class); + $object->rename('new_location/foo.txt', ['destinationBucket' => 'my_bucket'])->shouldBeCalled(); + $this->bucket->object('foo.txt')->willReturn($object->reveal()); + + $this->assertTrue(rename('gs://my_bucket/foo.txt', 'gs://my_bucket/new_location/foo.txt')); + } + + public function testRenameToDifferentBucket() + { + $this->mockDirectoryListing('foo.txt', ['foo.txt']); + $object = $this->prophesize(StorageObject::class); + $object->rename('bar/foo.txt', ['destinationBucket' => 'another_bucket'])->shouldBeCalled(); + $this->bucket->object('foo.txt')->willReturn($object->reveal()); + + $this->assertTrue(rename('gs://my_bucket/foo.txt', 'gs://another_bucket/bar/foo.txt')); + } + + public function testRenameDirectory() + { + $this->mockDirectoryListing('foo', ['foo/bar1.txt', 'foo/bar2.txt', 'foo/asdf/bar.txt']); + + $object = $this->prophesize(StorageObject::class); + $object->rename('nested/folder/bar1.txt', ['destinationBucket' => 'another_bucket'])->shouldBeCalled(); + $this->bucket->object('foo/bar1.txt')->willReturn($object->reveal()); + + $object = $this->prophesize(StorageObject::class); + $object->rename('nested/folder/bar2.txt', ['destinationBucket' => 'another_bucket'])->shouldBeCalled(); + $this->bucket->object('foo/bar2.txt')->willReturn($object->reveal()); + + $object = $this->prophesize(StorageObject::class); + $object->rename('nested/folder/asdf/bar.txt', ['destinationBucket' => 'another_bucket'])->shouldBeCalled(); + $this->bucket->object('foo/asdf/bar.txt')->willReturn($object->reveal()); + + $this->assertTrue(rename('gs://my_bucket/foo', 'gs://another_bucket/nested/folder')); + } + + public function testCanSpecifyChunkSizeViaContext() + { + + $uploader = $this->prophesize(StreamableUploader::class); + $upload = $uploader->upload(5)->willReturn(array())->shouldBeCalled(); + $uploader->upload()->shouldBeCalled(); + $uploader->getResumeUri()->willReturn('https://resume-uri/'); + $this->bucket->getStreamableUploader("", Argument::type('array'))->willReturn($uploader->reveal()); + + $context = stream_context_create(array( + 'gs' => array( + 'chunkSize' => 5 + ) + )); + $fp = fopen('gs://my_bucket/existing_file.txt', 'w', false, $context); + $this->assertEquals(9, fwrite($fp, "123456789")); + fclose($fp); + } + + private function mockObjectData($file, $data, $bucket = null) + { + $bucket = $bucket ?: $this->bucket; + $stream = new \GuzzleHttp\Psr7\BufferStream(100); + $stream->write($data); + $object = $this->prophesize(StorageObject::class); + $object->downloadAsStream(Argument::any())->willReturn($stream); + $bucket->object($file)->willReturn($object->reveal()); + } + + private function mockDownloadException($file, $exception) + { + $object = $this->prophesize(StorageObject::class); + $object->downloadAsStream(Argument::any())->willThrow($exception); + $this->bucket->object($file)->willReturn($object->reveal()); + } + + private function mockDirectoryListing($path, $filesToReturn) + { + $test = $this; + $this->bucket->objects( + Argument::that(function($options) use ($path) { + return $options['prefix'] == $path; + }) + )->will(function() use ($test, $filesToReturn) { + return $test->fileListGenerator($filesToReturn); + }); + } + + private function fileListGenerator($fileToReturn) + { + foreach($fileToReturn as $file) { + $obj = $this->prophesize(StorageObject::class); + $obj->name()->willReturn($file); + yield $obj->reveal(); + } + } +} diff --git a/tests/unit/Storage/WriteStreamTest.php b/tests/unit/Storage/WriteStreamTest.php new file mode 100644 index 000000000000..95c706b33b8f --- /dev/null +++ b/tests/unit/Storage/WriteStreamTest.php @@ -0,0 +1,56 @@ +prophesize(StreamableUploader::class); + $uploader->getResumeUri()->willReturn('https://some-resume-uri/'); + $stream = new WriteStream($uploader->reveal(), ['chunkSize' => 10]); + + // We should see 2 calls to upload with size of 10. + $upload = $uploader->upload(10)->will(function($args) use ($stream) { + if (count($args) > 0) { + $size = $args[0]; + $stream->read(10); + } + return array(); + }); + + // We should see a single call to finish the upload. + $uploader->upload()->shouldBeCalledTimes(1); + + $stream->write('1234567'); + $upload->shouldHaveBeenCalledTimes(0); + $stream->write('8901234'); + $upload->shouldHaveBeenCalledTimes(1); + $stream->write('5678901'); + $upload->shouldHaveBeenCalledTimes(2); + $stream->close(); + } +} diff --git a/tests/unit/Upload/StreamableUploaderTest.php b/tests/unit/Upload/StreamableUploaderTest.php new file mode 100644 index 000000000000..f162e0a2a8f3 --- /dev/null +++ b/tests/unit/Upload/StreamableUploaderTest.php @@ -0,0 +1,147 @@ +requestWrapper = $this->prophesize('Google\Cloud\RequestWrapper'); + $this->stream = new WriteStream(null, ['chunkSize' => 16]); + $this->successBody = '{"canI":"kickIt"}'; + } + + public function testStreamingWrites() + { + $resumeResponse = new Response(200, ['Location' => 'http://some-resume-uri.example.com'], $this->successBody); + $this->requestWrapper->send( + Argument::that(function($request){ + return (string) $request->getUri() == 'http://www.example.com'; + }), + Argument::type('array') + )->willReturn($resumeResponse); + + $uploadResponse = new Response(200, [], $this->successBody); + $upload = $this->requestWrapper->send( + Argument::that(function($request){ + return (string) $request->getUri() == 'http://some-resume-uri.example.com'; + }), + Argument::type('array') + )->willReturn($uploadResponse); + + $uploader = new StreamableUploader( + $this->requestWrapper->reveal(), + $this->stream, + 'http://www.example.com', + ['chunkSize' => 16] + ); + $this->stream->setUploader($uploader); + + // write some data smaller than the chunk size + $this->stream->write("0123456789"); + $upload->shouldHaveBeenCalledTimes(0); + + // write some more data that will put us over the chunk size. + $this->stream->write("more text"); + $upload->shouldHaveBeenCalledTimes(1); + + // finish the upload + $this->assertEquals(json_decode($this->successBody, true), $uploader->upload()); + $upload->shouldHaveBeenCalledTimes(2); + } + + public function testUploadsData() + { + $response = new Response(200, ['Location' => 'theResumeUri'], $this->successBody); + + $this->requestWrapper->send( + Argument::type('Psr\Http\Message\RequestInterface'), + Argument::type('array') + )->willReturn($response); + + $uploader = new StreamableUploader( + $this->requestWrapper->reveal(), + $this->stream, + 'http://www.example.com' + ); + $this->stream->setUploader($uploader); + + $this->assertEquals(json_decode($this->successBody, true), $uploader->upload()); + } + + public function testGetResumeUri() + { + $resumeUri = 'theResumeUri'; + $response = new Response(200, ['Location' => $resumeUri]); + + $this->requestWrapper->send( + Argument::type('Psr\Http\Message\RequestInterface'), + Argument::type('array') + )->willReturn($response); + + $uploader = new StreamableUploader( + $this->requestWrapper->reveal(), + $this->stream, + 'http://www.example.com' + ); + $this->stream->setUploader($uploader); + + $this->assertEquals($resumeUri, $uploader->getResumeUri()); + } + + /** + * @expectedException Google\Cloud\Exception\GoogleException + */ + public function testThrowsExceptionWithFailedUpload() + { + $resumeUriResponse = new Response(200, ['Location' => 'theResumeUri']); + + $this->requestWrapper->send( + Argument::which('getMethod', 'POST'), + Argument::type('array') + )->willReturn($resumeUriResponse); + + $this->requestWrapper->send( + Argument::which('getMethod', 'PUT'), + Argument::type('array') + )->willThrow('Google\Cloud\Exception\GoogleException'); + + $uploader = new StreamableUploader( + $this->requestWrapper->reveal(), + $this->stream, + 'http://www.example.com' + ); + + $uploader->upload(); + } +}