From 8b878a8be76ef1b70188adde29fbea846b7c7079 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 14 Mar 2022 07:17:33 +0000 Subject: [PATCH 01/49] basic definition --- src/Storage/Device.php | 11 +++++++++++ src/Storage/Device/S3.php | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/Storage/Device.php b/src/Storage/Device.php index 619e401a..dbecf142 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -84,6 +84,17 @@ abstract public function abort(string $path, string $extra = ''): bool; */ abstract public function read(string $path, int $offset = 0, int $length = null): string; + /** + * Transfer + * + * @param string $path + * @param string $destination + * @param Device $device + * + * @return string + */ + abstract public function transfer(string $path, string $destination, Device $device): bool; + /** * Write file by given path. * diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 1da94e98..afdec806 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -201,6 +201,19 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks return $metadata['chunks']; } + /** + * Transfer + * + * @param string $path + * @param string $destination + * @param Device $device + * + * @return string + */ + abstract public function transfer(string $path, string $destination, Device $device): bool { + + } + /** * Start Multipart Upload * From 6fe6ef0a436917696a227cc87a7e5d81e08f0554 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 15 Mar 2022 08:06:05 +0000 Subject: [PATCH 02/49] transfer functionality implementation --- src/Storage/Device.php | 25 +++++++++++++++++++++++++ src/Storage/Device/Local.php | 29 +++++++++++++++++++++++++++++ src/Storage/Device/S3.php | 21 +++++++++++++++++++-- 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/src/Storage/Device.php b/src/Storage/Device.php index dbecf142..3ab90d08 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -6,6 +6,31 @@ abstract class Device { + + /** + * Max chunk size while transfering file from one device to another + */ + protected int $transferChunkSize = 20000000; //20 MB + + /** + * Set Transfer Chunk Size + * + * @param int $chunkSize + * @return void + */ + public function setTransferChunkSize(int $chunkSize): void { + $this->transferChunkSize = $chunkSize; + } + + /** + * Get Transfer Chunk Size + * + * @return int + */ + public function getTransferChunkSize(): int { + return $this->transferChunkSize; + } + /** * Get Name. * diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index eed91cf9..87842fdf 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -137,6 +137,35 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks return $chunksReceived; } + /** + * Transfer + * + * @param string $path + * @param string $destination + * @param Device $device + * + * @return string + */ + public function transfer(string $path, string $destination, Device $device): bool { + $size = $this->getFileSize($path); + $contentType = $this->getFileMimeType($path); + + if($size <= $this->transferChunkSize) { + $source = $this->read($path); + return $device->write($destination, $source, $contentType); + } + + $totalChunks = \ceil($size / $this->transferChunkSize); + $counter = 0; + $metadata = ['content_type' => $contentType]; + for($counter; $counter < $totalChunks; $counter++) { + $start = $counter * $this->transferChunkSize; + $data = $this->read($path, $start, $this->transferChunkSize); + $device->upload($data, $destination, $counter+1, $totalChunks, $metadata); + } + return true; + } + /** * Abort Chunked Upload * diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index afdec806..61821cfa 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -210,8 +210,25 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks * * @return string */ - abstract public function transfer(string $path, string $destination, Device $device): bool { - + public function transfer(string $path, string $destination, Device $device): bool { + $response = $this->getInfo($path); + $size = (int)($response['content-length'] ?? 0); + $contentType = $response['content-type'] ?? ''; + + if($size <= $this->transferChunkSize) { + $source = $this->read($path); + return $device->write($destination, $source, $contentType); + } + + $totalChunks = \ceil($size / $this->transferChunkSize); + $counter = 0; + $metadata = ['content_type' => $contentType]; + for($counter; $counter < $totalChunks; $counter++) { + $start = $counter * $this->transferChunkSize; + $data = $this->read($path, $start, $this->transferChunkSize); + $device->upload($data, $destination, $counter+1, $totalChunks, $metadata); + } + return true; } /** From 238e13215180dba0eeb58b658b75fcf115606355 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 15 Mar 2022 08:41:58 +0000 Subject: [PATCH 03/49] update move to use transfer to make it efficient for larger size --- src/Storage/Device/S3.php | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 61821cfa..4943132e 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -392,13 +392,7 @@ public function write(string $path, string $data, string $contentType = ''): boo */ public function move(string $source, string $target): bool { - $type = $this->getFileMimeType($source); - - if ($this->write($target, $this->read($source), $type)) { - $this->delete($source); - } - - return true; + return $this->transfer($source, $target, $this); } /** From 89b130dacbc367d70ba03be839332b92707432a2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 15 Mar 2022 10:30:56 +0000 Subject: [PATCH 04/49] fix and tests in local --- composer.lock | 394 +++++++++++++++-------------- src/Storage/Device/Local.php | 5 +- src/Storage/Device/S3.php | 6 +- tests/Storage/Device/LocalTest.php | 24 +- 4 files changed, 230 insertions(+), 199 deletions(-) diff --git a/composer.lock b/composer.lock index 6f813ef2..497f3899 100644 --- a/composer.lock +++ b/composer.lock @@ -4,20 +4,20 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "124a0248853b18b623e8a315d291e9c0", + "content-hash": "1df212eaa94b7c555a893db0225a179b", "packages": [ { "name": "utopia-php/framework", - "version": "0.19.2", + "version": "0.19.7", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "49e4374b97c0f4d14bc84b424bdc9c3b7747e15f" + "reference": "f17afe77a21873b9be18ebc05283813468b4283a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/49e4374b97c0f4d14bc84b424bdc9c3b7747e15f", - "reference": "49e4374b97c0f4d14bc84b424bdc9c3b7747e15f", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/f17afe77a21873b9be18ebc05283813468b4283a", + "reference": "f17afe77a21873b9be18ebc05283813468b4283a", "shasum": "" }, "require": { @@ -51,9 +51,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.19.2" + "source": "https://github.com/utopia-php/framework/tree/0.19.7" }, - "time": "2021-12-07T09:29:35+00:00" + "time": "2022-02-18T00:04:49+00:00" } ], "packages-dev": [ @@ -63,12 +63,12 @@ "source": { "type": "git", "url": "https://github.com/amphp/amp.git", - "reference": "591e06ec427c2d2e5018172552b9354c208419e7" + "reference": "9d5100cebffa729aaffecd3ad25dc5aeea4f13bb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/amp/zipball/591e06ec427c2d2e5018172552b9354c208419e7", - "reference": "591e06ec427c2d2e5018172552b9354c208419e7", + "url": "https://api.github.com/repos/amphp/amp/zipball/9d5100cebffa729aaffecd3ad25dc5aeea4f13bb", + "reference": "9d5100cebffa729aaffecd3ad25dc5aeea4f13bb", "shasum": "" }, "require": { @@ -91,13 +91,13 @@ } }, "autoload": { - "psr-4": { - "Amp\\": "lib" - }, "files": [ "lib/functions.php", "lib/Internal/functions.php" - ] + ], + "psr-4": { + "Amp\\": "lib" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -137,7 +137,7 @@ "support": { "irc": "irc://irc.freenode.org/amphp", "issues": "https://github.com/amphp/amp/issues", - "source": "https://github.com/amphp/amp/tree/master" + "source": "https://github.com/amphp/amp/tree/v2.6.2" }, "funding": [ { @@ -145,7 +145,7 @@ "type": "github" } ], - "time": "2021-12-03T13:45:05+00:00" + "time": "2022-02-20T17:52:18+00:00" }, { "name": "amphp/byte-stream", @@ -181,12 +181,12 @@ } }, "autoload": { - "psr-4": { - "Amp\\ByteStream\\": "lib" - }, "files": [ "lib/functions.php" - ] + ], + "psr-4": { + "Amp\\ByteStream\\": "lib" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -231,12 +231,12 @@ "source": { "type": "git", "url": "https://github.com/composer/package-versions-deprecated.git", - "reference": "1927b0a81e48643ca45c4a55b34a7645b04f3acc" + "reference": "b4f54f74ef3453349c24a845d22392cd31e65f1d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/1927b0a81e48643ca45c4a55b34a7645b04f3acc", - "reference": "1927b0a81e48643ca45c4a55b34a7645b04f3acc", + "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/b4f54f74ef3453349c24a845d22392cd31e65f1d", + "reference": "b4f54f74ef3453349c24a845d22392cd31e65f1d", "shasum": "" }, "require": { @@ -281,7 +281,7 @@ "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)", "support": { "issues": "https://github.com/composer/package-versions-deprecated/issues", - "source": "https://github.com/composer/package-versions-deprecated/tree/master" + "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.5" }, "funding": [ { @@ -297,7 +297,7 @@ "type": "tidelift" } ], - "time": "2021-09-16T13:11:31+00:00" + "time": "2022-01-17T14:14:24+00:00" }, { "name": "composer/semver", @@ -305,19 +305,19 @@ "source": { "type": "git", "url": "https://github.com/composer/semver.git", - "reference": "83e511e247de329283478496f7a1e114c9517506" + "reference": "f79c90ad4e9b41ac4dfc5d77bf398cf61fbd718b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/semver/zipball/83e511e247de329283478496f7a1e114c9517506", - "reference": "83e511e247de329283478496f7a1e114c9517506", + "url": "https://api.github.com/repos/composer/semver/zipball/f79c90ad4e9b41ac4dfc5d77bf398cf61fbd718b", + "reference": "f79c90ad4e9b41ac4dfc5d77bf398cf61fbd718b", "shasum": "" }, "require": { "php": "^5.3.2 || ^7.0 || ^8.0" }, "require-dev": { - "phpstan/phpstan": "^0.12.54", + "phpstan/phpstan": "^1.4", "symfony/phpunit-bridge": "^4.2 || ^5" }, "default-branch": true, @@ -363,7 +363,7 @@ "support": { "irc": "irc://irc.freenode.org/composer", "issues": "https://github.com/composer/semver/issues", - "source": "https://github.com/composer/semver/tree/3.2.6" + "source": "https://github.com/composer/semver/tree/3.3.0" }, "funding": [ { @@ -379,7 +379,7 @@ "type": "tidelift" } ], - "time": "2021-10-25T11:34:17+00:00" + "time": "2022-03-15T08:35:57+00:00" }, { "name": "composer/xdebug-handler", @@ -387,12 +387,12 @@ "source": { "type": "git", "url": "https://github.com/composer/xdebug-handler.git", - "reference": "f27e06cd9675801df441b3656569b328e04aa37c" + "reference": "dee81bf97571cb7168019825ee9522e8dc2a5936" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/xdebug-handler/zipball/f27e06cd9675801df441b3656569b328e04aa37c", - "reference": "f27e06cd9675801df441b3656569b328e04aa37c", + "url": "https://api.github.com/repos/composer/xdebug-handler/zipball/dee81bf97571cb7168019825ee9522e8dc2a5936", + "reference": "dee81bf97571cb7168019825ee9522e8dc2a5936", "shasum": "" }, "require": { @@ -427,7 +427,7 @@ "support": { "irc": "irc://irc.freenode.org/composer", "issues": "https://github.com/composer/xdebug-handler/issues", - "source": "https://github.com/composer/xdebug-handler/tree/1.4.6" + "source": "https://github.com/composer/xdebug-handler/tree/1.4" }, "funding": [ { @@ -443,7 +443,7 @@ "type": "tidelift" } ], - "time": "2021-03-25T17:01:18+00:00" + "time": "2022-01-05T14:26:21+00:00" }, { "name": "dnoegel/php-xdg-base-dir", @@ -488,25 +488,26 @@ "source": { "type": "git", "url": "https://github.com/doctrine/instantiator.git", - "reference": "6410c4b8352cb64218641457cef64997e6b784fb" + "reference": "10dcfce151b967d20fde1b34ae6640712c3891bc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/instantiator/zipball/6410c4b8352cb64218641457cef64997e6b784fb", - "reference": "6410c4b8352cb64218641457cef64997e6b784fb", + "url": "https://api.github.com/repos/doctrine/instantiator/zipball/10dcfce151b967d20fde1b34ae6640712c3891bc", + "reference": "10dcfce151b967d20fde1b34ae6640712c3891bc", "shasum": "" }, "require": { "php": "^7.1 || ^8.0" }, "require-dev": { - "doctrine/coding-standard": "^8.0", + "doctrine/coding-standard": "^9", "ext-pdo": "*", "ext-phar": "*", - "phpbench/phpbench": "^0.13 || 1.0.0-alpha2", - "phpstan/phpstan": "^0.12", - "phpstan/phpstan-phpunit": "^0.12", - "phpunit/phpunit": "^7.0 || ^8.0 || ^9.0" + "phpbench/phpbench": "^0.16 || ^1", + "phpstan/phpstan": "^1.4", + "phpstan/phpstan-phpunit": "^1", + "phpunit/phpunit": "^7.5 || ^8.5 || ^9.5", + "vimeo/psalm": "^4.22" }, "type": "library", "autoload": { @@ -549,7 +550,7 @@ "type": "tidelift" } ], - "time": "2020-11-10T19:05:51+00:00" + "time": "2022-03-03T08:28:38+00:00" }, { "name": "felixfbecker/advanced-json-rpc", @@ -602,12 +603,12 @@ "source": { "type": "git", "url": "https://github.com/felixfbecker/php-language-server-protocol.git", - "reference": "9d846d1f5cf101deee7a61c8ba7caa0a975cd730" + "reference": "6e82196ffd7c62f7794d778ca52b69feec9f2842" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/felixfbecker/php-language-server-protocol/zipball/9d846d1f5cf101deee7a61c8ba7caa0a975cd730", - "reference": "9d846d1f5cf101deee7a61c8ba7caa0a975cd730", + "url": "https://api.github.com/repos/felixfbecker/php-language-server-protocol/zipball/6e82196ffd7c62f7794d778ca52b69feec9f2842", + "reference": "6e82196ffd7c62f7794d778ca52b69feec9f2842", "shasum": "" }, "require": { @@ -649,9 +650,9 @@ ], "support": { "issues": "https://github.com/felixfbecker/php-language-server-protocol/issues", - "source": "https://github.com/felixfbecker/php-language-server-protocol/tree/1.5.1" + "source": "https://github.com/felixfbecker/php-language-server-protocol/tree/master" }, - "time": "2021-02-22T14:02:09+00:00" + "time": "2022-03-02T22:36:06+00:00" }, { "name": "myclabs/deep-copy", @@ -659,34 +660,35 @@ "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "776f831124e9c62e1a2c601ecc52e776d8bb7220" + "reference": "14daed4296fae74d9e3201d2c4925d1acb7aa614" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/776f831124e9c62e1a2c601ecc52e776d8bb7220", - "reference": "776f831124e9c62e1a2c601ecc52e776d8bb7220", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/14daed4296fae74d9e3201d2c4925d1acb7aa614", + "reference": "14daed4296fae74d9e3201d2c4925d1acb7aa614", "shasum": "" }, "require": { "php": "^7.1 || ^8.0" }, - "replace": { - "myclabs/deep-copy": "self.version" + "conflict": { + "doctrine/collections": "<1.6.8", + "doctrine/common": "<2.13.3 || >=3,<3.2.2" }, "require-dev": { - "doctrine/collections": "^1.0", - "doctrine/common": "^2.6", - "phpunit/phpunit": "^7.1" + "doctrine/collections": "^1.6.8", + "doctrine/common": "^2.13.3 || ^3.2.2", + "phpunit/phpunit": "^7.5.20 || ^8.5.23 || ^9.5.13" }, "default-branch": true, "type": "library", "autoload": { - "psr-4": { - "DeepCopy\\": "src/DeepCopy/" - }, "files": [ "src/DeepCopy/deep_copy.php" - ] + ], + "psr-4": { + "DeepCopy\\": "src/DeepCopy/" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -702,7 +704,7 @@ ], "support": { "issues": "https://github.com/myclabs/DeepCopy/issues", - "source": "https://github.com/myclabs/DeepCopy/tree/1.10.2" + "source": "https://github.com/myclabs/DeepCopy/tree/1.11.0" }, "funding": [ { @@ -710,7 +712,7 @@ "type": "tidelift" } ], - "time": "2020-11-13T09:40:50+00:00" + "time": "2022-03-03T13:19:32+00:00" }, { "name": "netresearch/jsonmapper", @@ -878,16 +880,17 @@ "source": { "type": "git", "url": "https://github.com/phar-io/manifest.git", - "reference": "97803eca37d319dfa7826cc2437fc020857acb53" + "reference": "36d8a21e851a9512db2b086dc5ac2c61308f0138" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phar-io/manifest/zipball/97803eca37d319dfa7826cc2437fc020857acb53", - "reference": "97803eca37d319dfa7826cc2437fc020857acb53", + "url": "https://api.github.com/repos/phar-io/manifest/zipball/36d8a21e851a9512db2b086dc5ac2c61308f0138", + "reference": "36d8a21e851a9512db2b086dc5ac2c61308f0138", "shasum": "" }, "require": { "ext-dom": "*", + "ext-libxml": "*", "ext-phar": "*", "ext-xmlwriter": "*", "phar-io/version": "^3.0.1", @@ -929,22 +932,28 @@ "description": "Component for reading phar.io manifest information from a PHP Archive (PHAR)", "support": { "issues": "https://github.com/phar-io/manifest/issues", - "source": "https://github.com/phar-io/manifest/tree/2.0.3" + "source": "https://github.com/phar-io/manifest/tree/master" }, - "time": "2021-07-20T11:28:43+00:00" + "funding": [ + { + "url": "https://github.com/theseer", + "type": "github" + } + ], + "time": "2022-02-21T19:55:33+00:00" }, { "name": "phar-io/version", - "version": "3.1.0", + "version": "3.2.1", "source": { "type": "git", "url": "https://github.com/phar-io/version.git", - "reference": "bae7c545bef187884426f042434e561ab1ddb182" + "reference": "4f7fd7836c6f332bb2933569e566a0d6c4cbed74" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phar-io/version/zipball/bae7c545bef187884426f042434e561ab1ddb182", - "reference": "bae7c545bef187884426f042434e561ab1ddb182", + "url": "https://api.github.com/repos/phar-io/version/zipball/4f7fd7836c6f332bb2933569e566a0d6c4cbed74", + "reference": "4f7fd7836c6f332bb2933569e566a0d6c4cbed74", "shasum": "" }, "require": { @@ -980,9 +989,9 @@ "description": "Library for handling version information and constraints", "support": { "issues": "https://github.com/phar-io/version/issues", - "source": "https://github.com/phar-io/version/tree/3.1.0" + "source": "https://github.com/phar-io/version/tree/3.2.1" }, - "time": "2021-02-23T14:00:09+00:00" + "time": "2022-02-21T01:04:05+00:00" }, { "name": "phpdocumentor/reflection-common", @@ -1043,12 +1052,12 @@ "source": { "type": "git", "url": "https://github.com/phpDocumentor/ReflectionDocBlock.git", - "reference": "622548b623e81ca6d78b721c5e029f4ce664f170" + "reference": "21481a5c97e8332f7279e31219c32faa2da21c79" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/ReflectionDocBlock/zipball/622548b623e81ca6d78b721c5e029f4ce664f170", - "reference": "622548b623e81ca6d78b721c5e029f4ce664f170", + "url": "https://api.github.com/repos/phpDocumentor/ReflectionDocBlock/zipball/21481a5c97e8332f7279e31219c32faa2da21c79", + "reference": "21481a5c97e8332f7279e31219c32faa2da21c79", "shasum": "" }, "require": { @@ -1059,7 +1068,7 @@ "webmozart/assert": "^1.9.1" }, "require-dev": { - "mockery/mockery": "~1.3.2", + "mockery/mockery": "~1.3.5", "psalm/phar": "^4.8" }, "default-branch": true, @@ -1091,9 +1100,9 @@ "description": "With this component, a library can provide support for annotations via DocBlocks or otherwise retrieve information that is embedded in a DocBlock.", "support": { "issues": "https://github.com/phpDocumentor/ReflectionDocBlock/issues", - "source": "https://github.com/phpDocumentor/ReflectionDocBlock/tree/5.3.0" + "source": "https://github.com/phpDocumentor/ReflectionDocBlock/tree/master" }, - "time": "2021-10-19T17:43:47+00:00" + "time": "2021-12-27T22:36:43+00:00" }, { "name": "phpdocumentor/type-resolver", @@ -1101,12 +1110,12 @@ "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "f8ec4ab631de5a97769e66b13418c3b8b24e81f4" + "reference": "93ebd0014cab80c4ea9f5e297ea48672f1b87706" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/f8ec4ab631de5a97769e66b13418c3b8b24e81f4", - "reference": "f8ec4ab631de5a97769e66b13418c3b8b24e81f4", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/93ebd0014cab80c4ea9f5e297ea48672f1b87706", + "reference": "93ebd0014cab80c4ea9f5e297ea48672f1b87706", "shasum": "" }, "require": { @@ -1142,9 +1151,9 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.x" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.0" }, - "time": "2021-11-24T08:29:39+00:00" + "time": "2022-01-04T19:58:01+00:00" }, { "name": "phpspec/prophecy", @@ -1220,12 +1229,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "299e0a60b0f5bd3296fd52290a585ac2b2517b65" + "reference": "9a61ec72f0d2b0b1d3f7431f90ab936b9c0e2062" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/299e0a60b0f5bd3296fd52290a585ac2b2517b65", - "reference": "299e0a60b0f5bd3296fd52290a585ac2b2517b65", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/9a61ec72f0d2b0b1d3f7431f90ab936b9c0e2062", + "reference": "9a61ec72f0d2b0b1d3f7431f90ab936b9c0e2062", "shasum": "" }, "require": { @@ -1289,7 +1298,7 @@ "type": "github" } ], - "time": "2021-12-05T09:19:13+00:00" + "time": "2022-03-08T06:18:06+00:00" }, { "name": "phpunit/php-file-iterator", @@ -1297,12 +1306,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-file-iterator.git", - "reference": "cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf" + "reference": "38b24367e1b340aa78b96d7cab042942d917bb84" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf", - "reference": "cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/38b24367e1b340aa78b96d7cab042942d917bb84", + "reference": "38b24367e1b340aa78b96d7cab042942d917bb84", "shasum": "" }, "require": { @@ -1349,7 +1358,7 @@ "type": "github" } ], - "time": "2021-12-02T12:48:52+00:00" + "time": "2022-02-11T16:23:04+00:00" }, { "name": "phpunit/php-invoker", @@ -1538,12 +1547,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "02eb251b5a2c3dae82e16774eca8fabf7039fd7b" + "reference": "7699c48c8b2b75f885f9999786d3b92fe0802275" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/02eb251b5a2c3dae82e16774eca8fabf7039fd7b", - "reference": "02eb251b5a2c3dae82e16774eca8fabf7039fd7b", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/7699c48c8b2b75f885f9999786d3b92fe0802275", + "reference": "7699c48c8b2b75f885f9999786d3b92fe0802275", "shasum": "" }, "require": { @@ -1559,7 +1568,7 @@ "phar-io/version": "^3.0.2", "php": ">=7.3", "phpspec/prophecy": "^1.12.1", - "phpunit/php-code-coverage": "^9.2.7", + "phpunit/php-code-coverage": "^9.2.13", "phpunit/php-file-iterator": "^3.0.5", "phpunit/php-invoker": "^3.1.1", "phpunit/php-text-template": "^2.0.3", @@ -1573,7 +1582,7 @@ "sebastian/global-state": "^5.0.1", "sebastian/object-enumerator": "^4.0.3", "sebastian/resource-operations": "^3.0.3", - "sebastian/type": "^2.3.4", + "sebastian/type": "^3.0", "sebastian/version": "^3.0.2" }, "require-dev": { @@ -1594,11 +1603,11 @@ } }, "autoload": { - "classmap": [ - "src/" - ], "files": [ "src/Framework/Assert/Functions.php" + ], + "classmap": [ + "src/" ] }, "notification-url": "https://packagist.org/downloads/", @@ -1633,7 +1642,7 @@ "type": "github" } ], - "time": "2021-11-29T15:44:44+00:00" + "time": "2022-03-15T09:19:53+00:00" }, { "name": "psr/container", @@ -2172,12 +2181,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/exporter.git", - "reference": "65e8b7db476c5dd267e65eea9cab77584d3cfff9" + "reference": "428c31e2ea8b292aa814bc460cf28d58eba4d2ba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/65e8b7db476c5dd267e65eea9cab77584d3cfff9", - "reference": "65e8b7db476c5dd267e65eea9cab77584d3cfff9", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/428c31e2ea8b292aa814bc460cf28d58eba4d2ba", + "reference": "428c31e2ea8b292aa814bc460cf28d58eba4d2ba", "shasum": "" }, "require": { @@ -2241,20 +2250,20 @@ "type": "github" } ], - "time": "2021-11-11T14:18:36+00:00" + "time": "2022-03-06T06:59:32+00:00" }, { "name": "sebastian/global-state", - "version": "5.0.3", + "version": "5.0.x-dev", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "23bd5951f7ff26f12d4e3242864df3e08dec4e49" + "reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/23bd5951f7ff26f12d4e3242864df3e08dec4e49", - "reference": "23bd5951f7ff26f12d4e3242864df3e08dec4e49", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/0ca8db5a5fc9c8646244e629625ac486fa286bf2", + "reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2", "shasum": "" }, "require": { @@ -2297,7 +2306,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/global-state/issues", - "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.3" + "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.5" }, "funding": [ { @@ -2305,7 +2314,7 @@ "type": "github" } ], - "time": "2021-06-11T13:31:12+00:00" + "time": "2022-02-14T08:28:10+00:00" }, { "name": "sebastian/lines-of-code", @@ -2584,7 +2593,6 @@ "description": "Provides a list of PHP built-in functions that operate on resources", "homepage": "https://www.github.com/sebastianbergmann/resource-operations", "support": { - "issues": "https://github.com/sebastianbergmann/resource-operations/issues", "source": "https://github.com/sebastianbergmann/resource-operations/tree/3.0.3" }, "funding": [ @@ -2597,28 +2605,28 @@ }, { "name": "sebastian/type", - "version": "2.3.x-dev", + "version": "3.0.x-dev", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/type.git", - "reference": "f24cbc541026c3bb7d27c647f0f9ea337135b22a" + "reference": "b233b84bc4465aff7b57cf1c4bc75c86d00d6dad" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/f24cbc541026c3bb7d27c647f0f9ea337135b22a", - "reference": "f24cbc541026c3bb7d27c647f0f9ea337135b22a", + "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/b233b84bc4465aff7b57cf1c4bc75c86d00d6dad", + "reference": "b233b84bc4465aff7b57cf1c4bc75c86d00d6dad", "shasum": "" }, "require": { "php": ">=7.3" }, "require-dev": { - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^9.5" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "2.3-dev" + "dev-master": "3.0-dev" } }, "autoload": { @@ -2641,7 +2649,7 @@ "homepage": "https://github.com/sebastianbergmann/type", "support": { "issues": "https://github.com/sebastianbergmann/type/issues", - "source": "https://github.com/sebastianbergmann/type/tree/2.3" + "source": "https://github.com/sebastianbergmann/type/tree/3.0.0" }, "funding": [ { @@ -2649,7 +2657,7 @@ "type": "github" } ], - "time": "2021-06-18T06:28:45+00:00" + "time": "2022-03-15T09:54:48+00:00" }, { "name": "sebastian/version", @@ -2710,12 +2718,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "9130e1a0fc93cb0faadca4ee917171bd2ca9e5f4" + "reference": "d8111acc99876953f52fe16d4c50eb60940d49ad" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/9130e1a0fc93cb0faadca4ee917171bd2ca9e5f4", - "reference": "9130e1a0fc93cb0faadca4ee917171bd2ca9e5f4", + "url": "https://api.github.com/repos/symfony/console/zipball/d8111acc99876953f52fe16d4c50eb60940d49ad", + "reference": "d8111acc99876953f52fe16d4c50eb60940d49ad", "shasum": "" }, "require": { @@ -2786,7 +2794,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v5.4.1" + "source": "https://github.com/symfony/console/tree/v5.4.5" }, "funding": [ { @@ -2802,7 +2810,7 @@ "type": "tidelift" } ], - "time": "2021-12-09T11:22:43+00:00" + "time": "2022-02-24T12:45:35+00:00" }, { "name": "symfony/deprecation-contracts", @@ -2810,12 +2818,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "c726b64c1ccfe2896cb7df2e1331c357ad1c8ced" + "reference": "893fd20d9ae41a0bae2b9cbdd581ac0cf3917de3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/c726b64c1ccfe2896cb7df2e1331c357ad1c8ced", - "reference": "c726b64c1ccfe2896cb7df2e1331c357ad1c8ced", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/893fd20d9ae41a0bae2b9cbdd581ac0cf3917de3", + "reference": "893fd20d9ae41a0bae2b9cbdd581ac0cf3917de3", "shasum": "" }, "require": { @@ -2825,7 +2833,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "3.0-dev" + "dev-main": "3.1-dev" }, "thanks": { "name": "symfony/contracts", @@ -2854,7 +2862,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v3.0.0" + "source": "https://github.com/symfony/deprecation-contracts/tree/main" }, "funding": [ { @@ -2870,11 +2878,11 @@ "type": "tidelift" } ], - "time": "2021-11-01T23:48:49+00:00" + "time": "2021-11-29T18:10:03+00:00" }, { "name": "symfony/polyfill-ctype", - "version": "dev-main", + "version": "v1.25.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", @@ -2895,7 +2903,6 @@ "suggest": { "ext-ctype": "For best performance" }, - "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -2907,12 +2914,12 @@ } }, "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Ctype\\": "" - }, "files": [ "bootstrap.php" - ] + ], + "psr-4": { + "Symfony\\Polyfill\\Ctype\\": "" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -2937,7 +2944,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/main" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.25.0" }, "funding": [ { @@ -2957,7 +2964,7 @@ }, { "name": "symfony/polyfill-intl-grapheme", - "version": "dev-main", + "version": "v1.25.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-grapheme.git", @@ -2975,7 +2982,6 @@ "suggest": { "ext-intl": "For best performance" }, - "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -2987,12 +2993,12 @@ } }, "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Intl\\Grapheme\\": "" - }, "files": [ "bootstrap.php" - ] + ], + "psr-4": { + "Symfony\\Polyfill\\Intl\\Grapheme\\": "" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -3019,7 +3025,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/main" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.25.0" }, "funding": [ { @@ -3039,7 +3045,7 @@ }, { "name": "symfony/polyfill-intl-normalizer", - "version": "dev-main", + "version": "v1.25.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-normalizer.git", @@ -3057,7 +3063,6 @@ "suggest": { "ext-intl": "For best performance" }, - "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3069,12 +3074,12 @@ } }, "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Intl\\Normalizer\\": "" - }, "files": [ "bootstrap.php" ], + "psr-4": { + "Symfony\\Polyfill\\Intl\\Normalizer\\": "" + }, "classmap": [ "Resources/stubs" ] @@ -3104,7 +3109,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.23.0" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.25.0" }, "funding": [ { @@ -3124,7 +3129,7 @@ }, { "name": "symfony/polyfill-mbstring", - "version": "dev-main", + "version": "v1.25.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", @@ -3145,7 +3150,6 @@ "suggest": { "ext-mbstring": "For best performance" }, - "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3157,12 +3161,12 @@ } }, "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Mbstring\\": "" - }, "files": [ "bootstrap.php" - ] + ], + "psr-4": { + "Symfony\\Polyfill\\Mbstring\\": "" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -3188,7 +3192,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/main" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.25.0" }, "funding": [ { @@ -3208,7 +3212,7 @@ }, { "name": "symfony/polyfill-php73", - "version": "dev-main", + "version": "v1.25.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php73.git", @@ -3223,7 +3227,6 @@ "require": { "php": ">=7.1" }, - "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3235,12 +3238,12 @@ } }, "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Php73\\": "" - }, "files": [ "bootstrap.php" ], + "psr-4": { + "Symfony\\Polyfill\\Php73\\": "" + }, "classmap": [ "Resources/stubs" ] @@ -3268,7 +3271,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php73/tree/main" + "source": "https://github.com/symfony/polyfill-php73/tree/v1.25.0" }, "funding": [ { @@ -3288,22 +3291,21 @@ }, { "name": "symfony/polyfill-php80", - "version": "dev-main", + "version": "v1.25.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "57b712b08eddb97c762a8caa32c84e037892d2e9" + "reference": "4407588e0d3f1f52efb65fbe92babe41f37fe50c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/57b712b08eddb97c762a8caa32c84e037892d2e9", - "reference": "57b712b08eddb97c762a8caa32c84e037892d2e9", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/4407588e0d3f1f52efb65fbe92babe41f37fe50c", + "reference": "4407588e0d3f1f52efb65fbe92babe41f37fe50c", "shasum": "" }, "require": { "php": ">=7.1" }, - "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3315,12 +3317,12 @@ } }, "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Php80\\": "" - }, "files": [ "bootstrap.php" ], + "psr-4": { + "Symfony\\Polyfill\\Php80\\": "" + }, "classmap": [ "Resources/stubs" ] @@ -3352,7 +3354,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/main" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.25.0" }, "funding": [ { @@ -3368,7 +3370,7 @@ "type": "tidelift" } ], - "time": "2021-09-13T13:58:33+00:00" + "time": "2022-03-04T08:16:47+00:00" }, { "name": "symfony/service-contracts", @@ -3376,12 +3378,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "36715ebf9fb9db73db0cb24263c79077c6fe8603" + "reference": "bc0a2247c72d29241b5a06fb60dc1c9d9acf2a3a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/36715ebf9fb9db73db0cb24263c79077c6fe8603", - "reference": "36715ebf9fb9db73db0cb24263c79077c6fe8603", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/bc0a2247c72d29241b5a06fb60dc1c9d9acf2a3a", + "reference": "bc0a2247c72d29241b5a06fb60dc1c9d9acf2a3a", "shasum": "" }, "require": { @@ -3398,7 +3400,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "3.0-dev" + "dev-main": "3.1-dev" }, "thanks": { "name": "symfony/contracts", @@ -3435,7 +3437,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.0.0" + "source": "https://github.com/symfony/service-contracts/tree/main" }, "funding": [ { @@ -3451,7 +3453,7 @@ "type": "tidelift" } ], - "time": "2021-11-04T17:53:12+00:00" + "time": "2021-11-29T18:10:03+00:00" }, { "name": "symfony/string", @@ -3459,16 +3461,16 @@ "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "aa14a00c4701b9cfbe21aec46972c493b3edddf5" + "reference": "6e0bf37c9c4826549e1c601123aa9d1b36b4f36e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/aa14a00c4701b9cfbe21aec46972c493b3edddf5", - "reference": "aa14a00c4701b9cfbe21aec46972c493b3edddf5", + "url": "https://api.github.com/repos/symfony/string/zipball/6e0bf37c9c4826549e1c601123aa9d1b36b4f36e", + "reference": "6e0bf37c9c4826549e1c601123aa9d1b36b4f36e", "shasum": "" }, "require": { - "php": ">=8.0.2", + "php": ">=8.1", "symfony/polyfill-ctype": "~1.8", "symfony/polyfill-intl-grapheme": "~1.0", "symfony/polyfill-intl-normalizer": "~1.0", @@ -3485,12 +3487,12 @@ }, "type": "library", "autoload": { - "psr-4": { - "Symfony\\Component\\String\\": "" - }, "files": [ "Resources/functions.php" ], + "psr-4": { + "Symfony\\Component\\String\\": "" + }, "exclude-from-classmap": [ "/Tests/" ] @@ -3536,7 +3538,7 @@ "type": "tidelift" } ], - "time": "2021-12-11T14:57:13+00:00" + "time": "2022-02-25T11:15:52+00:00" }, { "name": "theseer/tokenizer", @@ -3664,13 +3666,13 @@ } }, "autoload": { - "psr-4": { - "Psalm\\": "src/Psalm/" - }, "files": [ "src/functions.php", "src/spl_object_id.php" - ] + ], + "psr-4": { + "Psalm\\": "src/Psalm/" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -3856,8 +3858,8 @@ "prefer-stable": false, "prefer-lowest": false, "platform": { - "php": ">=7.4" + "php": ">=8.0" }, "platform-dev": [], - "plugin-api-version": "2.1.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 87842fdf..e2841607 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -161,7 +161,10 @@ public function transfer(string $path, string $destination, Device $device): boo for($counter; $counter < $totalChunks; $counter++) { $start = $counter * $this->transferChunkSize; $data = $this->read($path, $start, $this->transferChunkSize); - $device->upload($data, $destination, $counter+1, $totalChunks, $metadata); + $tmp = $this->getPath('tmp_' . microtime()); + $this->write($tmp, $data, $contentType); + $device->upload($tmp, $destination, $counter+1, $totalChunks, $metadata); + $this->delete($tmp); } return true; } diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 4943132e..1318f0a3 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -223,10 +223,14 @@ public function transfer(string $path, string $destination, Device $device): boo $totalChunks = \ceil($size / $this->transferChunkSize); $counter = 0; $metadata = ['content_type' => $contentType]; + $local = new Local('/tmp'); for($counter; $counter < $totalChunks; $counter++) { $start = $counter * $this->transferChunkSize; $data = $this->read($path, $start, $this->transferChunkSize); - $device->upload($data, $destination, $counter+1, $totalChunks, $metadata); + $tmp = $this->getPath('tmp_' . microtime()); + $local->write($tmp, $data, $contentType); + $device->upload($tmp, $destination, $counter+1, $totalChunks, $metadata); + $local->delete($tmp); } return true; } diff --git a/tests/Storage/Device/LocalTest.php b/tests/Storage/Device/LocalTest.php index d21665fb..825fd185 100644 --- a/tests/Storage/Device/LocalTest.php +++ b/tests/Storage/Device/LocalTest.php @@ -4,6 +4,7 @@ use Utopia\Storage\Device\Local; use PHPUnit\Framework\TestCase; +use Utopia\Storage\Device\S3; class LocalTest extends TestCase { @@ -209,7 +210,6 @@ public function testPartRead($path) { $chunk = file_get_contents($source, false,null, 0, 500); $readChunk = $this->object->read($path, 0, 500); $this->assertEquals($chunk, $readChunk); - $this->object->delete($path); } public function testPartitionFreeSpace() @@ -221,4 +221,26 @@ public function testPartitionTotalSpace() { $this->assertGreaterThan(0, $this->object->getPartitionTotalSpace()); } + + /** + * @depends testPartUpload + */ + public function testTransfer($path) { + // smaller file + $this->object->setTransferChunkSize(10000000); //10 mb + + $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; + $secret = $_SERVER['S3_SECRET'] ?? ''; + $bucket = 'appwrite-storage'; + + $device = new S3('/root', $key, $secret, $bucket, S3::AP_SOUTH_1, S3::ACL_PRIVATE); + $destination = $device->getPath('largefile.mp4'); + + $this->assertTrue($this->object->transfer($path, $destination, $device )); + $this->assertTrue($device->exists($destination)); + $this->assertEquals($device->getFileMimeType($destination), 'video/mp4'); + + $device->delete($destination); + $this->object->delete($path); + } } From a987b602dc5bedc7ba081e3be94ef2d354220af4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 15 Mar 2022 10:53:12 +0000 Subject: [PATCH 05/49] more test --- tests/Storage/Device/LocalTest.php | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/Storage/Device/LocalTest.php b/tests/Storage/Device/LocalTest.php index 825fd185..6c4ee878 100644 --- a/tests/Storage/Device/LocalTest.php +++ b/tests/Storage/Device/LocalTest.php @@ -225,8 +225,9 @@ public function testPartitionTotalSpace() /** * @depends testPartUpload */ - public function testTransfer($path) { - // smaller file + public function testTransferLarge($path) { + + // chunked file $this->object->setTransferChunkSize(10000000); //10 mb $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; @@ -242,5 +243,28 @@ public function testTransfer($path) { $device->delete($destination); $this->object->delete($path); + + } + + public function testTransferSmall() { + + $this->object->setTransferChunkSize(10000000); //10 mb + + $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; + $secret = $_SERVER['S3_SECRET'] ?? ''; + $bucket = 'appwrite-storage'; + + $device = new S3('/root', $key, $secret, $bucket, S3::AP_SOUTH_1, S3::ACL_PRIVATE); + + $path = $this->object->getPath('text-for-read.txt'); + $this->object->write($path, 'Hello World'); + + $destination = $device->getPath('hello.txt'); + $this->assertTrue($this->object->transfer($path, $destination, $device)); + $this->assertTrue($device->exists($destination)); + $this->assertEquals($device->read($destination), 'Hello World'); + + $this->object->delete($path); + $device->delete($destination); } } From 8363ec1260dd0e34fbeadc4b2d4edc77c59f1252 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 15 Mar 2022 11:40:26 +0000 Subject: [PATCH 06/49] tests --- tests/Storage/Device/S3Test.php | 2 +- tests/Storage/S3Base.php | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/Storage/Device/S3Test.php b/tests/Storage/Device/S3Test.php index e9e81904..5489a2ac 100644 --- a/tests/Storage/Device/S3Test.php +++ b/tests/Storage/Device/S3Test.php @@ -13,7 +13,7 @@ protected function init(): void $this->root = '/root'; $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; - $bucket = 'utopia-storage-tests'; + $bucket = 'appwrite-storage'; $this->object = new S3($this->root, $key, $secret, $bucket, S3::AP_SOUTH_1, S3::ACL_PRIVATE); } diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index feeb8193..7189d1b2 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -3,6 +3,7 @@ namespace Utopia\Tests; use PHPUnit\Framework\TestCase; +use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; abstract class S3Base extends TestCase @@ -244,6 +245,43 @@ public function testPartRead($path) { $chunk = file_get_contents($source, false,null, 0, 500); $readChunk = $this->object->read($path, 0, 500); $this->assertEquals($chunk, $readChunk); + } + + /** + * @depends testPartUpload + */ + public function testTransferLarge($path) { + + // chunked file + $this->object->setTransferChunkSize(10000000); //10 mb + + $device = new Local('/root'); + $destination = $device->getPath('largefile.mp4'); + + $this->assertTrue($this->object->transfer($path, $destination, $device )); + $this->assertTrue($device->exists($destination)); + $this->assertEquals($device->getFileMimeType($destination), 'video/mp4'); + + $device->delete($destination); + $this->object->delete($path); + + } + + public function testTransferSmall() { + + $this->object->setTransferChunkSize(10000000); //10 mb + + $device = new Local('/root'); + + $path = $this->object->getPath('text-for-read.txt'); + $this->object->write($path, 'Hello World'); + + $destination = $device->getPath('hello.txt'); + $this->assertTrue($this->object->transfer($path, $destination, $device)); + $this->assertTrue($device->exists($destination)); + $this->assertEquals($device->read($destination), 'Hello World'); + $this->object->delete($path); + $device->delete($destination); } } From 3773001edaa7aa9438cc2fc50e505f40ae26d053 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 16 Mar 2022 10:36:12 +0000 Subject: [PATCH 07/49] refactored to implement transfer --- src/Storage/Device.php | 19 ++++++ src/Storage/Device/Local.php | 114 +++++++++++++++++++++++++++-------- src/Storage/Device/S3.php | 39 ++++++++---- 3 files changed, 135 insertions(+), 37 deletions(-) diff --git a/src/Storage/Device.php b/src/Storage/Device.php index 3ab90d08..3a15fbea 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -88,6 +88,25 @@ abstract public function getPath(string $filename, string $prefix = null): strin */ abstract public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int; + /** + * Upload Data. + * + * Upload file contents to desired destination in the selected disk. + * return number of chunks uploaded or 0 if it fails. + * + * @param string $source + * @param string $path + * @param string $contentType + * @param int chunk + * @param int chunks + * @param array $metadata + * + * @throws \Exception + * + * @return int + */ + abstract public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int; + /** * Abort Chunked Upload * diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index e2841607..172c61e3 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -83,11 +83,7 @@ public function getPath(string $filename, string $prefix = null): string */ public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { - if (!\file_exists(\dirname($path))) { // Checks if directory path to file exists - if (!@\mkdir(\dirname($path), 0755, true)) { - throw new Exception('Can\'t create directory: ' . \dirname($path)); - } - } + $this->createDirectoryToPath($path); //move_uploaded_file() verifies the file is not tampered with if($chunks === 1) { @@ -98,11 +94,7 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks } $tmp = \dirname($path) . DIRECTORY_SEPARATOR . 'tmp_' . \basename($path) . DIRECTORY_SEPARATOR . \basename($path) . '_chunks.log'; - if (!\file_exists(\dirname($tmp))) { // Checks if directory path to file exists - if (!@\mkdir(\dirname($tmp), 0755, true)) { - throw new Exception('Can\'t create directory: ' . \dirname($tmp)); - } - } + $this->createDirectoryToPath($tmp); if(!file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { throw new Exception('Can\'t write chunk log ' . $tmp); } @@ -119,24 +111,97 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks } if ($chunks === $chunksReceived) { - for($i = 1; $i <= $chunks; $i++) { - $part = dirname($tmp) . DIRECTORY_SEPARATOR . pathinfo($path, PATHINFO_FILENAME) . '.part.'. $i; - $data = file_get_contents($part); - if(!$data) { - throw new Exception('Failed to read chunk ' . $part); - } - - if(!file_put_contents($path, $data, FILE_APPEND)) { - throw new Exception('Failed to append chunk ' . $part); - } - \unlink($part); + $this->joinChunks($path, $chunks); + return $chunksReceived; + } + return $chunksReceived; + } + + /** + * Upload Data. + * + * Upload file contents to desired destination in the selected disk. + * return number of chunks uploaded or 0 if it fails. + * + * @param string $source + * @param string $path + * @param string $contentType + * @param int chunk + * @param int chunks + * @param array $metadata + * + * @throws \Exception + * + * @return int + */ + public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int + { + $this->createDirectoryToPath($path); + + if($chunks === 1) { + if(!\file_put_contents($path, $data)) { + throw new Exception('Can\'t write file ' . $path); } - \unlink($tmp); + return $chunks; + } + $tmp = \dirname($path) . DIRECTORY_SEPARATOR . 'tmp_' . \basename($path) . DIRECTORY_SEPARATOR . \basename($path) . '_chunks.log'; + + $this->createDirectoryToPath($tmp); + if(!file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { + throw new Exception('Can\'t write chunk log ' . $tmp); + } + + $chunkLogs = file($tmp); + if(!$chunkLogs) { + throw new Exception('Unable to read chunk log ' . $tmp); + } + + $chunksReceived = count(file($tmp)); + + if(!\file_put_contents(dirname($tmp) . DIRECTORY_SEPARATOR . pathinfo($path, PATHINFO_FILENAME) . '.part.' . $chunk, $data)) { + throw new Exception('Failed to write chunk ' . $chunk); + } + + if ($chunks === $chunksReceived) { + $this->joinChunks($path, $chunks); return $chunksReceived; } return $chunksReceived; } + private function joinChunks(string $path, int $chunks) { + $tmp = \dirname($path) . DIRECTORY_SEPARATOR . 'tmp_' . \basename($path) . DIRECTORY_SEPARATOR . \basename($path) . '_chunks.log'; + for($i = 1; $i <= $chunks; $i++) { + $part = dirname($tmp) . DIRECTORY_SEPARATOR . pathinfo($path, PATHINFO_FILENAME) . '.part.'. $i; + $data = file_get_contents($part); + if(!$data) { + throw new Exception('Failed to read chunk ' . $part); + } + + if(!file_put_contents($path, $data, FILE_APPEND)) { + throw new Exception('Failed to append chunk ' . $part); + } + \unlink($part); + } + \unlink($tmp); + } + + /** + * Create Directory to Path + * + * Create's missing directory of the path, returns true on success and false on failure + * + * @param string path + * @throws Exception + */ + private function createDirectoryToPath(string $path): void { + if (!\file_exists(\dirname($path))) { // Checks if directory path to file exists + if (!@\mkdir(\dirname($path), 0755, true)) { + throw new Exception('Can\'t create directory: ' . \dirname($path)); + } + } + } + /** * Transfer * @@ -161,10 +226,7 @@ public function transfer(string $path, string $destination, Device $device): boo for($counter; $counter < $totalChunks; $counter++) { $start = $counter * $this->transferChunkSize; $data = $this->read($path, $start, $this->transferChunkSize); - $tmp = $this->getPath('tmp_' . microtime()); - $this->write($tmp, $data, $contentType); - $device->upload($tmp, $destination, $counter+1, $totalChunks, $metadata); - $this->delete($tmp); + $device->uploadData($data, $destination, $contentType, $counter+1, $totalChunks, $metadata); } return true; } diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 7df03589..a1044833 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -180,17 +180,39 @@ public function getPath(string $filename, string $prefix = null): string * @return int */ public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int + { + return $this->uploadData(\file_get_contents($source), $path, \mime_content_type($source), $chunk, $chunks, $metadata); + } + + /** + * Upload Data. + * + * Upload file contents to desired destination in the selected disk. + * return number of chunks uploaded or 0 if it fails. + * + * @param string $source + * @param string $path + * @param string $contentType + * @param int chunk + * @param int chunks + * @param array $metadata + * + * @throws \Exception + * + * @return int + */ + public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { if($chunk == 1 && $chunks == 1) { - return $this->write($path, \file_get_contents($source), \mime_content_type($source)); + return $this->write($path, $data, $contentType); } $uploadId = $metadata['uploadId'] ?? null; if(empty($uploadId)) { - $uploadId = $this->createMultipartUpload($path, $metadata['content_type']); + $uploadId = $this->createMultipartUpload($path, $contentType); $metadata['uploadId'] = $uploadId; } - $etag = $this->uploadPart($source, $path, $chunk, $uploadId); + $etag = $this->uploadPart($data, $path, $contentType, $chunk, $uploadId); $metadata['parts'] ??= []; $metadata['parts'][] = ['partNumber' => $chunk, 'etag' => $etag]; $metadata['chunks'] ??= 0; @@ -223,14 +245,10 @@ public function transfer(string $path, string $destination, Device $device): boo $totalChunks = \ceil($size / $this->transferChunkSize); $counter = 0; $metadata = ['content_type' => $contentType]; - $local = new Local('/tmp'); for($counter; $counter < $totalChunks; $counter++) { $start = $counter * $this->transferChunkSize; $data = $this->read($path, $start, $this->transferChunkSize); - $tmp = $this->getPath('tmp_' . microtime()); - $local->write($tmp, $data, $contentType); - $device->upload($tmp, $destination, $counter+1, $totalChunks, $metadata); - $local->delete($tmp); + $device->uploadData($data, $destination, $contentType, $counter+1, $totalChunks, $metadata); } return true; } @@ -271,12 +289,11 @@ protected function createMultipartUpload(string $path, string $contentType): str * * @return string */ - protected function uploadPart(string $source, string $path, int $chunk, string $uploadId) : string + protected function uploadPart(string $data, string $path, string $contentType, int $chunk, string $uploadId) : string { $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; - $data = \file_get_contents($source); - $this->headers['content-type'] = \mime_content_type($source); + $this->headers['content-type'] = $contentType; $this->headers['content-md5'] = \base64_encode(md5($data, true)); $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $data); unset($this->amzHeaders['x-amz-acl']); // ACL header is not allowed in parts, only createMultipartUpload accepts this header. From 39734038c0f7e6da59ed7df61f6e96af2dcb85ec Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 16 Mar 2022 13:45:52 +0000 Subject: [PATCH 08/49] fix transfer test --- tests/Storage/S3Base.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index 7189d1b2..91cf3729 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -255,7 +255,7 @@ public function testTransferLarge($path) { // chunked file $this->object->setTransferChunkSize(10000000); //10 mb - $device = new Local('/root'); + $device = new Local(__DIR__ . '/../resources/disk-a'); $destination = $device->getPath('largefile.mp4'); $this->assertTrue($this->object->transfer($path, $destination, $device )); @@ -271,7 +271,7 @@ public function testTransferSmall() { $this->object->setTransferChunkSize(10000000); //10 mb - $device = new Local('/root'); + $device = new Local(__DIR__ . '/../resources/disk-a'); $path = $this->object->getPath('text-for-read.txt'); $this->object->write($path, 'Hello World'); From e6219d43b6bf0b171567002a49e2eb9f8370ce52 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 16 Mar 2022 14:08:56 +0000 Subject: [PATCH 09/49] fix bucket name --- tests/Storage/Device/LocalTest.php | 2 +- tests/Storage/Device/S3Test.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Storage/Device/LocalTest.php b/tests/Storage/Device/LocalTest.php index 6c4ee878..27c349a6 100644 --- a/tests/Storage/Device/LocalTest.php +++ b/tests/Storage/Device/LocalTest.php @@ -252,7 +252,7 @@ public function testTransferSmall() { $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; - $bucket = 'appwrite-storage'; + $bucket = 'utopia-storage-tests'; $device = new S3('/root', $key, $secret, $bucket, S3::AP_SOUTH_1, S3::ACL_PRIVATE); diff --git a/tests/Storage/Device/S3Test.php b/tests/Storage/Device/S3Test.php index 5489a2ac..e9e81904 100644 --- a/tests/Storage/Device/S3Test.php +++ b/tests/Storage/Device/S3Test.php @@ -13,7 +13,7 @@ protected function init(): void $this->root = '/root'; $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; - $bucket = 'appwrite-storage'; + $bucket = 'utopia-storage-tests'; $this->object = new S3($this->root, $key, $secret, $bucket, S3::AP_SOUTH_1, S3::ACL_PRIVATE); } From b8d9790484d15a9894f335ba334b44649f700246 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 18 Mar 2022 06:19:28 +0000 Subject: [PATCH 10/49] fix tests --- tests/Storage/Device/LocalTest.php | 2 +- tests/Storage/S3Base.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Storage/Device/LocalTest.php b/tests/Storage/Device/LocalTest.php index 27c349a6..80030708 100644 --- a/tests/Storage/Device/LocalTest.php +++ b/tests/Storage/Device/LocalTest.php @@ -232,7 +232,7 @@ public function testTransferLarge($path) { $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; - $bucket = 'appwrite-storage'; + $bucket = 'utopia-storage-tests'; $device = new S3('/root', $key, $secret, $bucket, S3::AP_SOUTH_1, S3::ACL_PRIVATE); $destination = $device->getPath('largefile.mp4'); diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index 91cf3729..89ff7029 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -274,7 +274,7 @@ public function testTransferSmall() { $device = new Local(__DIR__ . '/../resources/disk-a'); $path = $this->object->getPath('text-for-read.txt'); - $this->object->write($path, 'Hello World'); + $this->object->write($path, 'Hello World', 'text/plain'); $destination = $device->getPath('hello.txt'); $this->assertTrue($this->object->transfer($path, $destination, $device)); From b80a2e415d5dc3c3bc82413690ef7de13bcea625 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 18 Mar 2022 08:18:37 +0000 Subject: [PATCH 11/49] fix move --- src/Storage/Device/S3.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index a1044833..d6c931b4 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -416,7 +416,10 @@ public function write(string $path, string $data, string $contentType = ''): boo */ public function move(string $source, string $target): bool { - return $this->transfer($source, $target, $this); + if($this->transfer($source, $target, $this)) { + return $this->delete($source); + } + return false; } /** From eb1c769ae7eb6264909e7f191e9f543e824acf0b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 23 May 2022 06:54:15 +0000 Subject: [PATCH 12/49] move implementation in device --- src/Storage/Device.php | 10 ++++++++-- src/Storage/Device/S3.php | 20 -------------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/src/Storage/Device.php b/src/Storage/Device.php index 3a15fbea..e3c7d573 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -94,7 +94,7 @@ abstract public function upload(string $source, string $path, int $chunk = 1, in * Upload file contents to desired destination in the selected disk. * return number of chunks uploaded or 0 if it fails. * - * @param string $source + * @param string $data * @param string $path * @param string $contentType * @param int chunk @@ -159,7 +159,13 @@ abstract public function write(string $path, string $data, string $contentType): * * @return bool */ - abstract public function move(string $source, string $target): bool; + public function move(string $source, string $target): bool + { + if($this->transfer($source, $target, $this)) { + return $this->delete($source); + } + return false; + } /** * Delete file in given path return true on success and false on failure. diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index d6c931b4..1caacc59 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -402,26 +402,6 @@ public function write(string $path, string $data, string $contentType = ''): boo return true; } - /** - * Move file from given source to given path, Return true on success and false on failure. - * - * @see http://php.net/manual/en/function.filesize.php - * - * @param string $source - * @param string $target - * - * @throw \Exception - * - * @return bool - */ - public function move(string $source, string $target): bool - { - if($this->transfer($source, $target, $this)) { - return $this->delete($source); - } - return false; - } - /** * Delete file in given path, Return true on success and false on failure. * From 53e185589bc79feb195c1567c303885af35b7152 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 23 May 2022 08:43:56 +0000 Subject: [PATCH 13/49] update test bucket and region --- tests/Storage/Device/S3Test.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Storage/Device/S3Test.php b/tests/Storage/Device/S3Test.php index e9e81904..dd6e61e2 100644 --- a/tests/Storage/Device/S3Test.php +++ b/tests/Storage/Device/S3Test.php @@ -13,9 +13,9 @@ protected function init(): void $this->root = '/root'; $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; - $bucket = 'utopia-storage-tests'; + $bucket = 'appwrite-test-bucket'; - $this->object = new S3($this->root, $key, $secret, $bucket, S3::AP_SOUTH_1, S3::ACL_PRIVATE); + $this->object = new S3($this->root, $key, $secret, $bucket, S3::EU_WEST_1, S3::ACL_PRIVATE); } /** From 63dd53c366899f196378f8b5ffb49778e112e072 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 24 May 2022 04:14:42 +0000 Subject: [PATCH 14/49] fix transfer test s3 credentials --- tests/Storage/Device/LocalTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Storage/Device/LocalTest.php b/tests/Storage/Device/LocalTest.php index 80030708..ea7ce845 100644 --- a/tests/Storage/Device/LocalTest.php +++ b/tests/Storage/Device/LocalTest.php @@ -252,9 +252,9 @@ public function testTransferSmall() { $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; - $bucket = 'utopia-storage-tests'; + $bucket = 'appwrite-test-bucket'; - $device = new S3('/root', $key, $secret, $bucket, S3::AP_SOUTH_1, S3::ACL_PRIVATE); + $device = new S3('/root', $key, $secret, $bucket, S3::EU_WEST_1, S3::ACL_PRIVATE); $path = $this->object->getPath('text-for-read.txt'); $this->object->write($path, 'Hello World'); From aa2bb18c7680632efd646efc1b9987591e822ec3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 24 May 2022 05:15:59 +0000 Subject: [PATCH 15/49] update test bucket --- tests/Storage/Device/LocalTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Storage/Device/LocalTest.php b/tests/Storage/Device/LocalTest.php index ea7ce845..bff48044 100644 --- a/tests/Storage/Device/LocalTest.php +++ b/tests/Storage/Device/LocalTest.php @@ -232,9 +232,9 @@ public function testTransferLarge($path) { $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; - $bucket = 'utopia-storage-tests'; + $bucket = 'appwrite-test-bucket'; - $device = new S3('/root', $key, $secret, $bucket, S3::AP_SOUTH_1, S3::ACL_PRIVATE); + $device = new S3('/root', $key, $secret, $bucket, S3::EU_WEST_1, S3::ACL_PRIVATE); $destination = $device->getPath('largefile.mp4'); $this->assertTrue($this->object->transfer($path, $destination, $device )); From a7b838430b5e0d1e17816f427eb814c405c66828 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 2 Mar 2023 00:46:43 +0000 Subject: [PATCH 16/49] fix formatting --- src/Storage/Device.php | 34 ++++++------ src/Storage/Device/Local.php | 86 ++++++++++++++++-------------- src/Storage/Device/S3.php | 41 +++++++------- tests/Storage/Device/LocalTest.php | 19 ++++--- tests/Storage/S3Base.php | 17 +++--- 5 files changed, 102 insertions(+), 95 deletions(-) diff --git a/src/Storage/Device.php b/src/Storage/Device.php index 27b4b745..9bccd279 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -4,7 +4,6 @@ abstract class Device { - /** * Max chunk size while transfering file from one device to another */ @@ -12,20 +11,22 @@ abstract class Device /** * Set Transfer Chunk Size - * - * @param int $chunkSize + * + * @param int $chunkSize * @return void */ - public function setTransferChunkSize(int $chunkSize): void { + public function setTransferChunkSize(int $chunkSize): void + { $this->transferChunkSize = $chunkSize; } /** * Get Transfer Chunk Size - * + * * @return int */ - public function getTransferChunkSize(): int { + public function getTransferChunkSize(): int + { return $this->transferChunkSize; } @@ -99,16 +100,15 @@ abstract public function upload(string $source, string $path, int $chunk = 1, in * Upload file contents to desired destination in the selected disk. * return number of chunks uploaded or 0 if it fails. * - * @param string $data - * @param string $path - * @param string $contentType + * @param string $data + * @param string $path + * @param string $contentType * @param int chunk * @param int chunks - * @param array $metadata + * @param array $metadata + * @return int * * @throws \Exception - * - * @return int */ abstract public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int; @@ -134,10 +134,9 @@ abstract public function read(string $path, int $offset = 0, int $length = null) /** * Transfer * - * @param string $path - * @param string $destination - * @param Device $device - * + * @param string $path + * @param string $destination + * @param Device $device * @return string */ abstract public function transfer(string $path, string $destination, Device $device): bool; @@ -162,9 +161,10 @@ abstract public function write(string $path, string $data, string $contentType): */ public function move(string $source, string $target): bool { - if($this->transfer($source, $target, $this)) { + if ($this->transfer($source, $target, $this)) { return $this->delete($source); } + return false; } diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index ccca4eea..e10affd5 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -95,8 +95,8 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; $this->createDirectoryToPath($tmp); - if(!file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { - throw new Exception('Can\'t write chunk log ' . $tmp); + if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { + throw new Exception('Can\'t write chunk log '.$tmp); } $chunkLogs = file($tmp); @@ -112,6 +112,7 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks if ($chunks === $chunksReceived) { $this->joinChunks($path, $chunks); + return $chunksReceived; } @@ -124,63 +125,66 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks * Upload file contents to desired destination in the selected disk. * return number of chunks uploaded or 0 if it fails. * - * @param string $source - * @param string $path - * @param string $contentType + * @param string $source + * @param string $path + * @param string $contentType * @param int chunk * @param int chunks - * @param array $metadata + * @param array $metadata + * @return int * * @throws \Exception - * - * @return int */ public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { $this->createDirectoryToPath($path); - if($chunks === 1) { - if(!\file_put_contents($path, $data)) { - throw new Exception('Can\'t write file ' . $path); + if ($chunks === 1) { + if (! \file_put_contents($path, $data)) { + throw new Exception('Can\'t write file '.$path); } + return $chunks; } - $tmp = \dirname($path) . DIRECTORY_SEPARATOR . 'tmp_' . \basename($path) . DIRECTORY_SEPARATOR . \basename($path) . '_chunks.log'; + $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; $this->createDirectoryToPath($tmp); - if(!file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { - throw new Exception('Can\'t write chunk log ' . $tmp); + if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { + throw new Exception('Can\'t write chunk log '.$tmp); } $chunkLogs = file($tmp); - if(!$chunkLogs) { - throw new Exception('Unable to read chunk log ' . $tmp); + if (! $chunkLogs) { + throw new Exception('Unable to read chunk log '.$tmp); } $chunksReceived = count(file($tmp)); - if(!\file_put_contents(dirname($tmp) . DIRECTORY_SEPARATOR . pathinfo($path, PATHINFO_FILENAME) . '.part.' . $chunk, $data)) { - throw new Exception('Failed to write chunk ' . $chunk); + if (! \file_put_contents(dirname($tmp).DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$chunk, $data)) { + throw new Exception('Failed to write chunk '.$chunk); } - + if ($chunks === $chunksReceived) { $this->joinChunks($path, $chunks); + return $chunksReceived; } + return $chunksReceived; } - private function joinChunks(string $path, int $chunks) { - $tmp = \dirname($path) . DIRECTORY_SEPARATOR . 'tmp_' . \basename($path) . DIRECTORY_SEPARATOR . \basename($path) . '_chunks.log'; - for($i = 1; $i <= $chunks; $i++) { - $part = dirname($tmp) . DIRECTORY_SEPARATOR . pathinfo($path, PATHINFO_FILENAME) . '.part.'. $i; + private function joinChunks(string $path, int $chunks) + { + $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; + for ($i = 1; $i <= $chunks; $i++) { + $part = dirname($tmp).DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$i; $data = file_get_contents($part); - if(!$data) { - throw new Exception('Failed to read chunk ' . $part); + if (! $data) { + throw new Exception('Failed to read chunk '.$part); } - if(!file_put_contents($path, $data, FILE_APPEND)) { - throw new Exception('Failed to append chunk ' . $part); + if (! file_put_contents($path, $data, FILE_APPEND)) { + throw new Exception('Failed to append chunk '.$part); } \unlink($part); } @@ -189,13 +193,15 @@ private function joinChunks(string $path, int $chunks) { /** * Create Directory to Path - * + * * Create's missing directory of the path, returns true on success and false on failure - * + * * @param string path + * * @throws Exception */ - private function createDirectoryToPath(string $path): void { + private function createDirectoryToPath(string $path): void + { if (! \file_exists(\dirname($path))) { // Checks if directory path to file exists if (! @\mkdir(\dirname($path), 0755, true)) { throw new Exception('Can\'t create directory: '.\dirname($path)); @@ -206,30 +212,32 @@ private function createDirectoryToPath(string $path): void { /** * Transfer * - * @param string $path - * @param string $destination - * @param Device $device - * + * @param string $path + * @param string $destination + * @param Device $device * @return string */ - public function transfer(string $path, string $destination, Device $device): bool { + public function transfer(string $path, string $destination, Device $device): bool + { $size = $this->getFileSize($path); $contentType = $this->getFileMimeType($path); - if($size <= $this->transferChunkSize) { + if ($size <= $this->transferChunkSize) { $source = $this->read($path); + return $device->write($destination, $source, $contentType); } $totalChunks = \ceil($size / $this->transferChunkSize); $counter = 0; $metadata = ['content_type' => $contentType]; - for($counter; $counter < $totalChunks; $counter++) { + for ($counter; $counter < $totalChunks; $counter++) { $start = $counter * $this->transferChunkSize; $data = $this->read($path, $start, $this->transferChunkSize); - $device->uploadData($data, $destination, $contentType, $counter+1, $totalChunks, $metadata); + $device->uploadData($data, $destination, $contentType, $counter + 1, $totalChunks, $metadata); } - return true; + + return true; } /** diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index f0030fb1..487a25a8 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -232,24 +232,23 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks * Upload file contents to desired destination in the selected disk. * return number of chunks uploaded or 0 if it fails. * - * @param string $source - * @param string $path - * @param string $contentType + * @param string $source + * @param string $path + * @param string $contentType * @param int chunk * @param int chunks - * @param array $metadata + * @param array $metadata + * @return int * * @throws \Exception - * - * @return int */ public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { - if($chunk == 1 && $chunks == 1) { + if ($chunk == 1 && $chunks == 1) { return $this->write($path, $data, $contentType); } $uploadId = $metadata['uploadId'] ?? null; - if(empty($uploadId)) { + if (empty($uploadId)) { $uploadId = $this->createMultipartUpload($path, $contentType); $metadata['uploadId'] = $uploadId; } @@ -269,31 +268,33 @@ public function uploadData(string $data, string $path, string $contentType, int /** * Transfer * - * @param string $path - * @param string $destination - * @param Device $device - * + * @param string $path + * @param string $destination + * @param Device $device * @return string */ - public function transfer(string $path, string $destination, Device $device): bool { + public function transfer(string $path, string $destination, Device $device): bool + { $response = $this->getInfo($path); - $size = (int)($response['content-length'] ?? 0); + $size = (int) ($response['content-length'] ?? 0); $contentType = $response['content-type'] ?? ''; - if($size <= $this->transferChunkSize) { + if ($size <= $this->transferChunkSize) { $source = $this->read($path); + return $device->write($destination, $source, $contentType); } $totalChunks = \ceil($size / $this->transferChunkSize); $counter = 0; $metadata = ['content_type' => $contentType]; - for($counter; $counter < $totalChunks; $counter++) { + for ($counter; $counter < $totalChunks; $counter++) { $start = $counter * $this->transferChunkSize; $data = $this->read($path, $start, $this->transferChunkSize); - $device->uploadData($data, $destination, $contentType, $counter+1, $totalChunks, $metadata); + $device->uploadData($data, $destination, $contentType, $counter + 1, $totalChunks, $metadata); } - return true; + + return true; } /** @@ -331,10 +332,10 @@ protected function createMultipartUpload(string $path, string $contentType): str * * @throws \Exception */ - protected function uploadPart(string $data, string $path, string $contentType, int $chunk, string $uploadId) : string + protected function uploadPart(string $data, string $path, string $contentType, int $chunk, string $uploadId): string { $uri = $path !== '' ? '/'.\str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; - + $this->headers['content-type'] = $contentType; $this->headers['content-md5'] = \base64_encode(md5($data, true)); $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $data); diff --git a/tests/Storage/Device/LocalTest.php b/tests/Storage/Device/LocalTest.php index 98af9a80..846e366d 100644 --- a/tests/Storage/Device/LocalTest.php +++ b/tests/Storage/Device/LocalTest.php @@ -3,8 +3,8 @@ namespace Utopia\Tests\Storage\Device; use PHPUnit\Framework\TestCase; -use Utopia\Storage\Device\S3; use Utopia\Storage\Device\Local; +use Utopia\Storage\Device\S3; class LocalTest extends TestCase { @@ -250,11 +250,11 @@ public function testPartitionTotalSpace() /** * @depends testPartUpload */ - public function testTransferLarge($path) { - + public function testTransferLarge($path) + { // chunked file $this->object->setTransferChunkSize(10000000); //10 mb - + $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; $bucket = 'appwrite-test-bucket'; @@ -262,25 +262,24 @@ public function testTransferLarge($path) { $device = new S3('/root', $key, $secret, $bucket, S3::EU_WEST_1, S3::ACL_PRIVATE); $destination = $device->getPath('largefile.mp4'); - $this->assertTrue($this->object->transfer($path, $destination, $device )); + $this->assertTrue($this->object->transfer($path, $destination, $device)); $this->assertTrue($device->exists($destination)); $this->assertEquals($device->getFileMimeType($destination), 'video/mp4'); $device->delete($destination); $this->object->delete($path); - } - public function testTransferSmall() { - + public function testTransferSmall() + { $this->object->setTransferChunkSize(10000000); //10 mb - + $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; $bucket = 'appwrite-test-bucket'; $device = new S3('/root', $key, $secret, $bucket, S3::EU_WEST_1, S3::ACL_PRIVATE); - + $path = $this->object->getPath('text-for-read.txt'); $this->object->write($path, 'Hello World'); diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index 7a25d521..4da0c94c 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -259,29 +259,28 @@ public function testPartRead($path) /** * @depends testPartUpload */ - public function testTransferLarge($path) { - + public function testTransferLarge($path) + { // chunked file $this->object->setTransferChunkSize(10000000); //10 mb - $device = new Local(__DIR__ . '/../resources/disk-a'); + $device = new Local(__DIR__.'/../resources/disk-a'); $destination = $device->getPath('largefile.mp4'); - $this->assertTrue($this->object->transfer($path, $destination, $device )); + $this->assertTrue($this->object->transfer($path, $destination, $device)); $this->assertTrue($device->exists($destination)); $this->assertEquals($device->getFileMimeType($destination), 'video/mp4'); $device->delete($destination); $this->object->delete($path); - } - public function testTransferSmall() { - + public function testTransferSmall() + { $this->object->setTransferChunkSize(10000000); //10 mb - $device = new Local(__DIR__ . '/../resources/disk-a'); - + $device = new Local(__DIR__.'/../resources/disk-a'); + $path = $this->object->getPath('text-for-read.txt'); $this->object->write($path, 'Hello World', 'text/plain'); From 2ad603be850e6493d3981af9d92126dc0b8f4c60 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 2 Mar 2023 01:51:59 +0000 Subject: [PATCH 17/49] added check to make sure file exists --- src/Storage/Device/Local.php | 3 +++ src/Storage/Device/S3.php | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index e10affd5..57155c7c 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -219,6 +219,9 @@ private function createDirectoryToPath(string $path): void */ public function transfer(string $path, string $destination, Device $device): bool { + if (! $this->exists($path)) { + throw new Exception('File Not Found'); + } $size = $this->getFileSize($path); $contentType = $this->getFileMimeType($path); diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 487a25a8..3f78cd31 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -275,7 +275,12 @@ public function uploadData(string $data, string $path, string $contentType, int */ public function transfer(string $path, string $destination, Device $device): bool { - $response = $this->getInfo($path); + $response = []; + try { + $response = $this->getInfo($path); + } catch (\Throwable $e) { + throw new Exception('File not found'); + } $size = (int) ($response['content-length'] ?? 0); $contentType = $response['content-type'] ?? ''; From 851d515366436b0e7f40e8da52b39b28f0fa70bb Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 3 Mar 2023 00:25:41 +0000 Subject: [PATCH 18/49] update review suggestions --- src/Storage/Device.php | 2 +- src/Storage/Device/Local.php | 26 ++++---------------------- src/Storage/Device/S3.php | 3 +-- 3 files changed, 6 insertions(+), 25 deletions(-) diff --git a/src/Storage/Device.php b/src/Storage/Device.php index 9bccd279..c084589e 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -137,7 +137,7 @@ abstract public function read(string $path, int $offset = 0, int $length = null) * @param string $path * @param string $destination * @param Device $device - * @return string + * @return bool */ abstract public function transfer(string $path, string $destination, Device $device): bool; diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 57155c7c..79d22e5b 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -82,7 +82,7 @@ public function getPath(string $filename, string $prefix = null): string */ public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { - $this->createDirectoryToPath($path); + $this->createDirectory($path); //move_uploaded_file() verifies the file is not tampered with if ($chunks === 1) { @@ -94,7 +94,7 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks } $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; - $this->createDirectoryToPath($tmp); + $this->createDirectory($tmp); if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { throw new Exception('Can\'t write chunk log '.$tmp); } @@ -137,7 +137,7 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks */ public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { - $this->createDirectoryToPath($path); + $this->createDirectory($path); if ($chunks === 1) { if (! \file_put_contents($path, $data)) { @@ -148,7 +148,7 @@ public function uploadData(string $data, string $path, string $contentType, int } $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; - $this->createDirectoryToPath($tmp); + $this->createDirectory($tmp); if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { throw new Exception('Can\'t write chunk log '.$tmp); } @@ -191,24 +191,6 @@ private function joinChunks(string $path, int $chunks) \unlink($tmp); } - /** - * Create Directory to Path - * - * Create's missing directory of the path, returns true on success and false on failure - * - * @param string path - * - * @throws Exception - */ - private function createDirectoryToPath(string $path): void - { - if (! \file_exists(\dirname($path))) { // Checks if directory path to file exists - if (! @\mkdir(\dirname($path), 0755, true)) { - throw new Exception('Can\'t create directory: '.\dirname($path)); - } - } - } - /** * Transfer * diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 3f78cd31..ef30466f 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -291,9 +291,8 @@ public function transfer(string $path, string $destination, Device $device): boo } $totalChunks = \ceil($size / $this->transferChunkSize); - $counter = 0; $metadata = ['content_type' => $contentType]; - for ($counter; $counter < $totalChunks; $counter++) { + for ($counter = 0; $counter < $totalChunks; $counter++) { $start = $counter * $this->transferChunkSize; $data = $this->read($path, $start, $this->transferChunkSize); $device->uploadData($data, $destination, $contentType, $counter + 1, $totalChunks, $metadata); From e22aaf64c915eb65910dd2c6689d3f5a6f7c8b23 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 3 Mar 2023 00:27:21 +0000 Subject: [PATCH 19/49] added description --- src/Storage/Device.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storage/Device.php b/src/Storage/Device.php index c084589e..8b47c9ad 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -133,7 +133,8 @@ abstract public function read(string $path, int $offset = 0, int $length = null) /** * Transfer - * + * Transfer a file from current device to destination device. + * * @param string $path * @param string $destination * @param Device $device From 7eb73332d1ae900a0ff6ee79aab18c384ad61dd7 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 3 Mar 2023 00:45:59 +0000 Subject: [PATCH 20/49] fix create directory error --- src/Storage/Device/Local.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 79d22e5b..80d2a4c4 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -82,7 +82,7 @@ public function getPath(string $filename, string $prefix = null): string */ public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { - $this->createDirectory($path); + $this->createDirectory(\dirname($path)); //move_uploaded_file() verifies the file is not tampered with if ($chunks === 1) { @@ -94,7 +94,7 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks } $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; - $this->createDirectory($tmp); + $this->createDirectory(\dirname($tmp)); if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { throw new Exception('Can\'t write chunk log '.$tmp); } @@ -137,7 +137,7 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks */ public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { - $this->createDirectory($path); + $this->createDirectory(\dirname($path)); if ($chunks === 1) { if (! \file_put_contents($path, $data)) { @@ -148,7 +148,7 @@ public function uploadData(string $data, string $path, string $contentType, int } $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; - $this->createDirectory($tmp); + $this->createDirectory(\dirname($tmp)); if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { throw new Exception('Can\'t write chunk log '.$tmp); } From d318c7000619cb8cda2f85429cb78fc56a6ee143 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 18 Jul 2023 09:11:20 +0000 Subject: [PATCH 21/49] fix formatting --- composer.lock | 248 +++++++++++------- src/Storage/Compression/Algorithms/XZ.php | 6 +- src/Storage/Device.php | 2 +- .../Storage/Compression/Algorithms/XZTest.php | 26 +- tests/Storage/Device/LocalTest.php | 2 +- 5 files changed, 168 insertions(+), 116 deletions(-) diff --git a/composer.lock b/composer.lock index 56f1adf2..f3d8f364 100644 --- a/composer.lock +++ b/composer.lock @@ -4,27 +4,28 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "0e15ac0ae89a77c7420f825af10e33fc", + "content-hash": "31a62f29d84509b2ba01e75fd652d6ba", "packages": [ { "name": "utopia-php/framework", - "version": "0.27.0", + "version": "0.28.4", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "b8d0447f5c98291d7759db05460ecced29a0f9ee" + "reference": "98c5469efe195aeecc63745dbf8e2f357f8cedac" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/b8d0447f5c98291d7759db05460ecced29a0f9ee", - "reference": "b8d0447f5c98291d7759db05460ecced29a0f9ee", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/98c5469efe195aeecc63745dbf8e2f357f8cedac", + "reference": "98c5469efe195aeecc63745dbf8e2f357f8cedac", "shasum": "" }, "require": { - "php": ">=8.0.0" + "php": ">=8.0" }, "require-dev": { "laravel/pint": "^1.2", + "phpstan/phpstan": "1.9.x-dev", "phpunit/phpunit": "^9.5.25", "vimeo/psalm": "4.27.0" }, @@ -46,9 +47,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.27.0" + "source": "https://github.com/utopia-php/framework/tree/0.28.4" }, - "time": "2023-01-29T05:36:17+00:00" + "time": "2023-06-03T14:09:22+00:00" } ], "packages-dev": [ @@ -475,18 +476,66 @@ }, "time": "2019-12-04T15:06:13+00:00" }, + { + "name": "doctrine/deprecations", + "version": "1.1.x-dev", + "source": { + "type": "git", + "url": "https://github.com/doctrine/deprecations.git", + "reference": "eac01fbdf4e4a3635169b056608730ec0930e188" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/doctrine/deprecations/zipball/eac01fbdf4e4a3635169b056608730ec0930e188", + "reference": "eac01fbdf4e4a3635169b056608730ec0930e188", + "shasum": "" + }, + "require": { + "php": "^7.1 || ^8.0" + }, + "require-dev": { + "doctrine/coding-standard": "^9", + "phpstan/phpstan": "1.4.10 || 1.10.15", + "phpstan/phpstan-phpunit": "^1.0", + "phpunit/phpunit": "^7.5 || ^8.5 || ^9.5", + "psalm/plugin-phpunit": "0.18.4", + "psr/log": "^1 || ^2 || ^3", + "vimeo/psalm": "4.30.0 || 5.12.0" + }, + "suggest": { + "psr/log": "Allows logging deprecations via PSR-3 logger implementation" + }, + "default-branch": true, + "type": "library", + "autoload": { + "psr-4": { + "Doctrine\\Deprecations\\": "lib/Doctrine/Deprecations" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A small layer on top of trigger_error(E_USER_DEPRECATED) or PSR-3 logging with options to disable all deprecations or selectively for packages.", + "homepage": "https://www.doctrine-project.org/", + "support": { + "issues": "https://github.com/doctrine/deprecations/issues", + "source": "https://github.com/doctrine/deprecations/tree/1.1.x" + }, + "time": "2023-06-07T19:57:14+00:00" + }, { "name": "doctrine/instantiator", "version": "2.0.x-dev", "source": { "type": "git", "url": "https://github.com/doctrine/instantiator.git", - "reference": "d6eef505a6c46e963e54bf73bb9de43cdea70821" + "reference": "3aa0a786dd096763a5e4be2cc3bacb774859552a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/instantiator/zipball/d6eef505a6c46e963e54bf73bb9de43cdea70821", - "reference": "d6eef505a6c46e963e54bf73bb9de43cdea70821", + "url": "https://api.github.com/repos/doctrine/instantiator/zipball/3aa0a786dd096763a5e4be2cc3bacb774859552a", + "reference": "3aa0a786dd096763a5e4be2cc3bacb774859552a", "shasum": "" }, "require": { @@ -499,7 +548,7 @@ "phpbench/phpbench": "^1.2", "phpstan/phpstan": "^1.9.4", "phpstan/phpstan-phpunit": "^1.3", - "phpunit/phpunit": "^9.5.27", + "phpunit/phpunit": "^10.1.0", "vimeo/psalm": "^5.4" }, "default-branch": true, @@ -544,7 +593,7 @@ "type": "tidelift" } ], - "time": "2023-01-04T15:42:40+00:00" + "time": "2023-05-30T06:43:41+00:00" }, { "name": "felixfbecker/advanced-json-rpc", @@ -720,12 +769,12 @@ "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "14daed4296fae74d9e3201d2c4925d1acb7aa614" + "reference": "928a96f585b86224ebc78f8f09d0482cf15b04f5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/14daed4296fae74d9e3201d2c4925d1acb7aa614", - "reference": "14daed4296fae74d9e3201d2c4925d1acb7aa614", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/928a96f585b86224ebc78f8f09d0482cf15b04f5", + "reference": "928a96f585b86224ebc78f8f09d0482cf15b04f5", "shasum": "" }, "require": { @@ -733,11 +782,12 @@ }, "conflict": { "doctrine/collections": "<1.6.8", - "doctrine/common": "<2.13.3 || >=3,<3.2.2" + "doctrine/common": "<2.13.3 || >=3 <3.2.2" }, "require-dev": { "doctrine/collections": "^1.6.8", "doctrine/common": "^2.13.3 || ^3.2.2", + "phpspec/prophecy": "^1.10", "phpunit/phpunit": "^7.5.20 || ^8.5.23 || ^9.5.13" }, "default-branch": true, @@ -764,7 +814,7 @@ ], "support": { "issues": "https://github.com/myclabs/DeepCopy/issues", - "source": "https://github.com/myclabs/DeepCopy/tree/1.11.0" + "source": "https://github.com/myclabs/DeepCopy/tree/1.x" }, "funding": [ { @@ -772,7 +822,7 @@ "type": "tidelift" } ], - "time": "2022-03-03T13:19:32+00:00" + "time": "2023-03-08T17:24:01+00:00" }, { "name": "netresearch/jsonmapper", @@ -831,12 +881,12 @@ "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "570e980a201d8ed0236b0a62ddf2c9cbb2034039" + "reference": "19526a33fb561ef417e822e85f08a00db4059c17" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/570e980a201d8ed0236b0a62ddf2c9cbb2034039", - "reference": "570e980a201d8ed0236b0a62ddf2c9cbb2034039", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/19526a33fb561ef417e822e85f08a00db4059c17", + "reference": "19526a33fb561ef417e822e85f08a00db4059c17", "shasum": "" }, "require": { @@ -878,9 +928,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.3" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.16.0" }, - "time": "2023-01-16T22:05:37+00:00" + "time": "2023-06-25T14:52:30+00:00" }, { "name": "openlss/lib-array2xml", @@ -941,12 +991,12 @@ "source": { "type": "git", "url": "https://github.com/phar-io/manifest.git", - "reference": "36d8a21e851a9512db2b086dc5ac2c61308f0138" + "reference": "67729272c564ab9f953c81f48db44e8b1cb1e1c3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phar-io/manifest/zipball/36d8a21e851a9512db2b086dc5ac2c61308f0138", - "reference": "36d8a21e851a9512db2b086dc5ac2c61308f0138", + "url": "https://api.github.com/repos/phar-io/manifest/zipball/67729272c564ab9f953c81f48db44e8b1cb1e1c3", + "reference": "67729272c564ab9f953c81f48db44e8b1cb1e1c3", "shasum": "" }, "require": { @@ -955,7 +1005,7 @@ "ext-phar": "*", "ext-xmlwriter": "*", "phar-io/version": "^3.0.1", - "php": "^7.2 || ^8.0" + "php": "^7.3 || ^8.0" }, "default-branch": true, "type": "library", @@ -1001,7 +1051,7 @@ "type": "github" } ], - "time": "2022-02-21T19:55:33+00:00" + "time": "2023-06-01T14:19:47+00:00" }, { "name": "phar-io/version", @@ -1113,12 +1163,12 @@ "source": { "type": "git", "url": "https://github.com/phpDocumentor/ReflectionDocBlock.git", - "reference": "566af9fb94c556de91562fcfcbc392f66680111b" + "reference": "7b217217725dc991a0ae7b995041cee1d5019561" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/ReflectionDocBlock/zipball/566af9fb94c556de91562fcfcbc392f66680111b", - "reference": "566af9fb94c556de91562fcfcbc392f66680111b", + "url": "https://api.github.com/repos/phpDocumentor/ReflectionDocBlock/zipball/7b217217725dc991a0ae7b995041cee1d5019561", + "reference": "7b217217725dc991a0ae7b995041cee1d5019561", "shasum": "" }, "require": { @@ -1169,7 +1219,7 @@ "issues": "https://github.com/phpDocumentor/ReflectionDocBlock/issues", "source": "https://github.com/phpDocumentor/ReflectionDocBlock/tree/master" }, - "time": "2022-11-19T20:28:46+00:00" + "time": "2023-03-12T10:50:44+00:00" }, { "name": "phpdocumentor/type-resolver", @@ -1177,15 +1227,16 @@ "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "06f36c92b434ac686de06b6563e88046943bccbe" + "reference": "b2fe4d22a5426f38e014855322200b97b5362c0d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/06f36c92b434ac686de06b6563e88046943bccbe", - "reference": "06f36c92b434ac686de06b6563e88046943bccbe", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/b2fe4d22a5426f38e014855322200b97b5362c0d", + "reference": "b2fe4d22a5426f38e014855322200b97b5362c0d", "shasum": "" }, "require": { + "doctrine/deprecations": "^1.0", "php": "^7.4 || ^8.0", "phpdocumentor/reflection-common": "^2.0", "phpstan/phpdoc-parser": "^1.13" @@ -1225,28 +1276,30 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.x" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.7.2" }, - "time": "2022-12-16T10:25:14+00:00" + "time": "2023-05-30T18:13:47+00:00" }, { "name": "phpstan/phpdoc-parser", - "version": "1.16.1", + "version": "1.22.x-dev", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "e27e92d939e2e3636f0a1f0afaba59692c0bf571" + "reference": "2108d702baa4883362a8824def66b96733b8cf82" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/e27e92d939e2e3636f0a1f0afaba59692c0bf571", - "reference": "e27e92d939e2e3636f0a1f0afaba59692c0bf571", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/2108d702baa4883362a8824def66b96733b8cf82", + "reference": "2108d702baa4883362a8824def66b96733b8cf82", "shasum": "" }, "require": { "php": "^7.2 || ^8.0" }, "require-dev": { + "doctrine/annotations": "^2.0", + "nikic/php-parser": "^4.15", "php-parallel-lint/php-parallel-lint": "^1.2", "phpstan/extension-installer": "^1.0", "phpstan/phpstan": "^1.5", @@ -1255,6 +1308,7 @@ "phpunit/phpunit": "^9.5", "symfony/process": "^5.2" }, + "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -1270,9 +1324,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.16.1" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.22.x" }, - "time": "2023-02-07T18:11:17+00:00" + "time": "2023-07-17T07:08:06+00:00" }, { "name": "phpunit/php-code-coverage", @@ -1280,19 +1334,19 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "b5f3416da036fb951a557518e8304b3595ff966a" + "reference": "f1564623dc5330085a73cefcf824b48cdec997b2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/b5f3416da036fb951a557518e8304b3595ff966a", - "reference": "b5f3416da036fb951a557518e8304b3595ff966a", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/f1564623dc5330085a73cefcf824b48cdec997b2", + "reference": "f1564623dc5330085a73cefcf824b48cdec997b2", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", "ext-xmlwriter": "*", - "nikic/php-parser": "^4.14", + "nikic/php-parser": "^4.15", "php": ">=7.3", "phpunit/php-file-iterator": "^3.0.3", "phpunit/php-text-template": "^2.0.2", @@ -1307,8 +1361,8 @@ "phpunit/phpunit": "^9.3" }, "suggest": { - "ext-pcov": "*", - "ext-xdebug": "*" + "ext-pcov": "PHP extension that provides line coverage", + "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" }, "type": "library", "extra": { @@ -1341,6 +1395,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", + "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2" }, "funding": [ @@ -1349,7 +1404,7 @@ "type": "github" } ], - "time": "2023-02-18T16:27:54+00:00" + "time": "2023-07-17T04:56:01+00:00" }, { "name": "phpunit/php-file-iterator", @@ -1598,12 +1653,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "c1bf2dd252dec994ee27d804c038a98fd0d1d940" + "reference": "19cadf73b15656502e34bc4fc00e197fcc94802c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c1bf2dd252dec994ee27d804c038a98fd0d1d940", - "reference": "c1bf2dd252dec994ee27d804c038a98fd0d1d940", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/19cadf73b15656502e34bc4fc00e197fcc94802c", + "reference": "19cadf73b15656502e34bc4fc00e197fcc94802c", "shasum": "" }, "require": { @@ -1636,8 +1691,8 @@ "sebastian/version": "^3.0.2" }, "suggest": { - "ext-soap": "*", - "ext-xdebug": "*" + "ext-soap": "To be able to generate mocks based on WSDL files", + "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" }, "bin": [ "phpunit" @@ -1676,6 +1731,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", + "security": "https://github.com/sebastianbergmann/phpunit/security/policy", "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6" }, "funding": [ @@ -1692,7 +1748,7 @@ "type": "tidelift" } ], - "time": "2023-02-18T16:26:49+00:00" + "time": "2023-07-17T05:01:41+00:00" }, { "name": "psr/container", @@ -2098,16 +2154,16 @@ }, { "name": "sebastian/diff", - "version": "4.0.4", + "version": "4.0.x-dev", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/diff.git", - "reference": "3461e3fccc7cfdfc2720be910d3bd73c69be590d" + "reference": "74be17022044ebaaecfdf0c5cd504fc9cd5a7131" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/3461e3fccc7cfdfc2720be910d3bd73c69be590d", - "reference": "3461e3fccc7cfdfc2720be910d3bd73c69be590d", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/74be17022044ebaaecfdf0c5cd504fc9cd5a7131", + "reference": "74be17022044ebaaecfdf0c5cd504fc9cd5a7131", "shasum": "" }, "require": { @@ -2152,7 +2208,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/diff/issues", - "source": "https://github.com/sebastianbergmann/diff/tree/4.0.4" + "source": "https://github.com/sebastianbergmann/diff/tree/4.0" }, "funding": [ { @@ -2160,7 +2216,7 @@ "type": "github" } ], - "time": "2020-10-26T13:10:38+00:00" + "time": "2023-05-07T05:35:17+00:00" }, { "name": "sebastian/environment", @@ -2604,12 +2660,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/resource-operations.git", - "reference": "e1157eac767e4dc4ae40dd9aab7fb4de6e56bd32" + "reference": "20bdda85c7c585ab265c0c37ec052a019bae29c4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/resource-operations/zipball/e1157eac767e4dc4ae40dd9aab7fb4de6e56bd32", - "reference": "e1157eac767e4dc4ae40dd9aab7fb4de6e56bd32", + "url": "https://api.github.com/repos/sebastianbergmann/resource-operations/zipball/20bdda85c7c585ab265c0c37ec052a019bae29c4", + "reference": "20bdda85c7c585ab265c0c37ec052a019bae29c4", "shasum": "" }, "require": { @@ -2651,7 +2707,7 @@ "type": "github" } ], - "time": "2023-02-08T06:53:39+00:00" + "time": "2023-03-25T08:11:39+00:00" }, { "name": "sebastian/type", @@ -2768,12 +2824,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "bc04b7a3c23fd36e4b165afe7947b3d00989677b" + "reference": "560fc3ed7a43e6d30ea94a07d77f9a60b8ed0fb8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/bc04b7a3c23fd36e4b165afe7947b3d00989677b", - "reference": "bc04b7a3c23fd36e4b165afe7947b3d00989677b", + "url": "https://api.github.com/repos/symfony/console/zipball/560fc3ed7a43e6d30ea94a07d77f9a60b8ed0fb8", + "reference": "560fc3ed7a43e6d30ea94a07d77f9a60b8ed0fb8", "shasum": "" }, "require": { @@ -2838,7 +2894,7 @@ "homepage": "https://symfony.com", "keywords": [ "cli", - "command line", + "command-line", "console", "terminal" ], @@ -2859,7 +2915,7 @@ "type": "tidelift" } ], - "time": "2023-02-14T14:10:40+00:00" + "time": "2023-05-26T05:13:16+00:00" }, { "name": "symfony/deprecation-contracts", @@ -2867,12 +2923,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3" + "reference": "7c3aff79d10325257a001fcf92d991f24fc967cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/1ee04c65529dea5d8744774d474e7cbd2f1206d3", - "reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/7c3aff79d10325257a001fcf92d991f24fc967cf", + "reference": "7c3aff79d10325257a001fcf92d991f24fc967cf", "shasum": "" }, "require": { @@ -2882,7 +2938,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "3.3-dev" + "dev-main": "3.4-dev" }, "thanks": { "name": "symfony/contracts", @@ -2911,7 +2967,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v3.2.0" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.3.0" }, "funding": [ { @@ -2927,7 +2983,7 @@ "type": "tidelift" } ], - "time": "2022-11-25T10:21:52+00:00" + "time": "2023-05-23T14:45:45+00:00" }, { "name": "symfony/polyfill-ctype", @@ -3433,12 +3489,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "aac98028c69df04ee77eb69b96b86ee51fbf4b75" + "reference": "40da9cc13ec349d9e4966ce18b5fbcd724ab10a4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/aac98028c69df04ee77eb69b96b86ee51fbf4b75", - "reference": "aac98028c69df04ee77eb69b96b86ee51fbf4b75", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/40da9cc13ec349d9e4966ce18b5fbcd724ab10a4", + "reference": "40da9cc13ec349d9e4966ce18b5fbcd724ab10a4", "shasum": "" }, "require": { @@ -3448,14 +3504,11 @@ "conflict": { "ext-psr": "<1.1|>=2" }, - "suggest": { - "symfony/service-implementation": "" - }, "default-branch": true, "type": "library", "extra": { "branch-alias": { - "dev-main": "3.3-dev" + "dev-main": "3.4-dev" }, "thanks": { "name": "symfony/contracts", @@ -3495,7 +3548,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.2.0" + "source": "https://github.com/symfony/service-contracts/tree/v3.3.0" }, "funding": [ { @@ -3511,20 +3564,20 @@ "type": "tidelift" } ], - "time": "2022-11-25T10:21:52+00:00" + "time": "2023-05-23T14:45:45+00:00" }, { "name": "symfony/string", - "version": "6.3.x-dev", + "version": "6.4.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "6515bacc98212164f10db4371fa70ee7b8c70b80" + "reference": "34267d7e5422c9c41808b4e0d251d2414e9d7742" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/6515bacc98212164f10db4371fa70ee7b8c70b80", - "reference": "6515bacc98212164f10db4371fa70ee7b8c70b80", + "url": "https://api.github.com/repos/symfony/string/zipball/34267d7e5422c9c41808b4e0d251d2414e9d7742", + "reference": "34267d7e5422c9c41808b4e0d251d2414e9d7742", "shasum": "" }, "require": { @@ -3538,11 +3591,11 @@ "symfony/translation-contracts": "<2.5" }, "require-dev": { - "symfony/error-handler": "^5.4|^6.0", - "symfony/http-client": "^5.4|^6.0", - "symfony/intl": "^6.2", + "symfony/error-handler": "^5.4|^6.0|^7.0", + "symfony/http-client": "^5.4|^6.0|^7.0", + "symfony/intl": "^6.2|^7.0", "symfony/translation-contracts": "^2.5|^3.0", - "symfony/var-exporter": "^5.4|^6.0" + "symfony/var-exporter": "^5.4|^6.0|^7.0" }, "type": "library", "autoload": { @@ -3581,7 +3634,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/6.3" + "source": "https://github.com/symfony/string/tree/6.4" }, "funding": [ { @@ -3597,7 +3650,7 @@ "type": "tidelift" } ], - "time": "2023-02-14T09:04:20+00:00" + "time": "2023-07-05T09:40:08+00:00" }, { "name": "theseer/tokenizer", @@ -3919,6 +3972,7 @@ "ext-fileinfo": "*", "ext-zlib": "*", "ext-zstd": "*", + "ext-xz": "*", "ext-brotli": "*", "ext-lz4": "*", "ext-snappy": "*", diff --git a/src/Storage/Compression/Algorithms/XZ.php b/src/Storage/Compression/Algorithms/XZ.php index 3bd2c386..aaa79fae 100644 --- a/src/Storage/Compression/Algorithms/XZ.php +++ b/src/Storage/Compression/Algorithms/XZ.php @@ -17,8 +17,7 @@ public function getName(): string /** * Compress. * - * @param string $data - * + * @param string $data * @return string */ public function compress(string $data): string @@ -29,8 +28,7 @@ public function compress(string $data): string /** * Decompress. * - * @param string $data - * + * @param string $data * @return string */ public function decompress(string $data): string diff --git a/src/Storage/Device.php b/src/Storage/Device.php index 8b47c9ad..23027d94 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -134,7 +134,7 @@ abstract public function read(string $path, int $offset = 0, int $length = null) /** * Transfer * Transfer a file from current device to destination device. - * + * * @param string $path * @param string $destination * @param Device $device diff --git a/tests/Storage/Compression/Algorithms/XZTest.php b/tests/Storage/Compression/Algorithms/XZTest.php index 6fec876e..2b55ee01 100644 --- a/tests/Storage/Compression/Algorithms/XZTest.php +++ b/tests/Storage/Compression/Algorithms/XZTest.php @@ -2,8 +2,8 @@ namespace Utopia\Tests\Storage\Compression\Algorithms; -use Utopia\Storage\Compression\Algorithms\XZ; use PHPUnit\Framework\TestCase; +use Utopia\Storage\Compression\Algorithms\XZ; class XZTest extends TestCase { @@ -22,7 +22,7 @@ public function testName() { $this->assertEquals($this->object->getName(), 'xz'); } - + public function testCompressDecompressWithText() { $demo = 'This is a demo string'; @@ -33,13 +33,13 @@ public function testCompressDecompressWithText() $this->assertEquals($demoSize, 21); $this->assertEquals($dataSize, 80); - + $this->assertEquals($this->object->decompress($data), $demo); } - + public function testCompressDecompressWithJPGImage() { - $demo = \file_get_contents(__DIR__ . '/../../../resources/disk-a/kitten-1.jpg'); + $demo = \file_get_contents(__DIR__.'/../../../resources/disk-a/kitten-1.jpg'); $demoSize = mb_strlen($demo, '8bit'); $data = $this->object->compress($demo); @@ -47,18 +47,18 @@ public function testCompressDecompressWithJPGImage() $this->assertEquals($demoSize, 599639); $this->assertEquals($dataSize, 599432); - + $this->assertGreaterThan($dataSize, $demoSize); - + $data = $this->object->decompress($data); $dataSize = mb_strlen($data, '8bit'); - + $this->assertEquals($dataSize, 599639); } - + public function testCompressDecompressWithPNGImage() { - $demo = \file_get_contents(__DIR__ . '/../../../resources/disk-b/kitten-1.png'); + $demo = \file_get_contents(__DIR__.'/../../../resources/disk-b/kitten-1.png'); $demoSize = mb_strlen($demo, '8bit'); $data = $this->object->compress($demo); @@ -66,12 +66,12 @@ public function testCompressDecompressWithPNGImage() $this->assertEquals($demoSize, 3038056); $this->assertEquals($dataSize, 2981000); - + $this->assertGreaterThan($dataSize, $demoSize); - + $data = $this->object->decompress($data); $dataSize = mb_strlen($data, '8bit'); - + $this->assertEquals($dataSize, 3038056); } } diff --git a/tests/Storage/Device/LocalTest.php b/tests/Storage/Device/LocalTest.php index dc7657fd..f5ead457 100644 --- a/tests/Storage/Device/LocalTest.php +++ b/tests/Storage/Device/LocalTest.php @@ -291,7 +291,7 @@ public function testTransferSmall() $this->object->delete($path); $device->delete($destination); } - + public function testDeletePath() { // Test Single Object From 2ee3666511b3da604e7e4d66263f1dbea9f6fb04 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 19 Jul 2023 07:14:15 +0000 Subject: [PATCH 22/49] remove linter as it's added to Github action --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c7438e66..b9c4ab2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,5 +32,4 @@ install: script: - docker ps - docker-compose exec tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite unit - - docker-compose exec tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite e2e - - docker-compose exec tests vendor/bin/psalm --show-info=true \ No newline at end of file + - docker-compose exec tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite e2e \ No newline at end of file From 27d60cd64a3255814698125a128fce1742476b4e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 25 Jul 2023 07:32:22 +0000 Subject: [PATCH 23/49] remove the tmp directory as well --- src/Storage/Device/Local.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index b0bf68af..0e4a0bb5 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -189,6 +189,7 @@ private function joinChunks(string $path, int $chunks) \unlink($part); } \unlink($tmp); + \unlink(dirname($tmp)); } /** From ae190865088491d7f8aa3fc63e6906efe6e42295 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 25 Jul 2023 07:40:02 +0000 Subject: [PATCH 24/49] remove directory --- src/Storage/Device/Local.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 0e4a0bb5..d3b0db99 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -189,7 +189,7 @@ private function joinChunks(string $path, int $chunks) \unlink($part); } \unlink($tmp); - \unlink(dirname($tmp)); + \rmdir(dirname($tmp)); } /** From 0833dff2e0cf68ccd5d413a9a3b203d78f131cec Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 8 Aug 2023 02:47:38 +0000 Subject: [PATCH 25/49] add system library --- composer.json | 3 +- composer.lock | 349 +++++++++++++++++++++++++++++--------------------- 2 files changed, 205 insertions(+), 147 deletions(-) diff --git a/composer.json b/composer.json index 46d12e03..3dde33f0 100644 --- a/composer.json +++ b/composer.json @@ -24,7 +24,8 @@ "ext-lz4": "*", "ext-snappy": "*", "php": ">=8.0", - "utopia-php/framework": "0.*.*" + "utopia-php/framework": "0.*.*", + "utopia-php/system": "0.*.*" }, "require-dev": { "phpunit/phpunit": "^9.3", diff --git a/composer.lock b/composer.lock index f3d8f364..54da72ab 100644 --- a/composer.lock +++ b/composer.lock @@ -4,20 +4,86 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "31a62f29d84509b2ba01e75fd652d6ba", + "content-hash": "024174312546d59cde20dd0dd4e6186d", "packages": [ + { + "name": "laravel/pint", + "version": "v1.2.1", + "source": { + "type": "git", + "url": "https://github.com/laravel/pint.git", + "reference": "e60e2112ee779ce60f253695b273d1646a17d6f1" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/laravel/pint/zipball/e60e2112ee779ce60f253695b273d1646a17d6f1", + "reference": "e60e2112ee779ce60f253695b273d1646a17d6f1", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "ext-tokenizer": "*", + "ext-xml": "*", + "php": "^8.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.11.0", + "illuminate/view": "^9.32.0", + "laravel-zero/framework": "^9.2.0", + "mockery/mockery": "^1.5.1", + "nunomaduro/larastan": "^2.2.0", + "nunomaduro/termwind": "^1.14.0", + "pestphp/pest": "^1.22.1" + }, + "bin": [ + "builds/pint" + ], + "type": "project", + "autoload": { + "psr-4": { + "App\\": "app/", + "Database\\Seeders\\": "database/seeders/", + "Database\\Factories\\": "database/factories/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nuno Maduro", + "email": "enunomaduro@gmail.com" + } + ], + "description": "An opinionated code formatter for PHP.", + "homepage": "https://laravel.com", + "keywords": [ + "format", + "formatter", + "lint", + "linter", + "php" + ], + "support": { + "issues": "https://github.com/laravel/pint/issues", + "source": "https://github.com/laravel/pint" + }, + "time": "2022-11-29T16:25:20+00:00" + }, { "name": "utopia-php/framework", - "version": "0.28.4", + "version": "0.30.0", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "98c5469efe195aeecc63745dbf8e2f357f8cedac" + "reference": "0969d429997996ceade53e477fce31221b415651" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/98c5469efe195aeecc63745dbf8e2f357f8cedac", - "reference": "98c5469efe195aeecc63745dbf8e2f357f8cedac", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/0969d429997996ceade53e477fce31221b415651", + "reference": "0969d429997996ceade53e477fce31221b415651", "shasum": "" }, "require": { @@ -25,9 +91,9 @@ }, "require-dev": { "laravel/pint": "^1.2", - "phpstan/phpstan": "1.9.x-dev", - "phpunit/phpunit": "^9.5.25", - "vimeo/psalm": "4.27.0" + "phpbench/phpbench": "^1.2", + "phpstan/phpstan": "^1.10", + "phpunit/phpunit": "^9.5.25" }, "type": "library", "autoload": { @@ -47,9 +113,66 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.28.4" + "source": "https://github.com/utopia-php/framework/tree/0.30.0" }, - "time": "2023-06-03T14:09:22+00:00" + "time": "2023-08-07T07:55:48+00:00" + }, + { + "name": "utopia-php/system", + "version": "0.7.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/system.git", + "reference": "d385e9521ca3f68aa007ec1cadd11c4dbd871b90" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/system/zipball/d385e9521ca3f68aa007ec1cadd11c4dbd871b90", + "reference": "d385e9521ca3f68aa007ec1cadd11c4dbd871b90", + "shasum": "" + }, + "require": { + "laravel/pint": "1.2.*", + "php": ">=8.0.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.3", + "squizlabs/php_codesniffer": "^3.6", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\System\\": "src/System" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Eldad Fux", + "email": "eldad@appwrite.io" + }, + { + "name": "Torsten Dittmann", + "email": "torsten@appwrite.io" + } + ], + "description": "A simple library for obtaining information about the host's system.", + "keywords": [ + "framework", + "php", + "system", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/system/issues", + "source": "https://github.com/utopia-php/system/tree/0.7.0" + }, + "time": "2023-08-07T09:34:26+00:00" } ], "packages-dev": [ @@ -482,12 +605,12 @@ "source": { "type": "git", "url": "https://github.com/doctrine/deprecations.git", - "reference": "eac01fbdf4e4a3635169b056608730ec0930e188" + "reference": "bdaa697ed9c7f5ee2f7d3b5f9c2a6f2519523cd9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/deprecations/zipball/eac01fbdf4e4a3635169b056608730ec0930e188", - "reference": "eac01fbdf4e4a3635169b056608730ec0930e188", + "url": "https://api.github.com/repos/doctrine/deprecations/zipball/bdaa697ed9c7f5ee2f7d3b5f9c2a6f2519523cd9", + "reference": "bdaa697ed9c7f5ee2f7d3b5f9c2a6f2519523cd9", "shasum": "" }, "require": { @@ -522,7 +645,7 @@ "issues": "https://github.com/doctrine/deprecations/issues", "source": "https://github.com/doctrine/deprecations/tree/1.1.x" }, - "time": "2023-06-07T19:57:14+00:00" + "time": "2023-07-29T16:12:19+00:00" }, { "name": "doctrine/instantiator", @@ -530,12 +653,12 @@ "source": { "type": "git", "url": "https://github.com/doctrine/instantiator.git", - "reference": "3aa0a786dd096763a5e4be2cc3bacb774859552a" + "reference": "40fc8fcd149f6ccff851e767c3fc5ded4e6a96c0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/instantiator/zipball/3aa0a786dd096763a5e4be2cc3bacb774859552a", - "reference": "3aa0a786dd096763a5e4be2cc3bacb774859552a", + "url": "https://api.github.com/repos/doctrine/instantiator/zipball/40fc8fcd149f6ccff851e767c3fc5ded4e6a96c0", + "reference": "40fc8fcd149f6ccff851e767c3fc5ded4e6a96c0", "shasum": "" }, "require": { @@ -593,7 +716,7 @@ "type": "tidelift" } ], - "time": "2023-05-30T06:43:41+00:00" + "time": "2023-07-30T10:00:15+00:00" }, { "name": "felixfbecker/advanced-json-rpc", @@ -697,84 +820,18 @@ }, "time": "2022-06-19T17:15:06+00:00" }, - { - "name": "laravel/pint", - "version": "v1.2.1", - "source": { - "type": "git", - "url": "https://github.com/laravel/pint.git", - "reference": "e60e2112ee779ce60f253695b273d1646a17d6f1" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/e60e2112ee779ce60f253695b273d1646a17d6f1", - "reference": "e60e2112ee779ce60f253695b273d1646a17d6f1", - "shasum": "" - }, - "require": { - "ext-json": "*", - "ext-mbstring": "*", - "ext-tokenizer": "*", - "ext-xml": "*", - "php": "^8.0" - }, - "require-dev": { - "friendsofphp/php-cs-fixer": "^3.11.0", - "illuminate/view": "^9.32.0", - "laravel-zero/framework": "^9.2.0", - "mockery/mockery": "^1.5.1", - "nunomaduro/larastan": "^2.2.0", - "nunomaduro/termwind": "^1.14.0", - "pestphp/pest": "^1.22.1" - }, - "bin": [ - "builds/pint" - ], - "type": "project", - "autoload": { - "psr-4": { - "App\\": "app/", - "Database\\Seeders\\": "database/seeders/", - "Database\\Factories\\": "database/factories/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Nuno Maduro", - "email": "enunomaduro@gmail.com" - } - ], - "description": "An opinionated code formatter for PHP.", - "homepage": "https://laravel.com", - "keywords": [ - "format", - "formatter", - "lint", - "linter", - "php" - ], - "support": { - "issues": "https://github.com/laravel/pint/issues", - "source": "https://github.com/laravel/pint" - }, - "time": "2022-11-29T16:25:20+00:00" - }, { "name": "myclabs/deep-copy", "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "928a96f585b86224ebc78f8f09d0482cf15b04f5" + "reference": "f6f48cfecf52ab791fe18cc1b11d6345512dc4b8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/928a96f585b86224ebc78f8f09d0482cf15b04f5", - "reference": "928a96f585b86224ebc78f8f09d0482cf15b04f5", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/f6f48cfecf52ab791fe18cc1b11d6345512dc4b8", + "reference": "f6f48cfecf52ab791fe18cc1b11d6345512dc4b8", "shasum": "" }, "require": { @@ -822,7 +879,7 @@ "type": "tidelift" } ], - "time": "2023-03-08T17:24:01+00:00" + "time": "2023-07-30T10:01:33+00:00" }, { "name": "netresearch/jsonmapper", @@ -881,12 +938,12 @@ "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "19526a33fb561ef417e822e85f08a00db4059c17" + "reference": "cfc54e30a4f5e5af5f9d9ce86697cfcc5f7e7c24" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/19526a33fb561ef417e822e85f08a00db4059c17", - "reference": "19526a33fb561ef417e822e85f08a00db4059c17", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/cfc54e30a4f5e5af5f9d9ce86697cfcc5f7e7c24", + "reference": "cfc54e30a4f5e5af5f9d9ce86697cfcc5f7e7c24", "shasum": "" }, "require": { @@ -928,9 +985,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.16.0" + "source": "https://github.com/nikic/PHP-Parser/tree/4.x" }, - "time": "2023-06-25T14:52:30+00:00" + "time": "2023-07-30T21:38:32+00:00" }, { "name": "openlss/lib-array2xml", @@ -1227,12 +1284,12 @@ "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "b2fe4d22a5426f38e014855322200b97b5362c0d" + "reference": "07100e65d09fd50608d649fc656cae1c921a2495" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/b2fe4d22a5426f38e014855322200b97b5362c0d", - "reference": "b2fe4d22a5426f38e014855322200b97b5362c0d", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/07100e65d09fd50608d649fc656cae1c921a2495", + "reference": "07100e65d09fd50608d649fc656cae1c921a2495", "shasum": "" }, "require": { @@ -1276,22 +1333,22 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.7.2" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.x" }, - "time": "2023-05-30T18:13:47+00:00" + "time": "2023-07-20T19:57:33+00:00" }, { "name": "phpstan/phpdoc-parser", - "version": "1.22.x-dev", + "version": "1.23.x-dev", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "2108d702baa4883362a8824def66b96733b8cf82" + "reference": "846ae76eef31c6d7790fac9bc399ecee45160b26" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/2108d702baa4883362a8824def66b96733b8cf82", - "reference": "2108d702baa4883362a8824def66b96733b8cf82", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/846ae76eef31c6d7790fac9bc399ecee45160b26", + "reference": "846ae76eef31c6d7790fac9bc399ecee45160b26", "shasum": "" }, "require": { @@ -1324,9 +1381,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.22.x" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.23.1" }, - "time": "2023-07-17T07:08:06+00:00" + "time": "2023-08-03T16:32:59+00:00" }, { "name": "phpunit/php-code-coverage", @@ -1334,12 +1391,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "f1564623dc5330085a73cefcf824b48cdec997b2" + "reference": "9e1baee3a7525530f5600035759ce754a8b3750b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/f1564623dc5330085a73cefcf824b48cdec997b2", - "reference": "f1564623dc5330085a73cefcf824b48cdec997b2", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/9e1baee3a7525530f5600035759ce754a8b3750b", + "reference": "9e1baee3a7525530f5600035759ce754a8b3750b", "shasum": "" }, "require": { @@ -1404,7 +1461,7 @@ "type": "github" } ], - "time": "2023-07-17T04:56:01+00:00" + "time": "2023-08-02T09:00:29+00:00" }, { "name": "phpunit/php-file-iterator", @@ -1653,12 +1710,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "19cadf73b15656502e34bc4fc00e197fcc94802c" + "reference": "6904cfa5f4a6211e759de8ef36fb51fc939a83fa" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/19cadf73b15656502e34bc4fc00e197fcc94802c", - "reference": "19cadf73b15656502e34bc4fc00e197fcc94802c", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/6904cfa5f4a6211e759de8ef36fb51fc939a83fa", + "reference": "6904cfa5f4a6211e759de8ef36fb51fc939a83fa", "shasum": "" }, "require": { @@ -1748,7 +1805,7 @@ "type": "tidelift" } ], - "time": "2023-07-17T05:01:41+00:00" + "time": "2023-08-02T08:56:58+00:00" }, { "name": "psr/container", @@ -2364,12 +2421,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2" + "reference": "bde739e7565280bda77be70044ac1047bc007e34" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/0ca8db5a5fc9c8646244e629625ac486fa286bf2", - "reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/bde739e7565280bda77be70044ac1047bc007e34", + "reference": "bde739e7565280bda77be70044ac1047bc007e34", "shasum": "" }, "require": { @@ -2412,7 +2469,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/global-state/issues", - "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.5" + "source": "https://github.com/sebastianbergmann/global-state/tree/5.0" }, "funding": [ { @@ -2420,7 +2477,7 @@ "type": "github" } ], - "time": "2022-02-14T08:28:10+00:00" + "time": "2023-08-02T09:26:13+00:00" }, { "name": "sebastian/lines-of-code", @@ -2824,12 +2881,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "560fc3ed7a43e6d30ea94a07d77f9a60b8ed0fb8" + "reference": "b504a3d266ad2bb632f196c0936ef2af5ff6e273" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/560fc3ed7a43e6d30ea94a07d77f9a60b8ed0fb8", - "reference": "560fc3ed7a43e6d30ea94a07d77f9a60b8ed0fb8", + "url": "https://api.github.com/repos/symfony/console/zipball/b504a3d266ad2bb632f196c0936ef2af5ff6e273", + "reference": "b504a3d266ad2bb632f196c0936ef2af5ff6e273", "shasum": "" }, "require": { @@ -2915,7 +2972,7 @@ "type": "tidelift" } ], - "time": "2023-05-26T05:13:16+00:00" + "time": "2023-07-19T20:11:33+00:00" }, { "name": "symfony/deprecation-contracts", @@ -2987,7 +3044,7 @@ }, { "name": "symfony/polyfill-ctype", - "version": "dev-main", + "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", @@ -3050,7 +3107,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/main" + "source": "https://github.com/symfony/polyfill-ctype/tree/1.x" }, "funding": [ { @@ -3070,7 +3127,7 @@ }, { "name": "symfony/polyfill-intl-grapheme", - "version": "dev-main", + "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-grapheme.git", @@ -3132,7 +3189,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/main" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/1.x" }, "funding": [ { @@ -3152,7 +3209,7 @@ }, { "name": "symfony/polyfill-intl-normalizer", - "version": "dev-main", + "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-normalizer.git", @@ -3217,7 +3274,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/main" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/1.x" }, "funding": [ { @@ -3237,16 +3294,16 @@ }, { "name": "symfony/polyfill-mbstring", - "version": "dev-main", + "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "f9c7affe77a00ae32ca127ca6833d034e6d33f25" + "reference": "42292d99c55abe617799667f454222c54c60e229" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/f9c7affe77a00ae32ca127ca6833d034e6d33f25", - "reference": "f9c7affe77a00ae32ca127ca6833d034e6d33f25", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/42292d99c55abe617799667f454222c54c60e229", + "reference": "42292d99c55abe617799667f454222c54c60e229", "shasum": "" }, "require": { @@ -3301,7 +3358,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/main" + "source": "https://github.com/symfony/polyfill-mbstring/tree/1.x" }, "funding": [ { @@ -3317,11 +3374,11 @@ "type": "tidelift" } ], - "time": "2023-01-30T17:25:47+00:00" + "time": "2023-07-28T09:04:16+00:00" }, { "name": "symfony/polyfill-php73", - "version": "dev-main", + "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php73.git", @@ -3381,7 +3438,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php73/tree/main" + "source": "https://github.com/symfony/polyfill-php73/tree/1.x" }, "funding": [ { @@ -3401,7 +3458,7 @@ }, { "name": "symfony/polyfill-php80", - "version": "dev-main", + "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", @@ -3465,7 +3522,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/main" + "source": "https://github.com/symfony/polyfill-php80/tree/1.x" }, "funding": [ { @@ -3489,12 +3546,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "40da9cc13ec349d9e4966ce18b5fbcd724ab10a4" + "reference": "a4025a1c812c231d88ed0780e866b0cc644f4a84" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/40da9cc13ec349d9e4966ce18b5fbcd724ab10a4", - "reference": "40da9cc13ec349d9e4966ce18b5fbcd724ab10a4", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/a4025a1c812c231d88ed0780e866b0cc644f4a84", + "reference": "a4025a1c812c231d88ed0780e866b0cc644f4a84", "shasum": "" }, "require": { @@ -3548,7 +3605,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.3.0" + "source": "https://github.com/symfony/service-contracts/tree/main" }, "funding": [ { @@ -3564,7 +3621,7 @@ "type": "tidelift" } ], - "time": "2023-05-23T14:45:45+00:00" + "time": "2023-07-29T13:12:44+00:00" }, { "name": "symfony/string", @@ -3572,12 +3629,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "34267d7e5422c9c41808b4e0d251d2414e9d7742" + "reference": "fe9228ba417441e16f31d36ddad2b3076135f453" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/34267d7e5422c9c41808b4e0d251d2414e9d7742", - "reference": "34267d7e5422c9c41808b4e0d251d2414e9d7742", + "url": "https://api.github.com/repos/symfony/string/zipball/fe9228ba417441e16f31d36ddad2b3076135f453", + "reference": "fe9228ba417441e16f31d36ddad2b3076135f453", "shasum": "" }, "require": { @@ -3650,7 +3707,7 @@ "type": "tidelift" } ], - "time": "2023-07-05T09:40:08+00:00" + "time": "2023-07-27T06:52:43+00:00" }, { "name": "theseer/tokenizer", From 47dc2227957c716dc7a4f24f35b4c5325e565bb4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 8 Aug 2023 02:51:07 +0000 Subject: [PATCH 26/49] check file size against free memory --- src/Storage/Device/S3.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 2c771719..56781595 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -5,6 +5,7 @@ use Exception; use Utopia\Storage\Device; use Utopia\Storage\Storage; +use Utopia\System\System; class S3 extends Device { @@ -223,6 +224,12 @@ public function getPath(string $filename, string $prefix = null): string */ public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { + $fileSize = \filesize($source); + $freeMemory = System::getMemoryFree(); + + if($fileSize > $freeMemory) { + throw new Exception('Trying to load file larger than available memory. Please use chunked upload.'); + } return $this->uploadData(\file_get_contents($source), $path, \mime_content_type($source), $chunk, $chunks, $metadata); } From 79605177db196993fc19235cf61a5c7b757c7d4b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 8 Aug 2023 02:51:52 +0000 Subject: [PATCH 27/49] format --- src/Storage/Device/S3.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 56781595..2e30250f 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -227,9 +227,10 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks $fileSize = \filesize($source); $freeMemory = System::getMemoryFree(); - if($fileSize > $freeMemory) { + if ($fileSize > $freeMemory) { throw new Exception('Trying to load file larger than available memory. Please use chunked upload.'); } + return $this->uploadData(\file_get_contents($source), $path, \mime_content_type($source), $chunk, $chunks, $metadata); } From 25f6b80b3063b85d7c9483be85f7b6ffea4c1edd Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 8 Aug 2023 02:58:34 +0000 Subject: [PATCH 28/49] improve --- src/Storage/Device/Local.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index d3b0db99..6d5c0f13 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -215,9 +215,8 @@ public function transfer(string $path, string $destination, Device $device): boo } $totalChunks = \ceil($size / $this->transferChunkSize); - $counter = 0; $metadata = ['content_type' => $contentType]; - for ($counter; $counter < $totalChunks; $counter++) { + for ($counter = 0; $counter < $totalChunks; $counter++) { $start = $counter * $this->transferChunkSize; $data = $this->read($path, $start, $this->transferChunkSize); $device->uploadData($data, $destination, $contentType, $counter + 1, $totalChunks, $metadata); From b49f24b90dedfda45df65a2fae7ae8eeed552de1 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 8 Aug 2023 13:42:03 +0545 Subject: [PATCH 29/49] revert back free memory check --- src/Storage/Device/S3.php | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 2e30250f..04c9095d 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -224,13 +224,6 @@ public function getPath(string $filename, string $prefix = null): string */ public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { - $fileSize = \filesize($source); - $freeMemory = System::getMemoryFree(); - - if ($fileSize > $freeMemory) { - throw new Exception('Trying to load file larger than available memory. Please use chunked upload.'); - } - return $this->uploadData(\file_get_contents($source), $path, \mime_content_type($source), $chunk, $chunks, $metadata); } From cef63e2d76156c2069c98ccd1430ebac5df3a35c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 8 Aug 2023 08:09:59 +0000 Subject: [PATCH 30/49] fix format --- src/Storage/Device/S3.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 04c9095d..2c771719 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -5,7 +5,6 @@ use Exception; use Utopia\Storage\Device; use Utopia\Storage\Storage; -use Utopia\System\System; class S3 extends Device { From 69bd2f38d0e0bd953b85af59eb546d6c04d1f282 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 10 Aug 2023 06:08:04 +0000 Subject: [PATCH 31/49] github action for test --- .github/workflows/tests.yml | 39 +++++++++++++++++++++++++++++ tests/Storage/Device/LinodeTest.php | 2 +- 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..6272eb11 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,39 @@ +name: "Tests" + +on: [pull_request] +jobs: + tests: + name: Unit & E2E + runs-on: ubuntu-latest + + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build image + uses: docker/build-push-action@v3 + with: + context: . + push: false + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Start storage + env: + DO_ACCESS_KEY: ${{ secrets.DO_ACCESS_KEY }} + DO_SECRET: ${{ secrets.DO_SECRET }} + LINODE_ACCESS_KEY: ${{ secrets.LINODE_ACCESS_KEY }} + LINODE_SECRET: ${{ secrets.LINODE_SECRET }} + run: | + docker compose up -d + sleep 10 + + - name: Unit Tests + run: docker compose exec -T tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite unit + + - name: E2E Tests + run: docker compose exec -T tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite e2e diff --git a/tests/Storage/Device/LinodeTest.php b/tests/Storage/Device/LinodeTest.php index 397e62bc..b10ecf85 100644 --- a/tests/Storage/Device/LinodeTest.php +++ b/tests/Storage/Device/LinodeTest.php @@ -12,7 +12,7 @@ protected function init(): void $this->root = '/root'; $key = $_SERVER['LINODE_ACCESS_KEY'] ?? ''; $secret = $_SERVER['LINODE_SECRET'] ?? ''; - $bucket = 'everly-test'; + $bucket = 'storage-test'; $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::EU_CENTRAL_1, Linode::ACL_PRIVATE); } From a9d96f0826e63d358c8c113af6c8326e4c8f8cc4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 10 Aug 2023 06:17:31 +0000 Subject: [PATCH 32/49] doctor in tests --- .github/workflows/tests.yml | 6 ++++++ tests/Storage/Device/S3Test.php | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6272eb11..5e8ca2d7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,6 +31,12 @@ jobs: run: | docker compose up -d sleep 10 + + - name: Doctor + run: | + docker compose logs + docker ps + docker network ls - name: Unit Tests run: docker compose exec -T tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite unit diff --git a/tests/Storage/Device/S3Test.php b/tests/Storage/Device/S3Test.php index d8e05c3b..9c55f217 100644 --- a/tests/Storage/Device/S3Test.php +++ b/tests/Storage/Device/S3Test.php @@ -12,7 +12,7 @@ protected function init(): void $this->root = '/root'; $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; - $bucket = 'appwrite-test-bucket'; + $bucket = 'utopia-storage-test'; $this->object = new S3($this->root, $key, $secret, $bucket, S3::EU_WEST_1, S3::ACL_PRIVATE); } From 84ee9c11934bda3e24bed921a85b46c438bd78ab Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 10 Aug 2023 06:22:49 +0000 Subject: [PATCH 33/49] build without buildx --- .github/workflows/tests.yml | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5e8ca2d7..9ca92154 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,22 +7,10 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout Repo + - name: Checkout repo uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build image - uses: docker/build-push-action@v3 - with: - context: . - push: false - load: true - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Start storage + + - name: Start test stack env: DO_ACCESS_KEY: ${{ secrets.DO_ACCESS_KEY }} DO_SECRET: ${{ secrets.DO_SECRET }} @@ -30,7 +18,7 @@ jobs: LINODE_SECRET: ${{ secrets.LINODE_SECRET }} run: | docker compose up -d - sleep 10 + sleep 1 - name: Doctor run: | From f413bbefd1847e41b57103bc7418b8fcc8b8cad8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 10 Aug 2023 06:24:25 +0000 Subject: [PATCH 34/49] build cache --- .github/workflows/tests.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9ca92154..5e8ca2d7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,10 +7,22 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout repo + - name: Checkout Repo uses: actions/checkout@v3 - - - name: Start test stack + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build image + uses: docker/build-push-action@v3 + with: + context: . + push: false + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Start storage env: DO_ACCESS_KEY: ${{ secrets.DO_ACCESS_KEY }} DO_SECRET: ${{ secrets.DO_SECRET }} @@ -18,7 +30,7 @@ jobs: LINODE_SECRET: ${{ secrets.LINODE_SECRET }} run: | docker compose up -d - sleep 1 + sleep 10 - name: Doctor run: | From 6d8acf2fd644d4deeab9f0e2a10e6c44f09b65d2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 10 Aug 2023 06:25:24 +0000 Subject: [PATCH 35/49] modify doctor --- .github/workflows/tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5e8ca2d7..4fc9e5f1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,9 +34,8 @@ jobs: - name: Doctor run: | - docker compose logs + docker compose logs tests docker ps - docker network ls - name: Unit Tests run: docker compose exec -T tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite unit From 522c5f456dcdd43a527fe406360f4d983a2a3461 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 10 Aug 2023 06:28:01 +0000 Subject: [PATCH 36/49] ls to see container --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4fc9e5f1..b1adfe02 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,7 @@ jobs: run: | docker compose logs tests docker ps + docker compose exec -T tests ls - name: Unit Tests run: docker compose exec -T tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite unit From bc62d46e5aa557a9511563c8ace6336aebc2e192 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 10 Aug 2023 06:32:54 +0000 Subject: [PATCH 37/49] modify docker file --- .github/workflows/tests.yml | 1 - docker-compose.yml | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b1adfe02..4fc9e5f1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,6 @@ jobs: run: | docker compose logs tests docker ps - docker compose exec -T tests ls - name: Unit Tests run: docker compose exec -T tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite unit diff --git a/docker-compose.yml b/docker-compose.yml index e6af4bb2..01155a5e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,9 @@ services: build: context: . volumes: - - ./:/usr/src/code + - ./src:/usr/src/code/src + - ./tests:/usr/src/code/tests + - ./phpunit.xml:/usr/src/code/phpunit.xml environment: - S3_ACCESS_KEY - S3_SECRET From f6636da07eb52cf31c6a5e185cea5e9a718775c6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 10 Aug 2023 06:35:15 +0000 Subject: [PATCH 38/49] add s3 keys --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4fc9e5f1..9bfe2d8b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,6 +28,8 @@ jobs: DO_SECRET: ${{ secrets.DO_SECRET }} LINODE_ACCESS_KEY: ${{ secrets.LINODE_ACCESS_KEY }} LINODE_SECRET: ${{ secrets.LINODE_SECRET }} + S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} + S3_SECRET: ${{ secrets.S3_SECRET }} run: | docker compose up -d sleep 10 From 22d2590d7ad92df9be9924a90daad9c00bcd8d40 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 10 Aug 2023 08:10:35 +0000 Subject: [PATCH 39/49] fix test region --- tests/Storage/Device/S3Test.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Storage/Device/S3Test.php b/tests/Storage/Device/S3Test.php index 9c55f217..2b0255c8 100644 --- a/tests/Storage/Device/S3Test.php +++ b/tests/Storage/Device/S3Test.php @@ -14,7 +14,7 @@ protected function init(): void $secret = $_SERVER['S3_SECRET'] ?? ''; $bucket = 'utopia-storage-test'; - $this->object = new S3($this->root, $key, $secret, $bucket, S3::EU_WEST_1, S3::ACL_PRIVATE); + $this->object = new S3($this->root, $key, $secret, $bucket, S3::EU_CENTRAL_1, S3::ACL_PRIVATE); } /** From bf588bc57da62137de248f4f072df2a99290e709 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 10 Aug 2023 08:25:39 +0000 Subject: [PATCH 40/49] fix test.credentials --- tests/Storage/Device/LocalTest.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Storage/Device/LocalTest.php b/tests/Storage/Device/LocalTest.php index f5ead457..9db0510c 100644 --- a/tests/Storage/Device/LocalTest.php +++ b/tests/Storage/Device/LocalTest.php @@ -257,9 +257,9 @@ public function testTransferLarge($path) $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; - $bucket = 'appwrite-test-bucket'; + $bucket = 'utopia-storage-test'; - $device = new S3('/root', $key, $secret, $bucket, S3::EU_WEST_1, S3::ACL_PRIVATE); + $device = new S3('/root', $key, $secret, $bucket, S3::EU_CENTRAL_1, S3::ACL_PRIVATE); $destination = $device->getPath('largefile.mp4'); $this->assertTrue($this->object->transfer($path, $destination, $device)); @@ -276,9 +276,9 @@ public function testTransferSmall() $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; - $bucket = 'appwrite-test-bucket'; + $bucket = 'utopia-storage-test'; - $device = new S3('/root', $key, $secret, $bucket, S3::EU_WEST_1, S3::ACL_PRIVATE); + $device = new S3('/root', $key, $secret, $bucket, S3::EU_CENTRAL_1, S3::ACL_PRIVATE); $path = $this->object->getPath('text-for-read.txt'); $this->object->write($path, 'Hello World'); From ea17610d6975b690ca670bdbaa4e2e35478ca674 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sat, 12 Aug 2023 00:58:36 +0000 Subject: [PATCH 41/49] remove travis config --- .travis.yml | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b9c4ab2d..00000000 --- a/.travis.yml +++ /dev/null @@ -1,35 +0,0 @@ -dist: focal - -arch: - - amd64 - -os: linux - -language: shell - -notifications: - email: - - team@appwrite.io - -before_script: docker run --rm --interactive --tty --volume "$(pwd)":/app composer update --ignore-platform-reqs --optimize-autoloader --no-plugins --no-scripts --prefer-dist - -before_install: - - curl -fsSL https://get.docker.com | sh - - echo '{"experimental":"enabled"}' | sudo tee /etc/docker/daemon.json - - mkdir -p $HOME/.docker - - echo '{"experimental":"enabled"}' | sudo tee $HOME/.docker/config.json - - sudo service docker start - - > - if [ ! -z "${DOCKERHUB_PULL_USERNAME:-}" ]; then - echo "${DOCKERHUB_PULL_PASSWORD}" | docker login --username "${DOCKERHUB_PULL_USERNAME}" --password-stdin - fi - - docker --version - -install: - - docker-compose up -d - - sleep 10 - -script: - - docker ps - - docker-compose exec tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite unit - - docker-compose exec tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite e2e \ No newline at end of file From 62303deb8932a9885290167cfc73904ea5d78818 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sat, 12 Aug 2023 01:35:56 +0000 Subject: [PATCH 42/49] refactor test --- .github/workflows/tests.yml | 26 ++++++++++++++++++++++++-- tests/Storage/Device/BackblazeTest.php | 2 +- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9bfe2d8b..0906886a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,9 +1,13 @@ name: "Tests" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: [pull_request] jobs: - tests: - name: Unit & E2E + build: + name: Build & Unit runs-on: ubuntu-latest steps: @@ -44,3 +48,21 @@ jobs: - name: E2E Tests run: docker compose exec -T tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite e2e + e2e_test: + name: E2E Test + runs-on: ubuntu-latest + needs: build + strategy: + fail-fast: false + matrix: + devices: [BlackblazeTest, DOSpacesTest, LinodeTest, LocalTest, S3Test, WasabiTest] + + steps: + - name: checkout + uses: actions/checkout@v3 + - name: Start storage + run: | + docker compose up -d + sleep 10 + - name: Run ${{matrix.devices}} + run: docker compose exec -T vendor/bin/phpunit tests/Storage/Device/${{matrix.devices}}.php diff --git a/tests/Storage/Device/BackblazeTest.php b/tests/Storage/Device/BackblazeTest.php index 159bcb0f..91ce2f8f 100644 --- a/tests/Storage/Device/BackblazeTest.php +++ b/tests/Storage/Device/BackblazeTest.php @@ -12,7 +12,7 @@ protected function init(): void $this->root = '/root'; $key = $_SERVER['BACKBLAZE_ACCESS_KEY'] ?? ''; $secret = $_SERVER['BACKBLAZE_SECRET'] ?? ''; - $bucket = 'backblaze-demo'; + $bucket = 'utopia-storage-test'; $this->object = new Backblaze($this->root, $key, $secret, $bucket, Backblaze::US_WEST_004, Backblaze::ACL_PRIVATE); } From 52b187c153e3df20e4bf11b9358a2ee52f90524d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sat, 12 Aug 2023 01:41:55 +0000 Subject: [PATCH 43/49] remove e2e test from build workflow --- .github/workflows/tests.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0906886a..83fb5a62 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,9 +45,7 @@ jobs: - name: Unit Tests run: docker compose exec -T tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite unit - - - name: E2E Tests - run: docker compose exec -T tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite e2e + e2e_test: name: E2E Test runs-on: ubuntu-latest From 61fa79214fa4b644f36759a9ac6b93b9249108da Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sat, 12 Aug 2023 01:51:47 +0000 Subject: [PATCH 44/49] fix test issues --- .github/workflows/tests.yml | 10 ++-------- docker-compose.yml | 1 + 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 83fb5a62..503e409b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,17 +23,11 @@ jobs: context: . push: false load: true + tags: storage-dev cache-from: type=gha cache-to: type=gha,mode=max - name: Start storage - env: - DO_ACCESS_KEY: ${{ secrets.DO_ACCESS_KEY }} - DO_SECRET: ${{ secrets.DO_SECRET }} - LINODE_ACCESS_KEY: ${{ secrets.LINODE_ACCESS_KEY }} - LINODE_SECRET: ${{ secrets.LINODE_SECRET }} - S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} - S3_SECRET: ${{ secrets.S3_SECRET }} run: | docker compose up -d sleep 10 @@ -63,4 +57,4 @@ jobs: docker compose up -d sleep 10 - name: Run ${{matrix.devices}} - run: docker compose exec -T vendor/bin/phpunit tests/Storage/Device/${{matrix.devices}}.php + run: docker compose exec -T tests vendor/bin/phpunit tests/Storage/Device/${{matrix.devices}}.php diff --git a/docker-compose.yml b/docker-compose.yml index 01155a5e..4cd5e9f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,7 @@ version: '3.1' services: tests: container_name: tests + image: storage-dev build: context: . volumes: From 8fdf481f4ae5b75f2b98e49334300962376f7d81 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sat, 12 Aug 2023 01:56:41 +0000 Subject: [PATCH 45/49] fix test environments --- .github/workflows/tests.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 503e409b..c54abcd3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,6 +28,17 @@ jobs: cache-to: type=gha,mode=max - name: Start storage + env: + DO_ACCESS_KEY: ${{ secrets.DO_ACCESS_KEY }} + DO_SECRET: ${{ secrets.DO_SECRET }} + LINODE_ACCESS_KEY: ${{ secrets.LINODE_ACCESS_KEY }} + LINODE_SECRET: ${{ secrets.LINODE_SECRET }} + S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} + S3_SECRET: ${{ secrets.S3_SECRET }} + WASABI_ACCESS_KEY: ${{ secrets.WASABI_ACCESS_KEY }} + WASABI_SECRET: ${{ secrets.WASABI_SECRET }} + BLACKBLAZE_ACCESS_KEY: ${{ secrets.BLACKBLAZE_ACCESS_KEY }} + BLACKBLAZE_SECRET: ${{ secrets.BLACKBLAZE_SECRET }} run: | docker compose up -d sleep 10 @@ -53,6 +64,17 @@ jobs: - name: checkout uses: actions/checkout@v3 - name: Start storage + env: + DO_ACCESS_KEY: ${{ secrets.DO_ACCESS_KEY }} + DO_SECRET: ${{ secrets.DO_SECRET }} + LINODE_ACCESS_KEY: ${{ secrets.LINODE_ACCESS_KEY }} + LINODE_SECRET: ${{ secrets.LINODE_SECRET }} + S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} + S3_SECRET: ${{ secrets.S3_SECRET }} + WASABI_ACCESS_KEY: ${{ secrets.WASABI_ACCESS_KEY }} + WASABI_SECRET: ${{ secrets.WASABI_SECRET }} + BLACKBLAZE_ACCESS_KEY: ${{ secrets.BLACKBLAZE_ACCESS_KEY }} + BLACKBLAZE_SECRET: ${{ secrets.BLACKBLAZE_SECRET }} run: | docker compose up -d sleep 10 From 8ce26079f9f0737de99e506c3ec65c70fe194ecc Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sat, 12 Aug 2023 02:00:51 +0000 Subject: [PATCH 46/49] fix typo --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c54abcd3..adc23b67 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,7 +58,7 @@ jobs: strategy: fail-fast: false matrix: - devices: [BlackblazeTest, DOSpacesTest, LinodeTest, LocalTest, S3Test, WasabiTest] + devices: [BackblazeTest, DOSpacesTest, LinodeTest, LocalTest, S3Test, WasabiTest] steps: - name: checkout From 274f56eeba585b3616df6465227b52e3b4124607 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sat, 12 Aug 2023 02:02:32 +0000 Subject: [PATCH 47/49] fix linode bucket region --- tests/Storage/Device/LinodeTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Storage/Device/LinodeTest.php b/tests/Storage/Device/LinodeTest.php index b10ecf85..95118739 100644 --- a/tests/Storage/Device/LinodeTest.php +++ b/tests/Storage/Device/LinodeTest.php @@ -14,7 +14,7 @@ protected function init(): void $secret = $_SERVER['LINODE_SECRET'] ?? ''; $bucket = 'storage-test'; - $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::EU_CENTRAL_1, Linode::ACL_PRIVATE); + $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::AP_SOUTH_1, Linode::ACL_PRIVATE); } protected function getAdapterName(): string From 29e54ea116964c9e04a368274708cb630e8878b2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sat, 12 Aug 2023 02:10:33 +0000 Subject: [PATCH 48/49] fix typo in keys --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index adc23b67..43d7ca7e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,8 +37,8 @@ jobs: S3_SECRET: ${{ secrets.S3_SECRET }} WASABI_ACCESS_KEY: ${{ secrets.WASABI_ACCESS_KEY }} WASABI_SECRET: ${{ secrets.WASABI_SECRET }} - BLACKBLAZE_ACCESS_KEY: ${{ secrets.BLACKBLAZE_ACCESS_KEY }} - BLACKBLAZE_SECRET: ${{ secrets.BLACKBLAZE_SECRET }} + BACKBLAZE_ACCESS_KEY: ${{ secrets.BACKBLAZE_ACCESS_KEY }} + BACKBLAZE_SECRET: ${{ secrets.BACKBLAZE_SECRET }} run: | docker compose up -d sleep 10 @@ -73,8 +73,8 @@ jobs: S3_SECRET: ${{ secrets.S3_SECRET }} WASABI_ACCESS_KEY: ${{ secrets.WASABI_ACCESS_KEY }} WASABI_SECRET: ${{ secrets.WASABI_SECRET }} - BLACKBLAZE_ACCESS_KEY: ${{ secrets.BLACKBLAZE_ACCESS_KEY }} - BLACKBLAZE_SECRET: ${{ secrets.BLACKBLAZE_SECRET }} + BACKBLAZE_ACCESS_KEY: ${{ secrets.BACKBLAZE_ACCESS_KEY }} + BACKBLAZE_SECRET: ${{ secrets.BACKBLAZE_SECRET }} run: | docker compose up -d sleep 10 From bf55cf0a03f6bb104864d44af1a63d33a298cf5a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sat, 12 Aug 2023 02:16:53 +0000 Subject: [PATCH 49/49] fix bucket region --- tests/Storage/Device/LinodeTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Storage/Device/LinodeTest.php b/tests/Storage/Device/LinodeTest.php index b10ecf85..95118739 100644 --- a/tests/Storage/Device/LinodeTest.php +++ b/tests/Storage/Device/LinodeTest.php @@ -14,7 +14,7 @@ protected function init(): void $secret = $_SERVER['LINODE_SECRET'] ?? ''; $bucket = 'storage-test'; - $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::EU_CENTRAL_1, Linode::ACL_PRIVATE); + $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::AP_SOUTH_1, Linode::ACL_PRIVATE); } protected function getAdapterName(): string