Fix: failed upload retry with test

utopia-php · Nov 24, 2024 · 30326d7 · 30326d7
1 parent fdae7bf
commit 30326d7
Show file tree

Hide file tree

Showing 4 changed files with 140 additions and 4 deletions.
diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php
@@ -95,8 +95,14 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks
         $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log';
 
         $this->createDirectory(\dirname($tmp));
-        if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) {
-            throw new Exception('Can\'t write chunk log '.$tmp);
+
+        $chunkFilePath = dirname($tmp).DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$chunk;
+
+        // skip writing chunk if the chunk was re-uploaded
+        if (! file_exists($chunkFilePath)) {
+            if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) {
+                throw new Exception('Can\'t write chunk log '.$tmp);
+            }
         }
 
         $chunkLogs = file($tmp);
@@ -106,7 +112,7 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks
 
         $chunksReceived = count(file($tmp));
 
-        if (! \rename($source, dirname($tmp).DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$chunk)) {
+        if (! \rename($source, $chunkFilePath)) {
             throw new Exception('Failed to write chunk '.$chunk);
         }
 

diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php
@@ -318,8 +318,13 @@ public function uploadData(string $data, string $path, string $contentType, int
         $etag = $this->uploadPart($data, $path, $contentType, $chunk, $uploadId);
         $metadata['parts'] ??= [];
         $metadata['parts'][] = ['partNumber' => $chunk, 'etag' => $etag];
+        $metadata['chunksUploaded'] ??= [];
+        $metadata['chunksUploaded'][$chunk] = 1;
         $metadata['chunks'] ??= 0;
-        $metadata['chunks']++;
+        // skip incrementing if the chunk was re-uploaded
+        if (! array_key_exists($chunk, $metadata['chunksUploaded'])) {
+            $metadata['chunks']++;
+        }
         if ($metadata['chunks'] == $chunks) {
             $this->completeMultipartUpload($path, $uploadId, $metadata['parts']);
         }

diff --git a/tests/Storage/Device/LocalTest.php b/tests/Storage/Device/LocalTest.php
@@ -173,6 +173,60 @@ public function testPartUpload()
         return $dest;
     }
 
+    public function testPartUploadRetry()
+    {
+        $source = __DIR__.'/../../resources/disk-a/large_file.mp4';
+        $dest = $this->object->getPath('uploaded2.mp4');
+        $totalSize = \filesize($source);
+        // AWS S3 requires each part to be at least 5MB except for last part
+        $chunkSize = 5 * 1024 * 1024;
+
+        $chunks = ceil($totalSize / $chunkSize);
+
+        $chunk = 1;
+        $start = 0;
+        $handle = @fopen($source, 'rb');
+        $op = __DIR__.'/chunkx.part';
+        while ($start < $totalSize) {
+            $contents = fread($handle, $chunkSize);
+            $op = __DIR__.'/chunkx.part';
+            $cc = fopen($op, 'wb');
+            fwrite($cc, $contents);
+            fclose($cc);
+            $this->object->upload($op, $dest, $chunk, $chunks);
+            $start += strlen($contents);
+            $chunk++;
+            if ($chunk == 2) {
+                break;
+            }
+            fseek($handle, $start);
+        }
+        @fclose($handle);
+
+        $chunk = 1;
+        $start = 0;
+        // retry from first to make sure duplicate chunk re-upload works without issue
+        $handle = @fopen($source, 'rb');
+        $op = __DIR__.'/chunkx.part';
+        while ($start < $totalSize) {
+            $contents = fread($handle, $chunkSize);
+            $op = __DIR__.'/chunkx.part';
+            $cc = fopen($op, 'wb');
+            fwrite($cc, $contents);
+            fclose($cc);
+            $this->object->upload($op, $dest, $chunk, $chunks);
+            $start += strlen($contents);
+            $chunk++;
+            fseek($handle, $start);
+        }
+        @fclose($handle);
+
+        $this->assertEquals(\filesize($source), $this->object->getFileSize($dest));
+        $this->assertEquals(\md5_file($source), $this->object->getFileHash($dest));
+
+        return $dest;
+    }
+
     public function testAbort()
     {
         $source = __DIR__.'/../../resources/disk-a/large_file.mp4';

diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php
@@ -295,6 +295,77 @@ public function testPartUpload()
         return $dest;
     }
 
+    public function testPartUploadRetry()
+    {
+        $source = __DIR__.'/../resources/disk-a/large_file.mp4';
+        $dest = $this->object->getPath('uploaded.mp4');
+        $totalSize = \filesize($source);
+        // AWS S3 requires each part to be at least 5MB except for last part
+        $chunkSize = 5 * 1024 * 1024;
+
+        $chunks = ceil($totalSize / $chunkSize);
+
+        $chunk = 1;
+        $start = 0;
+
+        $metadata = [
+            'parts' => [],
+            'chunks' => 0,
+            'uploadId' => null,
+            'content_type' => \mime_content_type($source),
+        ];
+        $handle = @fopen($source, 'rb');
+        $op = __DIR__.'/chunk.part';
+        while ($start < $totalSize) {
+            $contents = fread($handle, $chunkSize);
+            $op = __DIR__.'/chunk.part';
+            $cc = fopen($op, 'wb');
+            fwrite($cc, $contents);
+            fclose($cc);
+            $etag = $this->object->upload($op, $dest, $chunk, $chunks, $metadata);
+            $parts[] = ['partNumber' => $chunk, 'etag' => $etag];
+            $start += strlen($contents);
+            $chunk++;
+            if ($chunk == 2) {
+                break;
+            }
+            fseek($handle, $start);
+        }
+        @fclose($handle);
+        unlink($op);
+
+        $chunk = 1;
+        $start = 0;
+        // retry from first to make sure duplicate chunk re-upload works without issue
+        $handle = @fopen($source, 'rb');
+        $op = __DIR__.'/chunk.part';
+        while ($start < $totalSize) {
+            $contents = fread($handle, $chunkSize);
+            $op = __DIR__.'/chunk.part';
+            $cc = fopen($op, 'wb');
+            fwrite($cc, $contents);
+            fclose($cc);
+            $etag = $this->object->upload($op, $dest, $chunk, $chunks, $metadata);
+            $parts[] = ['partNumber' => $chunk, 'etag' => $etag];
+            $start += strlen($contents);
+            $chunk++;
+            fseek($handle, $start);
+        }
+        @fclose($handle);
+        unlink($op);
+
+        $this->assertEquals(\filesize($source), $this->object->getFileSize($dest));
+
+        // S3 doesnt provide a method to get a proper MD5-hash of a file created using multipart upload
+        // https://stackoverflow.com/questions/8618218/amazon-s3-checksum
+        // More info on how AWS calculates ETag for multipart upload here
+        // https://savjee.be/2015/10/Verifying-Amazon-S3-multi-part-uploads-with-ETag-hash/
+        // TODO
+        // $this->assertEquals(\md5_file($source), $this->object->getFileHash($dest));
+        // $this->object->delete($dest);
+        return $dest;
+    }
+
     /**
      * @depends testPartUpload
      */