From bd4eab6832a5d656ecea0fc901272d98b30547be Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 14:32:14 +0100
Subject: [PATCH 01/22] fix: Etag is not a key, so it was always no

---
 EMS/common-bundle/src/Storage/Service/S3Storage.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/EMS/common-bundle/src/Storage/Service/S3Storage.php b/EMS/common-bundle/src/Storage/Service/S3Storage.php
index cb94411ac..09a64be38 100644
--- a/EMS/common-bundle/src/Storage/Service/S3Storage.php
+++ b/EMS/common-bundle/src/Storage/Service/S3Storage.php
@@ -347,8 +347,9 @@ public function copyFileInArchiveCache(string $archiveHash, string $fileHash, st
             'CopySource' => "$this->bucket/$sourceKey",
             'MetadataDirective' => 'REPLACE',
         ]);
+        $result = $result->get('CopyObjectResult');
 
-        return $result->hasKey('ETag');
+        return \is_string($result['ETag'] ?? null);
     }
 
     public function heads(string ...$hashes): array

From a45a9d2a16cfcb63fa62fb40b44bfe7dec8ee9ef Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 14:52:23 +0100
Subject: [PATCH 02/22] feat: loadArchiveItemsInCache

---
 .../FileStructure/FileStructurePushCommand.php     | 14 ++++++++++++++
 .../src/Contracts/File/FileManagerInterface.php    |  3 +++
 EMS/common-bundle/src/Storage/StorageManager.php   | 14 ++++++++++++++
 3 files changed, 31 insertions(+)

diff --git a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
index 7fd03db03..e6c17b5eb 100644
--- a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
+++ b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
@@ -118,6 +118,20 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
             return self::EXECUTE_ERROR;
         }
+
+        if (!$this->quiet) {
+            $this->io->section('Building cache');
+        }
+        $progressBar = $this->io->createProgressBar($archive->getCount());
+
+        $this->fileManager->loadArchiveItemsInCache($hash, $archive, $this->quiet ? null : function () use ($progressBar) {
+            $progressBar->advance();
+        });
+        if (!$this->quiet) {
+            $progressBar->finish();
+            $this->io->newLine();
+        }
+
         \file_put_contents($hashFilename, $hash);
 
         if ($this->quiet) {
diff --git a/EMS/common-bundle/src/Contracts/File/FileManagerInterface.php b/EMS/common-bundle/src/Contracts/File/FileManagerInterface.php
index 2acef4437..97c5198ba 100644
--- a/EMS/common-bundle/src/Contracts/File/FileManagerInterface.php
+++ b/EMS/common-bundle/src/Contracts/File/FileManagerInterface.php
@@ -4,6 +4,7 @@
 
 namespace EMS\CommonBundle\Contracts\File;
 
+use EMS\CommonBundle\Storage\Archive;
 use Psr\Http\Message\StreamInterface;
 
 interface FileManagerInterface
@@ -31,4 +32,6 @@ public function uploadFile(string $realPath, ?string $mimeType = null, ?string $
      * @param int<1, max> $chunkSize
      */
     public function setHeadChunkSize(int $chunkSize): void;
+
+    public function loadArchiveItemsInCache(string $archiveHash, Archive $archive, callable $callback = null): void;
 }
diff --git a/EMS/common-bundle/src/Storage/StorageManager.php b/EMS/common-bundle/src/Storage/StorageManager.php
index 10b341d70..c6251033b 100644
--- a/EMS/common-bundle/src/Storage/StorageManager.php
+++ b/EMS/common-bundle/src/Storage/StorageManager.php
@@ -692,4 +692,18 @@ public function setHeadChunkSize(int $chunkSize): void
     {
         $this->headChunkSize = $chunkSize;
     }
+
+    public function loadArchiveItemsInCache(string $archiveHash, Archive $archive, callable $callback = null): void
+    {
+        foreach ($archive->iterator() as $archiveItem) {
+            foreach ($this->adapters as $adapter) {
+                if ($adapter->copyFileInArchiveCache($archiveHash, $archiveItem->hash, $archiveItem->filename, $archiveItem->type)) {
+                    if (null !== $callback) {
+                        $callback();
+                    }
+                    break;
+                }
+            }
+        }
+    }
 }

From 08e4b1e2990b01bc84b6a8504fa6be8400dc0f58 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 15:07:12 +0100
Subject: [PATCH 03/22] ref: quiet option is automatically taken into account
 by SymfonyConsole

---
 .../FileStructure/FileStructurePushCommand.php | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)

diff --git a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
index e6c17b5eb..1843e386b 100644
--- a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
+++ b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
@@ -30,7 +30,6 @@ class FileStructurePushCommand extends AbstractCommand
     private const DEFAULT_SAVE_HASH_FILE = '.hash';
     private string $folderPath;
     private FileManagerInterface $fileManager;
-    private bool $quiet;
     private int $chunkSize;
     private string $saveHashFilename;
 
@@ -62,21 +61,16 @@ protected function initialize(InputInterface $input, OutputInterface $output): v
             true => $this->adminHelper->getCoreApi()->file(),
             false => $this->storageManager,
         };
-        $this->quiet = $this->getOptionBool(self::OPTION_QUIET);
         $this->chunkSize = $this->getOptionInt(self::OPTION_CHUNK_SIZE);
         $this->saveHashFilename = $this->getOptionString(self::OPTION_SAVE_HASH_FILENAME);
     }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
-        if (!$this->quiet) {
-            $this->io->title('EMS - File structure - Push');
-        }
+        $this->io->title('EMS - File structure - Push');
         $algo = $this->fileManager->getHashAlgo();
 
-        if (!$this->quiet) {
-            $this->io->section('Building archive');
-        }
+        $this->io->section('Building archive');
         $archive = Archive::fromDirectory($this->folderPath, $algo);
         $previousArchive = null;
         $hashFilename = \implode(DIRECTORY_SEPARATOR, [$this->folderPath, $this->saveHashFilename]);
@@ -84,9 +78,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             $previousArchive = Archive::fromStructure($this->fileManager->getContents(File::fromFilename($hashFilename)->getContents()), $algo);
         }
 
-        if (!$this->quiet) {
-            $this->io->section('Pushing archive');
-        }
+        $this->io->section('Pushing archive');
         $progressBar = $this->io->createProgressBar($archive->getCount());
         $failedCount = 0;
         if ($this->chunkSize < 1) {
@@ -134,8 +126,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         \file_put_contents($hashFilename, $hash);
 
-        if ($this->quiet) {
-            $this->io->write($hash);
+        if ($this->output->isQuiet()) {
+            echo $hash;
         } else {
             $this->io->success(\sprintf('Archive %s have been uploaded with the directory content of %s', $hash, $this->folderPath));
         }

From 23327267fe4d48d0f31113dadbbb3f33875af1f3 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 15:07:57 +0100
Subject: [PATCH 04/22] ref: quiet option is automatically taken into account
 by SymfonyConsole

---
 .../FileStructurePushCommand.php              | 27 +++++++++----------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
index 1843e386b..52aec602e 100644
--- a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
+++ b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
@@ -23,9 +23,7 @@ class FileStructurePushCommand extends AbstractCommand
     protected static $defaultName = Commands::FILE_STRUCTURE_PUSH;
     private const ARGUMENT_FOLDER = 'folder';
     private const OPTION_ADMIN = 'admin';
-    private const OPTION_QUIET = 'quiet';
     private const OPTION_CHUNK_SIZE = 'chunk-size';
-    private const OPTION_QUIET_SHORTCUT = 'q';
     private const OPTION_SAVE_HASH_FILENAME = 'save-hash-filename';
     private const DEFAULT_SAVE_HASH_FILE = '.hash';
     private string $folderPath;
@@ -47,7 +45,6 @@ protected function configure(): void
             ->setDescription('Push an EMS Archive file structure into a EMS Admin storage services (via the API)')
             ->addArgument(self::ARGUMENT_FOLDER, InputArgument::REQUIRED, 'Source folder')
             ->addOption(self::OPTION_ADMIN, null, InputOption::VALUE_NONE, 'Push to admin')
-            ->addOption(self::OPTION_QUIET, self::OPTION_QUIET_SHORTCUT, InputOption::VALUE_NONE, 'only displays the archive hash (if succeed)')
             ->addOption(self::OPTION_CHUNK_SIZE, null, InputOption::VALUE_OPTIONAL, 'Set the heads method chunk size', FileManagerInterface::HEADS_CHUNK_SIZE)
             ->addOption(self::OPTION_SAVE_HASH_FILENAME, null, InputOption::VALUE_OPTIONAL, 'File where to save the structure hash within the source folder (used to avoid head request). Delete the file to force recheck all files.', self::DEFAULT_SAVE_HASH_FILE)
         ;
@@ -111,18 +108,18 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             return self::EXECUTE_ERROR;
         }
 
-        if (!$this->quiet) {
-            $this->io->section('Building cache');
-        }
-        $progressBar = $this->io->createProgressBar($archive->getCount());
-
-        $this->fileManager->loadArchiveItemsInCache($hash, $archive, $this->quiet ? null : function () use ($progressBar) {
-            $progressBar->advance();
-        });
-        if (!$this->quiet) {
-            $progressBar->finish();
-            $this->io->newLine();
-        }
+//        if (!$this->quiet) {
+//            $this->io->section('Building cache');
+//        }
+//        $progressBar = $this->io->createProgressBar($archive->getCount());
+//
+//        $this->fileManager->loadArchiveItemsInCache($hash, $archive, $this->quiet ? null : function () use ($progressBar) {
+//            $progressBar->advance();
+//        });
+//        if (!$this->quiet) {
+//            $progressBar->finish();
+//            $this->io->newLine();
+//        }
 
         \file_put_contents($hashFilename, $hash);
 

From 34d63aec0457d867e098dc2d0de4cb581f081de4 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 15:10:55 +0100
Subject: [PATCH 05/22] ref: quiet option is automatically taken into account
 by SymfonyConsole

---
 .../FileStructurePushCommand.php              | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
index 52aec602e..416437599 100644
--- a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
+++ b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
@@ -108,18 +108,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             return self::EXECUTE_ERROR;
         }
 
-//        if (!$this->quiet) {
-//            $this->io->section('Building cache');
-//        }
-//        $progressBar = $this->io->createProgressBar($archive->getCount());
-//
-//        $this->fileManager->loadArchiveItemsInCache($hash, $archive, $this->quiet ? null : function () use ($progressBar) {
-//            $progressBar->advance();
-//        });
-//        if (!$this->quiet) {
-//            $progressBar->finish();
-//            $this->io->newLine();
-//        }
+        $this->io->section('Building cache');
+        $progressBar = $this->io->createProgressBar($archive->getCount());
+        $this->fileManager->loadArchiveItemsInCache($hash, $archive, $this->output->isQuiet() ? null : function () use ($progressBar) {
+            $progressBar->advance();
+        });
+        $progressBar->finish();
+        $this->io->newLine();
 
         \file_put_contents($hashFilename, $hash);
 

From 89144460d7bb0f01f90e8f566175e5fa26dff9e9 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 15:19:50 +0100
Subject: [PATCH 06/22] feat: progressbar on building archive

---
 .../Command/FileStructure/FileStructurePushCommand.php | 10 +++++++++-
 EMS/common-bundle/src/Storage/Archive.php              |  7 ++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
index 416437599..07d6731bb 100644
--- a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
+++ b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
@@ -68,7 +68,15 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $algo = $this->fileManager->getHashAlgo();
 
         $this->io->section('Building archive');
-        $archive = Archive::fromDirectory($this->folderPath, $algo);
+        $progressBar = $this->io->createProgressBar();
+        $archive = Archive::fromDirectory($this->folderPath, $algo, $this->output->isQuiet() ? null : function ($maxSteps, $progress) use ($progressBar) {
+            if ($maxSteps !== $progressBar->getMaxSteps()) {
+                $progressBar->setMaxSteps($maxSteps);
+            }
+            $progressBar->setProgress($progress);
+        });
+        $progressBar->finish();
+        $this->io->newLine();
         $previousArchive = null;
         $hashFilename = \implode(DIRECTORY_SEPARATOR, [$this->folderPath, $this->saveHashFilename]);
         if (\file_exists($hashFilename)) {
diff --git a/EMS/common-bundle/src/Storage/Archive.php b/EMS/common-bundle/src/Storage/Archive.php
index afbea083e..068ff3f12 100644
--- a/EMS/common-bundle/src/Storage/Archive.php
+++ b/EMS/common-bundle/src/Storage/Archive.php
@@ -21,7 +21,7 @@ public function __construct(private readonly string $hashAlgo)
     {
     }
 
-    public static function fromDirectory(string $directory, string $hashAlgo): self
+    public static function fromDirectory(string $directory, string $hashAlgo, callable $callback = null): self
     {
         $archive = new self($hashAlgo);
         $finder = new Finder();
@@ -31,8 +31,13 @@ public static function fromDirectory(string $directory, string $hashAlgo): self
             throw new \RuntimeException('The directory is empty');
         }
 
+        $maxSteps = $finder->count();
+        $counter = 0;
         foreach ($finder as $file) {
             $archive->addFile($file);
+            if (null !== $callback) {
+                $callback($maxSteps, ++$counter);
+            }
         }
 
         return $archive;

From c3c6c3552e597dcea71dc12224ff2bad706dae7a Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 15:20:11 +0100
Subject: [PATCH 07/22] ref: cleaner, a bit

---
 .../src/Command/FileStructure/FileStructurePushCommand.php      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
index 07d6731bb..cec7413e7 100644
--- a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
+++ b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
@@ -108,8 +108,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             $progressBar->advance();
         }
         $progressBar->finish();
-        $hash = $this->fileManager->uploadContents(Json::encode($archive), 'archive.json', MimeTypes::APPLICATION_JSON->value);
         $this->io->newLine();
+        $hash = $this->fileManager->uploadContents(Json::encode($archive), 'archive.json', MimeTypes::APPLICATION_JSON->value);
         if (0 !== $failedCount) {
             $this->io->error(\sprintf('%d files faced an issue while uploading, please retry.', $failedCount));
 

From 04b3b99597f40000f6afefb15e203ed5aa979cdb Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 15:44:54 +0100
Subject: [PATCH 08/22] feat: ems:storage:load-archive-in-cache command

---
 .../LoadArchiveItemsInCacheCommand.php        | 56 +++++++++++++++++++
 EMS/common-bundle/src/Commands.php            |  1 +
 .../src/Resources/config/commands.xml         |  4 ++
 3 files changed, 61 insertions(+)
 create mode 100644 EMS/common-bundle/src/Command/Storage/LoadArchiveItemsInCacheCommand.php

diff --git a/EMS/common-bundle/src/Command/Storage/LoadArchiveItemsInCacheCommand.php b/EMS/common-bundle/src/Command/Storage/LoadArchiveItemsInCacheCommand.php
new file mode 100644
index 000000000..ca54b7b8e
--- /dev/null
+++ b/EMS/common-bundle/src/Command/Storage/LoadArchiveItemsInCacheCommand.php
@@ -0,0 +1,56 @@
+<?php
+
+declare(strict_types=1);
+
+namespace EMS\CommonBundle\Command\Storage;
+
+use EMS\CommonBundle\Commands;
+use EMS\CommonBundle\Common\Command\AbstractCommand;
+use EMS\CommonBundle\Storage\Archive;
+use EMS\CommonBundle\Storage\StorageManager;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class LoadArchiveItemsInCacheCommand extends AbstractCommand
+{
+    public const ARGUMENT_ARCHIVE_HASH = 'archive-hash';
+    protected static $defaultName = Commands::LOAD_ARCHIVE_IN_CACHE;
+    private string $archiveHash;
+
+    public function __construct(private readonly StorageManager $storageManager)
+    {
+        parent::__construct();
+    }
+
+    protected function configure(): void
+    {
+        parent::configure();
+        $this
+            ->setDescription('Load archive\'s items in cache')
+            ->addArgument(self::ARGUMENT_ARCHIVE_HASH, InputArgument::REQUIRED, 'Hash of the archive file')
+        ;
+    }
+
+    protected function initialize(InputInterface $input, OutputInterface $output): void
+    {
+        parent::initialize($input, $output);
+        $this->archiveHash = $this->getArgumentString(self::ARGUMENT_ARCHIVE_HASH);
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $this->io->title('Load archive\'s items in storage cache');
+
+        $archive = Archive::fromStructure($this->storageManager->getContents($this->archiveHash), $this->storageManager->getHashAlgo());
+        $progressBar = $this->io->createProgressBar($archive->getCount());
+        $this->storageManager->loadArchiveItemsInCache($this->archiveHash, $archive, $this->output->isQuiet() ? null : function () use ($progressBar) {
+            $progressBar->advance();
+        });
+        $progressBar->finish();
+        $this->io->newLine();
+        $this->io->writeln(\sprintf('%d files have been pushed in cache', $archive->getCount()));
+
+        return self::EXECUTE_SUCCESS;
+    }
+}
diff --git a/EMS/common-bundle/src/Commands.php b/EMS/common-bundle/src/Commands.php
index d9b00b330..f3e6a2462 100644
--- a/EMS/common-bundle/src/Commands.php
+++ b/EMS/common-bundle/src/Commands.php
@@ -11,6 +11,7 @@ class Commands
     final public const STATUS = 'ems:status';
     final public const CLEAR_LOGS = 'ems:logs:clear';
     final public const CLEAR_CACHE = 'ems:storage:clear-cache';
+    final public const LOAD_ARCHIVE_IN_CACHE = 'ems:storage:load-archive-in-cache';
     final public const ADMIN_COMMAND = 'ems:admin:command';
     final public const ADMIN_NEXT_JOB = 'ems:admin:next-job';
     final public const FILE_STRUCTURE_PUBLISH = 'ems:file-structure:publish';
diff --git a/EMS/common-bundle/src/Resources/config/commands.xml b/EMS/common-bundle/src/Resources/config/commands.xml
index bbc9f1f90..cfffffe51 100644
--- a/EMS/common-bundle/src/Resources/config/commands.xml
+++ b/EMS/common-bundle/src/Resources/config/commands.xml
@@ -109,5 +109,9 @@
             <argument type="service" id="ems_common.storage.manager"/>
             <tag name="console.command" />
         </service>
+        <service id="ems.command.storage.load_archive_times_in_cache" class="EMS\CommonBundle\Command\Storage\LoadArchiveItemsInCacheCommand">
+            <argument type="service" id="ems_common.storage.manager"/>
+            <tag name="console.command" />
+        </service>
     </services>
 </container>

From 9d6e504ff4edad8f8c8c735aae04a8be68bc3a49 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 15:59:51 +0100
Subject: [PATCH 09/22] feat: loadArchiveItemsInCache

---
 .../src/Common/CoreApi/Endpoint/File/File.php          | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/EMS/common-bundle/src/Common/CoreApi/Endpoint/File/File.php b/EMS/common-bundle/src/Common/CoreApi/Endpoint/File/File.php
index 6b0cc568d..109779abd 100644
--- a/EMS/common-bundle/src/Common/CoreApi/Endpoint/File/File.php
+++ b/EMS/common-bundle/src/Common/CoreApi/Endpoint/File/File.php
@@ -4,8 +4,11 @@
 
 namespace EMS\CommonBundle\Common\CoreApi\Endpoint\File;
 
+use EMS\CommonBundle\Commands;
 use EMS\CommonBundle\Common\CoreApi\Client;
+use EMS\CommonBundle\Common\CoreApi\Endpoint\Admin\Admin;
 use EMS\CommonBundle\Contracts\CoreApi\Endpoint\File\FileInterface;
+use EMS\CommonBundle\Storage\Archive;
 use EMS\CommonBundle\Storage\File\StorageFile;
 use EMS\CommonBundle\Storage\Service\HttpStorage;
 use EMS\CommonBundle\Storage\StorageManager;
@@ -204,4 +207,11 @@ public function setHeadChunkSize(int $chunkSize): void
     {
         $this->headChunkSize = $chunkSize;
     }
+
+    public function loadArchiveItemsInCache(string $archiveHash, Archive $archive, callable $callback = null): void
+    {
+        $admin = new Admin($this->client);
+        $command = \sprintf('%s %s', Commands::LOAD_ARCHIVE_IN_CACHE, $archiveHash);
+        $admin->runCommand($command);
+    }
 }

From e6a5d6355ebd0180a25c46ccecba804005e9f87d Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 16:20:42 +0100
Subject: [PATCH 10/22] feat: Archive::diff

---
 .../FileStructurePushCommand.php              |  7 +++--
 EMS/common-bundle/src/Storage/Archive.php     | 26 ++++++++++++++++---
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
index cec7413e7..9acfe086e 100644
--- a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
+++ b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
@@ -77,11 +77,14 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         });
         $progressBar->finish();
         $this->io->newLine();
+
+        $this->io->section('Comparing with previous archive');
         $previousArchive = null;
         $hashFilename = \implode(DIRECTORY_SEPARATOR, [$this->folderPath, $this->saveHashFilename]);
         if (\file_exists($hashFilename)) {
             $previousArchive = Archive::fromStructure($this->fileManager->getContents(File::fromFilename($hashFilename)->getContents()), $algo);
         }
+        $diffArchive = $archive->diff($previousArchive);
 
         $this->io->section('Pushing archive');
         $progressBar = $this->io->createProgressBar($archive->getCount());
@@ -90,7 +93,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             throw new \RuntimeException(\sprintf('Chunk size must greater than 0, %d given', $this->chunkSize));
         }
         $this->fileManager->setHeadChunkSize($this->chunkSize);
-        foreach ($this->fileManager->heads(...$archive->getHashes($previousArchive)) as $hash) {
+        foreach ($this->fileManager->heads(...$diffArchive->getHashes()) as $hash) {
             if (true === $hash) {
                 $progressBar->advance();
                 continue;
@@ -118,7 +121,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         $this->io->section('Building cache');
         $progressBar = $this->io->createProgressBar($archive->getCount());
-        $this->fileManager->loadArchiveItemsInCache($hash, $archive, $this->output->isQuiet() ? null : function () use ($progressBar) {
+        $this->fileManager->loadArchiveItemsInCache($hash, $diffArchive, $this->output->isQuiet() ? null : function () use ($progressBar) {
             $progressBar->advance();
         });
         $progressBar->finish();
diff --git a/EMS/common-bundle/src/Storage/Archive.php b/EMS/common-bundle/src/Storage/Archive.php
index 068ff3f12..4e83288b8 100644
--- a/EMS/common-bundle/src/Storage/Archive.php
+++ b/EMS/common-bundle/src/Storage/Archive.php
@@ -58,12 +58,9 @@ public static function fromStructure(string $structure, string $hashAlgo): self
     /**
      * @return iterable<string>
      */
-    public function getHashes(Archive $previousArchive = null): iterable
+    public function getHashes(): iterable
     {
         foreach ($this->files as $file) {
-            if (null !== $previousArchive && $previousArchive->containsByHash($file->hash)) {
-                continue;
-            }
             yield $file->hash;
         }
     }
@@ -162,4 +159,25 @@ private function containsByHash(string $hash): bool
 
         return false;
     }
+
+    public function diff(?Archive $otherArchive): self
+    {
+        if (null === $otherArchive) {
+            return $this;
+        }
+        $newArchive = new self($this->hashAlgo);
+        foreach ($this->files as $file) {
+            if (null !== $otherArchive && $otherArchive->containsByHash($file->hash)) {
+                continue;
+            }
+            $newArchive->addArchiveItem($file);
+        }
+
+        return $newArchive;
+    }
+
+    private function addArchiveItem(ArchiveItem $file): void
+    {
+        $this->files[$file->filename] = $file;
+    }
 }

From 89865283f3e5a0a86bfdd0f929ad7b96be6ecde8 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 16:42:30 +0100
Subject: [PATCH 11/22] fix: heads assume that files must be know by the first
 storage service

---
 EMS/common-bundle/src/Storage/StorageManager.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/EMS/common-bundle/src/Storage/StorageManager.php b/EMS/common-bundle/src/Storage/StorageManager.php
index c6251033b..fe70cd8f4 100644
--- a/EMS/common-bundle/src/Storage/StorageManager.php
+++ b/EMS/common-bundle/src/Storage/StorageManager.php
@@ -103,6 +103,7 @@ public function heads(string ...$fileHashes): \Traversable
         foreach ($pagedHashes as $hashes) {
             foreach ($this->adapters as $adapter) {
                 yield from $adapter->heads(...$hashes);
+                break;
             }
         }
     }

From ae9c7e09b4291157260fce147ab833d279a8e700 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 17:00:27 +0100
Subject: [PATCH 12/22] feat: skip already loaded files

---
 .../Storage/LoadArchiveItemsInCacheCommand.php  |  7 ++++++-
 EMS/common-bundle/src/Storage/Archive.php       | 17 +++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/EMS/common-bundle/src/Command/Storage/LoadArchiveItemsInCacheCommand.php b/EMS/common-bundle/src/Command/Storage/LoadArchiveItemsInCacheCommand.php
index ca54b7b8e..0eec5dde4 100644
--- a/EMS/common-bundle/src/Command/Storage/LoadArchiveItemsInCacheCommand.php
+++ b/EMS/common-bundle/src/Command/Storage/LoadArchiveItemsInCacheCommand.php
@@ -10,13 +10,16 @@
 use EMS\CommonBundle\Storage\StorageManager;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class LoadArchiveItemsInCacheCommand extends AbstractCommand
 {
     public const ARGUMENT_ARCHIVE_HASH = 'archive-hash';
+    public const OPTION_CONTINUE = 'continue';
     protected static $defaultName = Commands::LOAD_ARCHIVE_IN_CACHE;
     private string $archiveHash;
+    private int $continue;
 
     public function __construct(private readonly StorageManager $storageManager)
     {
@@ -29,6 +32,7 @@ protected function configure(): void
         $this
             ->setDescription('Load archive\'s items in cache')
             ->addArgument(self::ARGUMENT_ARCHIVE_HASH, InputArgument::REQUIRED, 'Hash of the archive file')
+            ->addOption(self::OPTION_CONTINUE, null, InputOption::VALUE_OPTIONAL, 'Restart the load in cache from the specified item in the archive', 0)
         ;
     }
 
@@ -36,6 +40,7 @@ protected function initialize(InputInterface $input, OutputInterface $output): v
     {
         parent::initialize($input, $output);
         $this->archiveHash = $this->getArgumentString(self::ARGUMENT_ARCHIVE_HASH);
+        $this->continue = $this->getOptionInt(self::OPTION_CONTINUE);
     }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
@@ -44,7 +49,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         $archive = Archive::fromStructure($this->storageManager->getContents($this->archiveHash), $this->storageManager->getHashAlgo());
         $progressBar = $this->io->createProgressBar($archive->getCount());
-        $this->storageManager->loadArchiveItemsInCache($this->archiveHash, $archive, $this->output->isQuiet() ? null : function () use ($progressBar) {
+        $this->storageManager->loadArchiveItemsInCache($this->archiveHash, $archive->skip($this->continue), $this->output->isQuiet() ? null : function () use ($progressBar) {
             $progressBar->advance();
         });
         $progressBar->finish();
diff --git a/EMS/common-bundle/src/Storage/Archive.php b/EMS/common-bundle/src/Storage/Archive.php
index 4e83288b8..5ce4081ab 100644
--- a/EMS/common-bundle/src/Storage/Archive.php
+++ b/EMS/common-bundle/src/Storage/Archive.php
@@ -180,4 +180,21 @@ private function addArchiveItem(ArchiveItem $file): void
     {
         $this->files[$file->filename] = $file;
     }
+
+    public function skip(int $skip): self
+    {
+        if ($skip <= 0) {
+            return $this;
+        }
+        $newArchive = new self($this->hashAlgo);
+        $counter = 0;
+        foreach ($this->files as $file) {
+            if ($counter++ < $skip) {
+                continue;
+            }
+            $newArchive->addArchiveItem($file);
+        }
+
+        return $newArchive;
+    }
 }

From 03184546c9a1438f7a54bbecabfdf8dfabf8ad6e Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 19:55:20 +0100
Subject: [PATCH 13/22] fix: if the archive hash changes, all cache must be
 loaded for that brand new hash

---
 .../src/Command/FileStructure/FileStructurePushCommand.php      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
index 9acfe086e..63bf31ec6 100644
--- a/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
+++ b/EMS/common-bundle/src/Command/FileStructure/FileStructurePushCommand.php
@@ -121,7 +121,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         $this->io->section('Building cache');
         $progressBar = $this->io->createProgressBar($archive->getCount());
-        $this->fileManager->loadArchiveItemsInCache($hash, $diffArchive, $this->output->isQuiet() ? null : function () use ($progressBar) {
+        $this->fileManager->loadArchiveItemsInCache($hash, $archive, $this->output->isQuiet() ? null : function () use ($progressBar) {
             $progressBar->advance();
         });
         $progressBar->finish();

From 412f92bd72c2d11fe9e342ce616a4786b33e2405 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 20:03:35 +0100
Subject: [PATCH 14/22] ref: loadArchiveItemsInCache

---
 .../Storage/Service/AbstractUrlStorage.php    |  3 +-
 .../src/Storage/Service/EntityStorage.php     |  3 +-
 .../src/Storage/Service/FileSystemStorage.php |  3 +-
 .../src/Storage/Service/S3Storage.php         | 42 ++++++++++++-------
 .../src/Storage/Service/StorageInterface.php  |  3 +-
 .../src/Storage/StorageManager.php            | 21 ++++------
 6 files changed, 42 insertions(+), 33 deletions(-)

diff --git a/EMS/common-bundle/src/Storage/Service/AbstractUrlStorage.php b/EMS/common-bundle/src/Storage/Service/AbstractUrlStorage.php
index 9cef2369b..398140b6b 100644
--- a/EMS/common-bundle/src/Storage/Service/AbstractUrlStorage.php
+++ b/EMS/common-bundle/src/Storage/Service/AbstractUrlStorage.php
@@ -4,6 +4,7 @@
 
 namespace EMS\CommonBundle\Storage\Service;
 
+use EMS\CommonBundle\Storage\Archive;
 use EMS\CommonBundle\Storage\File\FileInterface;
 use EMS\CommonBundle\Storage\Processor\Config;
 use EMS\CommonBundle\Storage\StreamWrapper;
@@ -250,7 +251,7 @@ public function addFileInArchiveCache(string $hash, SplFileInfo $file, string $m
         return false;
     }
 
-    public function copyFileInArchiveCache(string $archiveHash, string $fileHash, string $path, string $mimeType): bool
+    public function loadArchiveItemsInCache(string $archiveHash, Archive $archive, callable $callback = null): bool
     {
         return false;
     }
diff --git a/EMS/common-bundle/src/Storage/Service/EntityStorage.php b/EMS/common-bundle/src/Storage/Service/EntityStorage.php
index 424886210..30e6fda6b 100644
--- a/EMS/common-bundle/src/Storage/Service/EntityStorage.php
+++ b/EMS/common-bundle/src/Storage/Service/EntityStorage.php
@@ -8,6 +8,7 @@
 use Doctrine\Persistence\ObjectManager;
 use EMS\CommonBundle\Entity\AssetStorage;
 use EMS\CommonBundle\Repository\AssetStorageRepository;
+use EMS\CommonBundle\Storage\Archive;
 use EMS\CommonBundle\Storage\File\FileInterface;
 use EMS\CommonBundle\Storage\Processor\Config;
 use EMS\CommonBundle\Storage\StreamWrapper;
@@ -246,7 +247,7 @@ public function addFileInArchiveCache(string $hash, SplFileInfo $file, string $m
         return false;
     }
 
-    public function copyFileInArchiveCache(string $archiveHash, string $fileHash, string $path, string $mimeType): bool
+    public function loadArchiveItemsInCache(string $archiveHash, Archive $archive, callable $callback = null): bool
     {
         return false;
     }
diff --git a/EMS/common-bundle/src/Storage/Service/FileSystemStorage.php b/EMS/common-bundle/src/Storage/Service/FileSystemStorage.php
index 214e0b10a..912b7c6dd 100644
--- a/EMS/common-bundle/src/Storage/Service/FileSystemStorage.php
+++ b/EMS/common-bundle/src/Storage/Service/FileSystemStorage.php
@@ -5,6 +5,7 @@
 namespace EMS\CommonBundle\Storage\Service;
 
 use EMS\CommonBundle\Helper\MimeTypeHelper;
+use EMS\CommonBundle\Storage\Archive;
 use EMS\CommonBundle\Storage\File\FileInterface;
 use EMS\CommonBundle\Storage\Processor\Config;
 use EMS\CommonBundle\Storage\StreamWrapper;
@@ -108,7 +109,7 @@ public function addFileInArchiveCache(string $hash, SplFileInfo $file, string $m
         return \copy($file->getPathname(), $filename);
     }
 
-    public function copyFileInArchiveCache(string $archiveHash, string $fileHash, string $path, string $mimeType): bool
+    public function loadArchiveItemsInCache(string $archiveHash, Archive $archive, callable $callback = null): bool
     {
         return false;
     }
diff --git a/EMS/common-bundle/src/Storage/Service/S3Storage.php b/EMS/common-bundle/src/Storage/Service/S3Storage.php
index 09a64be38..aaeea0baa 100644
--- a/EMS/common-bundle/src/Storage/Service/S3Storage.php
+++ b/EMS/common-bundle/src/Storage/Service/S3Storage.php
@@ -4,9 +4,11 @@
 
 namespace EMS\CommonBundle\Storage\Service;
 
+use Aws\CommandPool;
 use Aws\Exception\AwsException;
 use Aws\S3\S3Client;
 use EMS\CommonBundle\Common\Cache\Cache;
+use EMS\CommonBundle\Storage\Archive;
 use EMS\CommonBundle\Storage\File\FileInterface;
 use EMS\CommonBundle\Storage\Processor\Config;
 use EMS\CommonBundle\Storage\StreamWrapper;
@@ -332,24 +334,34 @@ public function addFileInArchiveCache(string $hash, SplFileInfo $file, string $m
         return $result->hasKey('ETag');
     }
 
-    public function copyFileInArchiveCache(string $archiveHash, string $fileHash, string $path, string $mimeType): bool
+    public function loadArchiveItemsInCache(string $archiveHash, Archive $archive, callable $callback = null): bool
     {
-        $sourceKey = $this->key($fileHash);
-        $result = $this->getS3Client()->copyObject([
-            'Bucket' => $this->bucket,
-            'ContentType' => $mimeType,
-            'Key' => \implode('/', [
-                'cache',
-                \substr($archiveHash, 0, 3),
-                \substr($archiveHash, 3),
-                $path,
-            ]),
-            'CopySource' => "$this->bucket/$sourceKey",
-            'MetadataDirective' => 'REPLACE',
+        $batch = [];
+        $client = $this->getS3Client();
+        foreach ($archive->iterator() as $item) {
+            $sourceKey = $this->key($item->hash);
+            $batch[] = $client->getCommand('CopyObject', [
+                'Bucket' => $this->bucket,
+                'ContentType' => $item->type,
+                'Key' => \implode('/', [
+                    'cache',
+                    \substr($archiveHash, 0, 3),
+                    \substr($archiveHash, 3),
+                    $item->filename,
+                ]),
+                'CopySource' => "$this->bucket/$sourceKey",
+                'MetadataDirective' => 'REPLACE',
+            ]);
+        }
+        $pool = new CommandPool($client, $batch, [
+            'concurrency' => 250,
+            'fulfilled' => $callback,
+            'rejected' => $callback,
         ]);
-        $result = $result->get('CopyObjectResult');
+        $promise = $pool->promise();
+        $promise->wait();
 
-        return \is_string($result['ETag'] ?? null);
+        return true;
     }
 
     public function heads(string ...$hashes): array
diff --git a/EMS/common-bundle/src/Storage/Service/StorageInterface.php b/EMS/common-bundle/src/Storage/Service/StorageInterface.php
index be23ce304..36947a630 100644
--- a/EMS/common-bundle/src/Storage/Service/StorageInterface.php
+++ b/EMS/common-bundle/src/Storage/Service/StorageInterface.php
@@ -4,6 +4,7 @@
 
 namespace EMS\CommonBundle\Storage\Service;
 
+use EMS\CommonBundle\Storage\Archive;
 use EMS\CommonBundle\Storage\File\FileInterface;
 use EMS\CommonBundle\Storage\Processor\Config;
 use EMS\CommonBundle\Storage\StreamWrapper;
@@ -85,5 +86,5 @@ public function readFromArchiveInCache(string $hash, string $path): ?StreamWrapp
 
     public function addFileInArchiveCache(string $hash, SplFileInfo $file, string $mimeType): bool;
 
-    public function copyFileInArchiveCache(string $archiveHash, string $fileHash, string $path, string $mimeType): bool;
+    public function loadArchiveItemsInCache(string $archiveHash, Archive $archive, callable $callback = null): bool;
 }
diff --git a/EMS/common-bundle/src/Storage/StorageManager.php b/EMS/common-bundle/src/Storage/StorageManager.php
index fe70cd8f4..c71c9329b 100644
--- a/EMS/common-bundle/src/Storage/StorageManager.php
+++ b/EMS/common-bundle/src/Storage/StorageManager.php
@@ -620,12 +620,10 @@ private function getStreamFromJsonArchive(string $hash, string $path, TempFile $
             throw new NotFoundHttpException(\sprintf('File %s not found in archive %s', $path, $hash));
         }
         $counter = 0;
-        foreach ($archive->iterator() as $item) {
-            foreach ($this->adapters as $adapter) {
-                if ($adapter->copyFileInArchiveCache($hash, $item->hash, $item->filename, $item->type)) {
-                    ++$counter;
-                    break;
-                }
+        foreach ($this->adapters as $adapter) {
+            if ($adapter->loadArchiveItemsInCache($hash, $archive)) {
+                ++$counter;
+                break;
             }
         }
         if ($archive->getCount() === $counter) {
@@ -696,14 +694,9 @@ public function setHeadChunkSize(int $chunkSize): void
 
     public function loadArchiveItemsInCache(string $archiveHash, Archive $archive, callable $callback = null): void
     {
-        foreach ($archive->iterator() as $archiveItem) {
-            foreach ($this->adapters as $adapter) {
-                if ($adapter->copyFileInArchiveCache($archiveHash, $archiveItem->hash, $archiveItem->filename, $archiveItem->type)) {
-                    if (null !== $callback) {
-                        $callback();
-                    }
-                    break;
-                }
+        foreach ($this->adapters as $adapter) {
+            if ($adapter->loadArchiveItemsInCache($archiveHash, $archive, $callback)) {
+                break;
             }
         }
     }

From 4468bba9738ed55b7dd5cafed7fce3a27a1f87a7 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 20:55:24 +0100
Subject: [PATCH 15/22] ref: kind if symlink in S3 storage services

---
 .../src/Storage/Service/S3Storage.php         | 24 +++++++++++++++----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/EMS/common-bundle/src/Storage/Service/S3Storage.php b/EMS/common-bundle/src/Storage/Service/S3Storage.php
index aaeea0baa..c8cf6a054 100644
--- a/EMS/common-bundle/src/Storage/Service/S3Storage.php
+++ b/EMS/common-bundle/src/Storage/Service/S3Storage.php
@@ -309,12 +309,25 @@ public function readFromArchiveInCache(string $hash, string $path): ?StreamWrapp
         } catch (\RuntimeException) {
             return null;
         }
-        $stream = $response['Body'] ?? null;
+
+        $hash = $response->get('Metadata')['hash'] ?? null;
+        if (\is_string($hash)) {
+            $masterResponse = $this->getS3Client()->getObject([
+                'Bucket' => $this->bucket,
+                'Key' => $this->key($hash)
+            ]);
+            $stream = $masterResponse['Body'] ?? null;
+            $size = \intval($masterResponse['ContentLength']);
+        } else {
+            $stream = $response['Body'] ?? null;
+            $size = \intval($response['ContentLength']);
+        }
+
         if (!$stream instanceof StreamInterface) {
             return null;
         }
 
-        return new StreamWrapper($stream, $response['ContentType'] ?? MimeTypes::APPLICATION_OCTET_STREAM->value, \intval($response['ContentLength']));
+        return new StreamWrapper($stream, $response['ContentType'] ?? MimeTypes::APPLICATION_OCTET_STREAM->value, $size);
     }
 
     public function addFileInArchiveCache(string $hash, SplFileInfo $file, string $mimeType): bool
@@ -339,8 +352,7 @@ public function loadArchiveItemsInCache(string $archiveHash, Archive $archive, c
         $batch = [];
         $client = $this->getS3Client();
         foreach ($archive->iterator() as $item) {
-            $sourceKey = $this->key($item->hash);
-            $batch[] = $client->getCommand('CopyObject', [
+            $batch[] = $client->getCommand('PutObject', [
                 'Bucket' => $this->bucket,
                 'ContentType' => $item->type,
                 'Key' => \implode('/', [
@@ -349,8 +361,10 @@ public function loadArchiveItemsInCache(string $archiveHash, Archive $archive, c
                     \substr($archiveHash, 3),
                     $item->filename,
                 ]),
-                'CopySource' => "$this->bucket/$sourceKey",
                 'MetadataDirective' => 'REPLACE',
+                'Metadata' => [
+                    'hash' => $item->hash,
+                ],
             ]);
         }
         $pool = new CommandPool($client, $batch, [

From ec898cd65a12bc6a5a29abf90a9b88393baeba20 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 20:56:55 +0100
Subject: [PATCH 16/22] chore: phpcs

---
 EMS/common-bundle/src/Storage/Service/S3Storage.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EMS/common-bundle/src/Storage/Service/S3Storage.php b/EMS/common-bundle/src/Storage/Service/S3Storage.php
index c8cf6a054..9468cfaa3 100644
--- a/EMS/common-bundle/src/Storage/Service/S3Storage.php
+++ b/EMS/common-bundle/src/Storage/Service/S3Storage.php
@@ -314,7 +314,7 @@ public function readFromArchiveInCache(string $hash, string $path): ?StreamWrapp
         if (\is_string($hash)) {
             $masterResponse = $this->getS3Client()->getObject([
                 'Bucket' => $this->bucket,
-                'Key' => $this->key($hash)
+                'Key' => $this->key($hash),
             ]);
             $stream = $masterResponse['Body'] ?? null;
             $size = \intval($masterResponse['ContentLength']);

From 8cc4708456e723e73beb34116eef4b2ac933f4f2 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 21:34:48 +0100
Subject: [PATCH 17/22] feat: extra parameter to not try extract again if not
 found in cache

---
 EMS/common-bundle/src/Controller/FileController.php   | 4 ++--
 EMS/common-bundle/src/Storage/Processor/Processor.php | 4 ++--
 EMS/common-bundle/src/Storage/StorageManager.php      | 5 ++++-
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/EMS/common-bundle/src/Controller/FileController.php b/EMS/common-bundle/src/Controller/FileController.php
index 4cf87773a..a00883fbd 100644
--- a/EMS/common-bundle/src/Controller/FileController.php
+++ b/EMS/common-bundle/src/Controller/FileController.php
@@ -88,11 +88,11 @@ public function generateLocalImage(Request $request, string $filename, string $c
         return $response;
     }
 
-    public function assetInArchive(Request $request, string $hash, string $path, int $maxAge = 604800): Response
+    public function assetInArchive(Request $request, string $hash, string $path, int $maxAge = 604800, bool $extract = true): Response
     {
         $this->closeSession($request);
 
-        return $this->processor->getResponseFromArchive($request, $hash, $path, $maxAge);
+        return $this->processor->getResponseFromArchive($request, $hash, $path, $maxAge, $extract);
     }
 
     private function getFile(Request $request, string $hash, string $disposition): Response
diff --git a/EMS/common-bundle/src/Storage/Processor/Processor.php b/EMS/common-bundle/src/Storage/Processor/Processor.php
index 41910bd1b..5f94aae6c 100644
--- a/EMS/common-bundle/src/Storage/Processor/Processor.php
+++ b/EMS/common-bundle/src/Storage/Processor/Processor.php
@@ -339,9 +339,9 @@ private function locate(string $filename): string
         return $path;
     }
 
-    public function getResponseFromArchive(Request $request, string $hash, string $path, int $maxAge): Response
+    public function getResponseFromArchive(Request $request, string $hash, string $path, int $maxAge, bool $extract): Response
     {
-        $streamWrapper = $this->storageManager->getStreamFromArchive($hash, $path);
+        $streamWrapper = $this->storageManager->getStreamFromArchive($hash, $path, $extract);
         $response = $this->getResponseFromStreamInterface($streamWrapper->getStream(), $request);
         $response->headers->add([
             Headers::CONTENT_DISPOSITION => HeaderUtils::DISPOSITION_INLINE.'; '.HeaderUtils::toString(['filename' => \basename($path)], ';'),
diff --git a/EMS/common-bundle/src/Storage/StorageManager.php b/EMS/common-bundle/src/Storage/StorageManager.php
index c71c9329b..2193de7b3 100644
--- a/EMS/common-bundle/src/Storage/StorageManager.php
+++ b/EMS/common-bundle/src/Storage/StorageManager.php
@@ -527,7 +527,7 @@ public function saveCache(Config $config, FileInterface $file): void
         }
     }
 
-    public function getStreamFromArchive(string $hash, string $path): StreamWrapper
+    public function getStreamFromArchive(string $hash, string $path, bool $extract = true): StreamWrapper
     {
         foreach ($this->adapters as $adapter) {
             $stream = $adapter->readFromArchiveInCache($hash, $path);
@@ -535,6 +535,9 @@ public function getStreamFromArchive(string $hash, string $path): StreamWrapper
                 return $stream;
             }
         }
+        if (!$extract) {
+            throw new NotFoundHttpException(\sprintf('File %s not found in cache for archive %s', $path, $hash));
+        }
         $this->logger->debug(\sprintf('File %s from archive %s is not in cache', $path, $hash));
 
         if (!$this->head($hash)) {

From 12216388f674659c5132b2d8eb2fbd500c1e3f66 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 22:03:22 +0100
Subject: [PATCH 18/22] feat: extra indexResource parameter for directory path

---
 EMS/common-bundle/src/Controller/FileController.php   | 4 ++--
 EMS/common-bundle/src/Storage/Processor/Processor.php | 4 ++--
 EMS/common-bundle/src/Storage/StorageManager.php      | 5 ++++-
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/EMS/common-bundle/src/Controller/FileController.php b/EMS/common-bundle/src/Controller/FileController.php
index a00883fbd..5030f47e3 100644
--- a/EMS/common-bundle/src/Controller/FileController.php
+++ b/EMS/common-bundle/src/Controller/FileController.php
@@ -88,11 +88,11 @@ public function generateLocalImage(Request $request, string $filename, string $c
         return $response;
     }
 
-    public function assetInArchive(Request $request, string $hash, string $path, int $maxAge = 604800, bool $extract = true): Response
+    public function assetInArchive(Request $request, string $hash, string $path, int $maxAge = 604800, bool $extract = true, string $indexResource = null): Response
     {
         $this->closeSession($request);
 
-        return $this->processor->getResponseFromArchive($request, $hash, $path, $maxAge, $extract);
+        return $this->processor->getResponseFromArchive($request, $hash, $path, $maxAge, $extract, $indexResource);
     }
 
     private function getFile(Request $request, string $hash, string $disposition): Response
diff --git a/EMS/common-bundle/src/Storage/Processor/Processor.php b/EMS/common-bundle/src/Storage/Processor/Processor.php
index 5f94aae6c..a8a234c79 100644
--- a/EMS/common-bundle/src/Storage/Processor/Processor.php
+++ b/EMS/common-bundle/src/Storage/Processor/Processor.php
@@ -339,9 +339,9 @@ private function locate(string $filename): string
         return $path;
     }
 
-    public function getResponseFromArchive(Request $request, string $hash, string $path, int $maxAge, bool $extract): Response
+    public function getResponseFromArchive(Request $request, string $hash, string $path, int $maxAge, bool $extract, ?string $indexResource): Response
     {
-        $streamWrapper = $this->storageManager->getStreamFromArchive($hash, $path, $extract);
+        $streamWrapper = $this->storageManager->getStreamFromArchive($hash, $path, $extract, $indexResource);
         $response = $this->getResponseFromStreamInterface($streamWrapper->getStream(), $request);
         $response->headers->add([
             Headers::CONTENT_DISPOSITION => HeaderUtils::DISPOSITION_INLINE.'; '.HeaderUtils::toString(['filename' => \basename($path)], ';'),
diff --git a/EMS/common-bundle/src/Storage/StorageManager.php b/EMS/common-bundle/src/Storage/StorageManager.php
index 2193de7b3..6da897830 100644
--- a/EMS/common-bundle/src/Storage/StorageManager.php
+++ b/EMS/common-bundle/src/Storage/StorageManager.php
@@ -527,8 +527,11 @@ public function saveCache(Config $config, FileInterface $file): void
         }
     }
 
-    public function getStreamFromArchive(string $hash, string $path, bool $extract = true): StreamWrapper
+    public function getStreamFromArchive(string $hash, string $path, bool $extract = true, string $indexResource = null): StreamWrapper
     {
+        if (null !== $indexResource && ('' === $path || \str_ends_with($path, '/'))) {
+            $path .= $indexResource;
+        }
         foreach ($this->adapters as $adapter) {
             $stream = $adapter->readFromArchiveInCache($hash, $path);
             if (null !== $stream) {

From 81745f7804343e3c310e3ba7d69a1adb134c9e3a Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Mon, 25 Nov 2024 22:03:43 +0100
Subject: [PATCH 19/22] feat: better not found message

---
 EMS/common-bundle/src/Storage/StorageManager.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EMS/common-bundle/src/Storage/StorageManager.php b/EMS/common-bundle/src/Storage/StorageManager.php
index 6da897830..3c4f7a75c 100644
--- a/EMS/common-bundle/src/Storage/StorageManager.php
+++ b/EMS/common-bundle/src/Storage/StorageManager.php
@@ -539,7 +539,7 @@ public function getStreamFromArchive(string $hash, string $path, bool $extract =
             }
         }
         if (!$extract) {
-            throw new NotFoundHttpException(\sprintf('File %s not found in cache for archive %s', $path, $hash));
+            throw new NotFoundHttpException(\sprintf('File %s not found', $path));
         }
         $this->logger->debug(\sprintf('File %s from archive %s is not in cache', $path, $hash));
 

From e30261a3907d62d6c017eedce84ecb970a57ee08 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Tue, 26 Nov 2024 12:59:23 +0100
Subject: [PATCH 20/22] feat: publish api (client&server)

---
 .../src/Common/CoreApi/Endpoint/Data/Data.php | 11 ++++
 .../CoreApi/Endpoint/Data/DataInterface.php   |  2 +
 .../Controller/Api/Data/PublishController.php | 57 +++++++++++++++++++
 .../src/Resources/config/controllers.xml      |  7 +++
 .../src/Resources/config/routing/api.xml      |  2 +
 .../src/Resources/config/routing/api/data.xml | 14 +++++
 6 files changed, 93 insertions(+)
 create mode 100644 EMS/core-bundle/src/Controller/Api/Data/PublishController.php
 create mode 100644 EMS/core-bundle/src/Resources/config/routing/api/data.xml

diff --git a/EMS/common-bundle/src/Common/CoreApi/Endpoint/Data/Data.php b/EMS/common-bundle/src/Common/CoreApi/Endpoint/Data/Data.php
index 44e8ba2eb..dfb1f5693 100644
--- a/EMS/common-bundle/src/Common/CoreApi/Endpoint/Data/Data.php
+++ b/EMS/common-bundle/src/Common/CoreApi/Endpoint/Data/Data.php
@@ -149,4 +149,15 @@ private function makeResource(?string ...$path): string
     {
         return \implode('/', \array_merge($this->endPoint, \array_filter($path)));
     }
+
+    public function publish(string $ouuid, string $environment, string $revisionId = null): bool
+    {
+        $resource = $this->makeResource('publish', $ouuid, $environment, $revisionId ?? '');
+        $success = $this->client->post($resource)->getData()['success'] ?? null;
+        if (!\is_bool($success)) {
+            throw new \RuntimeException('Unexpected: search must be a boolean');
+        }
+
+        return $success;
+    }
 }
diff --git a/EMS/common-bundle/src/Contracts/CoreApi/Endpoint/Data/DataInterface.php b/EMS/common-bundle/src/Contracts/CoreApi/Endpoint/Data/DataInterface.php
index da632d7f3..977d86656 100644
--- a/EMS/common-bundle/src/Contracts/CoreApi/Endpoint/Data/DataInterface.php
+++ b/EMS/common-bundle/src/Contracts/CoreApi/Endpoint/Data/DataInterface.php
@@ -77,4 +77,6 @@ public function update(string $ouuid, array $rawData): DraftInterface;
      * @throws CoreApiExceptionInterface
      */
     public function save(string $ouuid, array $rawData, int $mode = self::MODE_UPDATE, bool $discardDraft = true): int;
+
+    public function publish(string $ouuid, string $environment, string $revisionId = null): bool;
 }
diff --git a/EMS/core-bundle/src/Controller/Api/Data/PublishController.php b/EMS/core-bundle/src/Controller/Api/Data/PublishController.php
new file mode 100644
index 000000000..64557f672
--- /dev/null
+++ b/EMS/core-bundle/src/Controller/Api/Data/PublishController.php
@@ -0,0 +1,57 @@
+<?php
+
+declare(strict_types=1);
+
+namespace EMS\CoreBundle\Controller\Api\Data;
+
+use Doctrine\ORM\NonUniqueResultException;
+use EMS\CoreBundle\Entity\Revision;
+use EMS\CoreBundle\Service\ContentTypeService;
+use EMS\CoreBundle\Service\EnvironmentService;
+use EMS\CoreBundle\Service\PublishService;
+use EMS\CoreBundle\Service\Revision\RevisionService;
+use Symfony\Component\HttpFoundation\JsonResponse;
+use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
+
+class PublishController
+{
+    public function __construct(
+        private readonly PublishService $publishService,
+        private readonly RevisionService $revisionService,
+        private readonly ContentTypeService $contentTypeService,
+        private readonly EnvironmentService $environmentService,
+    ) {
+    }
+
+    public function publish(string $contentTypeName, string $ouuid, string $targetEnvironmentName, Revision $revision = null): JsonResponse
+    {
+        if (null === $revision) {
+            $contentType = $this->contentTypeService->getByName($contentTypeName);
+            if (false === $contentType) {
+                throw new \RuntimeException(\sprintf('Content type %s not found', $contentTypeName));
+            }
+            $revision = $this->revisionService->getCurrentRevisionForEnvironment($ouuid, $contentType, $contentType->giveEnvironment());
+        } elseif ($revision->giveContentType()->getName() !== $contentTypeName) {
+            throw new \RuntimeException(\sprintf('Content type mismatch for revision %d: Expected %s is in fact of type %s', $revision->getId(), $contentTypeName, $revision->giveContentType()->getName()));
+        }
+        if (null === $revision) {
+            throw new \RuntimeException(\sprintf('Revision not found for OUUID %s and Content type %s', $ouuid, $contentTypeName));
+        }
+
+        $targetEnvironment = $this->environmentService->getByName($targetEnvironmentName);
+        if (false === $targetEnvironment) {
+            throw new \RuntimeException(\sprintf('Target environment %s not found', $targetEnvironmentName));
+        }
+
+        try {
+            $publishedCounter = $this->publishService->publish($revision, $targetEnvironment);
+        } catch (NonUniqueResultException) {
+            throw new NotFoundHttpException('Document not found');
+        }
+
+        return new JsonResponse([
+           'success' => true,
+           'already-published' => 0 === $publishedCounter,
+        ]);
+    }
+}
diff --git a/EMS/core-bundle/src/Resources/config/controllers.xml b/EMS/core-bundle/src/Resources/config/controllers.xml
index da94e7107..91e54f765 100644
--- a/EMS/core-bundle/src/Resources/config/controllers.xml
+++ b/EMS/core-bundle/src/Resources/config/controllers.xml
@@ -628,6 +628,13 @@
             <tag name="container.service_subscriber"/>
             <tag name="controller.service_arguments"/>
         </service>
+        <service id="EMS\CoreBundle\Controller\Api\Data\PublishController">
+            <argument type="service" id="ems.service.publish" />
+            <argument type="service" id="ems.service.revision" />
+            <argument type="service" id="ems.service.contenttype" />
+            <argument type="service" id="ems.service.environment" />
+            <tag name="controller.service_arguments"/>
+        </service>
 
     </services>
 </container>
diff --git a/EMS/core-bundle/src/Resources/config/routing/api.xml b/EMS/core-bundle/src/Resources/config/routing/api.xml
index 3394806e5..24f8e109b 100644
--- a/EMS/core-bundle/src/Resources/config/routing/api.xml
+++ b/EMS/core-bundle/src/Resources/config/routing/api.xml
@@ -23,5 +23,7 @@
             prefix="/api/meta"/>
     <import resource="@EMSCoreBundle/Resources/config/routing/api/search.xml"
             prefix="/api/search"/>
+    <import resource="@EMSCoreBundle/Resources/config/routing/api/data.xml"
+            prefix="/api/data"/>
 
 </routes>
diff --git a/EMS/core-bundle/src/Resources/config/routing/api/data.xml b/EMS/core-bundle/src/Resources/config/routing/api/data.xml
new file mode 100644
index 000000000..456ea4033
--- /dev/null
+++ b/EMS/core-bundle/src/Resources/config/routing/api/data.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+<routes xmlns="http://symfony.com/schema/routing"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://symfony.com/schema/routing https://symfony.com/schema/routing/routing-1.0.xsd">
+
+    <route id="emsco_api_data_publish" path="/{contentTypeName}/publish/{ouuid}/{targetEnvironmentName}/{revision}"
+           controller="EMS\CoreBundle\Controller\Api\Data\PublishController::publish"
+           methods="POST"
+           format="json">
+        <default key="revision" xsi:nil="true"/>
+    </route>
+
+</routes>

From 3ff5eae212faa333a6ebda0f66529677d9423b4b Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Tue, 26 Nov 2024 13:28:39 +0100
Subject: [PATCH 21/22] feat: publish command

---
 .../src/Command/Document/PublishCommand.php   | 64 +++++++++++++++++++
 .../src/Resources/config/commands.xml         |  4 ++
 2 files changed, 68 insertions(+)
 create mode 100644 EMS/common-bundle/src/Command/Document/PublishCommand.php

diff --git a/EMS/common-bundle/src/Command/Document/PublishCommand.php b/EMS/common-bundle/src/Command/Document/PublishCommand.php
new file mode 100644
index 000000000..5bbf008fe
--- /dev/null
+++ b/EMS/common-bundle/src/Command/Document/PublishCommand.php
@@ -0,0 +1,64 @@
+<?php
+
+declare(strict_types=1);
+
+namespace EMS\CommonBundle\Command\Document;
+
+use EMS\CommonBundle\Common\Admin\AdminHelper;
+use EMS\CommonBundle\Common\Command\AbstractCommand;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Logger\ConsoleLogger;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class PublishCommand extends AbstractCommand
+{
+    private const ARGUMENT_CONTENT_TYPE = 'content-type';
+    private const ARGUMENT_OUUID = 'ouuid';
+    private const ARGUMENT_TARGET_ENVIRONMENT = 'target-environment';
+    private const ARGUMENT_REVISION_ID = 'revisison-id';
+    private string $contentTypeName;
+    private string $ouuid;
+    private string $targetEnvironmentName;
+    private ?string $revisionId;
+
+    public function __construct(private readonly AdminHelper $adminHelper)
+    {
+        parent::__construct();
+    }
+
+    protected function configure(): void
+    {
+        parent::configure();
+        $this
+            ->addArgument(self::ARGUMENT_CONTENT_TYPE, InputArgument::REQUIRED, \sprintf('Content-type\'s name'))
+            ->addArgument(self::ARGUMENT_OUUID, InputArgument::REQUIRED, \sprintf('OUUID of the document to publish'))
+            ->addArgument(self::ARGUMENT_TARGET_ENVIRONMENT, InputArgument::REQUIRED, \sprintf('Environment\'s name to publish in'))
+            ->addArgument(self::ARGUMENT_REVISION_ID, InputArgument::OPTIONAL, \sprintf('Revision ID of the revision to publish, will take the revision publish in the default environment otherwise.'))
+        ;
+    }
+
+    public function initialize(InputInterface $input, OutputInterface $output): void
+    {
+        parent::initialize($input, $output);
+        $this->adminHelper->setLogger(new ConsoleLogger($output));
+        $this->contentTypeName = $this->getArgumentString(self::ARGUMENT_CONTENT_TYPE);
+        $this->targetEnvironmentName = $this->getArgumentString(self::ARGUMENT_TARGET_ENVIRONMENT);
+        $this->ouuid = $this->getArgumentString(self::ARGUMENT_OUUID);
+        $this->revisionId = $this->getArgumentStringNull(self::ARGUMENT_REVISION_ID);
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $this->io->title(\sprintf('Publish the document %s to the environment %s', $this->ouuid, $this->targetEnvironmentName));
+        $api = $this->adminHelper->getCoreApi()->data($this->contentTypeName);
+        if (!$api->publish($this->ouuid, $this->targetEnvironmentName, $this->revisionId)) {
+            $this->io->error('The document was not published');
+
+            return self::EXECUTE_ERROR;
+        }
+        $this->io->error('The document has been successfully published');
+
+        return self::EXECUTE_SUCCESS;
+    }
+}
diff --git a/EMS/common-bundle/src/Resources/config/commands.xml b/EMS/common-bundle/src/Resources/config/commands.xml
index cfffffe51..5f323df30 100644
--- a/EMS/common-bundle/src/Resources/config/commands.xml
+++ b/EMS/common-bundle/src/Resources/config/commands.xml
@@ -81,6 +81,10 @@
             <argument type="string">%kernel.project_dir%</argument>
             <tag name="console.command" command="ems:document:download"/>
         </service>
+        <service id="ems.command.document.publish" class="EMS\CommonBundle\Command\Document\PublishCommand">
+            <argument type="service" id="ems.helper.admin_api"/>
+            <tag name="console.command" command="ems:document:publish"/>
+        </service>
         <service id="ems.command.document.update" class="EMS\CommonBundle\Command\Document\UpdateCommand">
             <argument type="service" id="ems.helper.admin_api"/>
             <argument type="string">%kernel.project_dir%</argument>

From 1ddd534c1f8ab0ccfb14f4caf6533766085ba0d3 Mon Sep 17 00:00:00 2001
From: Mathieu De Keyzer <me@theus.be>
Date: Tue, 26 Nov 2024 13:52:01 +0100
Subject: [PATCH 22/22] ref: using giveByName functions

---
 .../src/Controller/Api/Data/PublishController.php      | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/EMS/core-bundle/src/Controller/Api/Data/PublishController.php b/EMS/core-bundle/src/Controller/Api/Data/PublishController.php
index 64557f672..982a0abcb 100644
--- a/EMS/core-bundle/src/Controller/Api/Data/PublishController.php
+++ b/EMS/core-bundle/src/Controller/Api/Data/PublishController.php
@@ -26,10 +26,7 @@ public function __construct(
     public function publish(string $contentTypeName, string $ouuid, string $targetEnvironmentName, Revision $revision = null): JsonResponse
     {
         if (null === $revision) {
-            $contentType = $this->contentTypeService->getByName($contentTypeName);
-            if (false === $contentType) {
-                throw new \RuntimeException(\sprintf('Content type %s not found', $contentTypeName));
-            }
+            $contentType = $this->contentTypeService->giveByName($contentTypeName);
             $revision = $this->revisionService->getCurrentRevisionForEnvironment($ouuid, $contentType, $contentType->giveEnvironment());
         } elseif ($revision->giveContentType()->getName() !== $contentTypeName) {
             throw new \RuntimeException(\sprintf('Content type mismatch for revision %d: Expected %s is in fact of type %s', $revision->getId(), $contentTypeName, $revision->giveContentType()->getName()));
@@ -38,10 +35,7 @@ public function publish(string $contentTypeName, string $ouuid, string $targetEn
             throw new \RuntimeException(\sprintf('Revision not found for OUUID %s and Content type %s', $ouuid, $contentTypeName));
         }
 
-        $targetEnvironment = $this->environmentService->getByName($targetEnvironmentName);
-        if (false === $targetEnvironment) {
-            throw new \RuntimeException(\sprintf('Target environment %s not found', $targetEnvironmentName));
-        }
+        $targetEnvironment = $this->environmentService->giveByName($targetEnvironmentName);
 
         try {
             $publishedCounter = $this->publishService->publish($revision, $targetEnvironment);