refactor: Use multi-process documents:downscale command. Reduce con…

…ditions deepness with early return pattern
roadiz · Aug 27, 2024 · e255e0f · e255e0f
1 parent a6aac04
commit e255e0f
Show file tree

Hide file tree

Showing 37 changed files with 808 additions and 736 deletions.
diff --git a/lib/Documents/src/Console/AbstractDocumentCommand.php b/lib/Documents/src/Console/AbstractDocumentCommand.php
@@ -16,16 +16,13 @@
 
 abstract class AbstractDocumentCommand extends Command
 {
-    protected ManagerRegistry $managerRegistry;
-    protected ImageManager $imageManager;
-    protected FilesystemOperator $documentsStorage;
-
-    public function __construct(ManagerRegistry $managerRegistry, ImageManager $imageManager, FilesystemOperator $documentsStorage)
-    {
-        parent::__construct();
-        $this->managerRegistry = $managerRegistry;
-        $this->imageManager = $imageManager;
-        $this->documentsStorage = $documentsStorage;
+    public function __construct(
+        protected ManagerRegistry $managerRegistry,
+        protected ImageManager $imageManager,
+        protected FilesystemOperator $documentsStorage,
+        ?string $name = null
+    ) {
+        parent::__construct($name);
     }
 
     protected function getManager(): ObjectManager
@@ -49,11 +46,11 @@ protected function getDocumentRepository(): DocumentRepositoryInterface
      * @param callable $method
      * @param SymfonyStyle $io
      * @param int $batchSize
-     * @return int|void
+     * @return int
      * @throws \Doctrine\ORM\NoResultException
      * @throws \Doctrine\ORM\NonUniqueResultException
      */
-    protected function onEachDocument(callable $method, SymfonyStyle $io, int $batchSize = 20)
+    protected function onEachDocument(callable $method, SymfonyStyle $io, int $batchSize = 20): int
     {
         $i = 0;
         $manager = $this->getManager();
@@ -85,5 +82,6 @@ protected function onEachDocument(callable $method, SymfonyStyle $io, int $batch
         }
         $manager->flush();
         $io->progressFinish();
+        return 0;
     }
 }
diff --git a/lib/Documents/src/Console/DocumentAverageColorCommand.php b/lib/Documents/src/Console/DocumentAverageColorCommand.php
@@ -27,32 +27,32 @@ protected function execute(InputInterface $input, OutputInterface $output): int
     {
         $this->io = new SymfonyStyle($input, $output);
 
-        $this->onEachDocument(function (DocumentInterface $document) {
+        return $this->onEachDocument(function (DocumentInterface $document) {
             $this->updateDocumentColor($document);
         }, new SymfonyStyle($input, $output));
-
-        return 0;
     }
 
     private function updateDocumentColor(DocumentInterface $document): void
     {
-        if ($document->isImage() && $document instanceof AdvancedDocumentInterface) {
-            $mountPath = $document->getMountPath();
-            if (null === $mountPath) {
-                return;
-            }
-            try {
-                $mediumColor = (new AverageColorResolver())->getAverageColor($this->imageManager->make(
-                    $this->documentsStorage->readStream($mountPath)
-                ));
-                $document->setImageAverageColor($mediumColor);
-            } catch (NotReadableException $exception) {
-                /*
-                 * Do nothing
-                 * just return 0 width and height
-                 */
-                $this->io->error($mountPath . ' is not a readable image.');
-            }
+        if (!$document->isImage() || !($document instanceof AdvancedDocumentInterface)) {
+            return;
+        }
+
+        $mountPath = $document->getMountPath();
+        if (null === $mountPath) {
+            return;
+        }
+        try {
+            $mediumColor = (new AverageColorResolver())->getAverageColor($this->imageManager->make(
+                $this->documentsStorage->readStream($mountPath)
+            ));
+            $document->setImageAverageColor($mediumColor);
+        } catch (NotReadableException $exception) {
+            /*
+             * Do nothing
+             * just return 0 width and height
+             */
+            $this->io->error($mountPath . ' is not a readable image.');
         }
     }
 }
diff --git a/lib/Documents/src/Console/DocumentClearFolderCommand.php b/lib/Documents/src/Console/DocumentClearFolderCommand.php
@@ -61,30 +61,30 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             return 0;
         }
 
-        if (
-            $this->io->askQuestion(new ConfirmationQuestion(
-                sprintf('Are you sure to delete permanently %d documents?', $count),
-                false
-            ))
-        ) {
-            /** @var DocumentInterface[] $results */
-            $results = $this->getDocumentQueryBuilder($folder)
-                ->select('d')
-                ->getQuery()
-                ->getResult();
+        if (!$this->io->askQuestion(new ConfirmationQuestion(
+            sprintf('Are you sure to delete permanently %d documents?', $count),
+            false
+        ))) {
+            return 0;
+        }
+
+        /** @var DocumentInterface[] $results */
+        $results = $this->getDocumentQueryBuilder($folder)
+            ->select('d')
+            ->getQuery()
+            ->getResult();
 
-            $this->io->progressStart($count);
-            foreach ($results as $document) {
-                $em->remove($document);
-                if (($i % $batchSize) === 0) {
-                    $em->flush(); // Executes all updates.
-                }
-                ++$i;
-                $this->io->progressAdvance();
+        $this->io->progressStart($count);
+        foreach ($results as $document) {
+            $em->remove($document);
+            if (($i % $batchSize) === 0) {
+                $em->flush(); // Executes all updates.
             }
-            $em->flush();
-            $this->io->progressFinish();
+            ++$i;
+            $this->io->progressAdvance();
         }
+        $em->flush();
+        $this->io->progressFinish();
 
         return 0;
     }

diff --git a/lib/Documents/src/Console/DocumentDownscaleCommand.php b/lib/Documents/src/Console/DocumentDownscaleCommand.php
@@ -12,86 +12,135 @@
 use RZ\Roadiz\Documents\Events\CachePurgeAssetsRequestEvent;
 use RZ\Roadiz\Documents\Models\DocumentInterface;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Question\ConfirmationQuestion;
 use Symfony\Component\Console\Style\SymfonyStyle;
+use Symfony\Component\Process\PhpSubprocess;
 use Symfony\Contracts\EventDispatcher\EventDispatcherInterface;
 
 /**
  * Command line utils for process document downscale.
  */
 class DocumentDownscaleCommand extends AbstractDocumentCommand
 {
-    private ?int $maxPixelSize;
-    private DownscaleImageManager $downscaler;
-    private EventDispatcherInterface $dispatcher;
-
     public function __construct(
         ManagerRegistry $managerRegistry,
         ImageManager $imageManager,
         FilesystemOperator $documentsStorage,
-        ?int $maxPixelSize,
-        DownscaleImageManager $downscaler,
-        EventDispatcherInterface $dispatcher
+        private readonly ?int $maxPixelSize,
+        private readonly DownscaleImageManager $downscaler,
+        private readonly EventDispatcherInterface $dispatcher,
+        ?string $name = null
     ) {
-        parent::__construct($managerRegistry, $imageManager, $documentsStorage);
-        $this->maxPixelSize = $maxPixelSize;
-        $this->downscaler = $downscaler;
-        $this->dispatcher = $dispatcher;
+        parent::__construct($managerRegistry, $imageManager, $documentsStorage, $name);
     }
 
     protected function configure(): void
     {
         $this->setName('documents:downscale')
+            ->addOption('process-count', 'p', InputOption::VALUE_REQUIRED, 'Number of processes to run in parallel.', 1)
+            ->addOption('limit', null, InputOption::VALUE_REQUIRED, 'Number of document to process in one process', null)
+            ->addOption('offset', null, InputOption::VALUE_REQUIRED, 'Offset number of document to process after', null)
             ->setDescription('Downscale every document according to max pixel size defined in configuration.');
     }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
         $io = new SymfonyStyle($input, $output);
 
-        if (null !== $this->maxPixelSize && $this->maxPixelSize > 0) {
-            $confirmation = new ConfirmationQuestion(
-                '<question>Are you sure to downscale all your image documents to ' . $this->maxPixelSize . 'px?</question>',
-                false
-            );
-            if (
-                $io->askQuestion(
-                    $confirmation
-                )
-            ) {
-                /** @var DocumentInterface[] $documents */
-                $documents = $this->getDocumentRepository()
-                    ->findBy([
-                        'mimeType' => [
-                            'image/png',
-                            'image/jpeg',
-                            'image/gif',
-                            'image/tiff',
-                        ],
-                        'raw' => false,
-                    ]);
-                $io->progressStart(\count($documents));
-
-                foreach ($documents as $document) {
-                    try {
-                        $this->downscaler->processDocumentFromExistingRaw($document);
-                    } catch (NotReadableException $exception) {
-                        $io->error($exception->getMessage() . ' - ' . (string) $document);
-                    }
-                    $io->progressAdvance();
-                }
-
-                $io->progressFinish();
-                $io->success('Every documents have been downscaled, a raw version has been kept.');
-
-                $this->dispatcher->dispatch(new CachePurgeAssetsRequestEvent());
-            }
-            return 0;
-        } else {
+        if (null === $this->maxPixelSize || $this->maxPixelSize <= 0) {
             $io->warning('Your configuration is not set for downscaling documents.');
             $io->note('Add <info>assetsProcessing.maxPixelSize</info> parameter in your <info>config.yml</info> file.');
             return 1;
         }
+
+        $confirmation = new ConfirmationQuestion(
+            '<question>Are you sure to downscale all your image documents to ' . $this->maxPixelSize . 'px?</question>',
+            false
+        );
+        if ($input->isInteractive() && !$io->askQuestion($confirmation)) {
+            return 0;
+        }
+
+        $criteria = [
+            'mimeType' => [
+                'image/avif',
+                'image/bmp',
+                'image/gif',
+                'image/heic',
+                'image/heif',
+                'image/jpeg',
+                'image/png',
+                'image/tiff',
+                'image/webp',
+            ],
+            'raw' => false,
+        ];
+        $processCount = (int) $input->getOption('process-count');
+        /*
+         * Switch to async processes to batch document downscaling.
+         */
+        if ($processCount > 1) {
+            $documentsCount = $this->getDocumentRepository()->countBy($criteria);
+            $io->info(sprintf('Using %d processes to downscale %d documents.', $processCount, $documentsCount));
+
+            // Spawn processes for current command with limit and offset parameters
+            $documentsPerProcess = (int) ceil($documentsCount / $processCount);
+            /** @var array<PhpSubprocess> $processes */
+            $processes = [];
+
+            for ($i = 0; $i < $processCount; $i++) {
+                $offset = $i * $documentsPerProcess;
+                $limit = $documentsPerProcess;
+
+                $command = [
+                    'bin/console',
+                    'documents:downscale',
+                    '-n',
+                    '--process-count=1',
+                    '--limit=' . $limit,
+                    '--offset=' . $offset,
+                ];
+
+                $process = new PhpSubprocess($command);
+                $process->setTimeout(3600);
+                $process->start();
+                $processes[] = $process;
+
+                $io->text(sprintf('Started documents:downscale process %d with offset %d and limit %d', $i + 1, $offset, $limit));
+            }
+            // Wait for all processes to finish
+            foreach ($processes as $process) {
+                $process->wait();
+            }
+
+            $io->success('All processes have finished.');
+            return 0;
+        }
+
+        /** @var DocumentInterface[] $documents */
+        $documents = $this->getDocumentRepository()->findBy(
+            $criteria,
+            [],
+            is_numeric($input->getOption('limit')) ? (int) $input->getOption('limit') : null,
+            is_numeric($input->getOption('offset')) ? (int) $input->getOption('offset') : null
+        );
+        $io->progressStart(count($documents));
+
+        foreach ($documents as $document) {
+            try {
+                $this->downscaler->processDocumentFromExistingRaw($document);
+            } catch (NotReadableException $exception) {
+                $io->error($exception->getMessage() . ' - ' . (string) $document);
+            }
+            $io->progressAdvance();
+        }
+
+        $io->progressFinish();
+        $io->success('Every documents have been downscaled, a raw version has been kept.');
+
+        $this->dispatcher->dispatch(new CachePurgeAssetsRequestEvent());
+        return 0;
     }
 }
diff --git a/lib/Documents/src/Console/DocumentDuplicatesCommand.php b/lib/Documents/src/Console/DocumentDuplicatesCommand.php
@@ -29,7 +29,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $count = \count($documents);
         $rows = [];
 
-        if ($count <= 0) {
+        if ($count === 0) {
             $this->io->success('No duplicated documents were found.');
             return 0;
         }

diff --git a/lib/Documents/src/Console/DocumentFileHashCommand.php b/lib/Documents/src/Console/DocumentFileHashCommand.php
@@ -59,22 +59,25 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         /** @var DocumentInterface $document */
         foreach ($documents as $document) {
             $mountPath = $document->getMountPath();
-            if (null !== $mountPath && $document instanceof FileHashInterface) {
-                $algorithm = $document->getFileHashAlgorithm() ?? $defaultAlgorithm;
-                # https://flysystem.thephpleague.com/docs/usage/checksums/
-                $this->documentsStorage->checksum($mountPath, ['checksum_algo' => $algorithm]);
-                if ($this->documentsStorage->fileExists($mountPath)) {
-                    $fileHash = $this->documentsStorage->checksum($mountPath, ['checksum_algo' => $algorithm]);
-                    $document->setFileHash($fileHash);
-                    $document->setFileHashAlgorithm($algorithm);
-                }
-
-                if (($i % $batchSize) === 0) {
-                    $em->flush(); // Executes all updates.
-                }
-                ++$i;
+            if (null === $mountPath || !($document instanceof FileHashInterface)) {
                 $this->io->progressAdvance();
+                continue;
+            }
+
+            $algorithm = $document->getFileHashAlgorithm() ?? $defaultAlgorithm;
+            # https://flysystem.thephpleague.com/docs/usage/checksums/
+            $this->documentsStorage->checksum($mountPath, ['checksum_algo' => $algorithm]);
+            if ($this->documentsStorage->fileExists($mountPath)) {
+                $fileHash = $this->documentsStorage->checksum($mountPath, ['checksum_algo' => $algorithm]);
+                $document->setFileHash($fileHash);
+                $document->setFileHashAlgorithm($algorithm);
+            }
+
+            if (($i % $batchSize) === 0) {
+                $em->flush(); // Executes all updates.
             }
+            ++$i;
+            $this->io->progressAdvance();
         }
         $em->flush();
         $this->io->progressFinish();