feat(common/fileReader): hash file support and implementation in cli …

…import (ems-project#601)

Co-authored-by: David mattei <[email protected]>

EMS/common-bundle/src/Common/File/FileReader.php

@@ -6,15 +6,21 @@
 
 use EMS\CommonBundle\Contracts\File\FileReaderInterface;
 use PhpOffice\PhpSpreadsheet\IOFactory;
+use PhpOffice\PhpSpreadsheet\Reader\Csv;
+use PhpOffice\PhpSpreadsheet\Reader\Html;
+use PhpOffice\PhpSpreadsheet\Reader\Slk;
 
 final class FileReader implements FileReaderInterface
 {
     /**
      * {@inheritDoc}
      */
-    public function getData(string $filename, bool $skipFirstRow = false): array
+    public function getData(string $filename, bool $skipFirstRow = false, string $encoding = null): array
     {
         $reader = IOFactory::createReaderForFile($filename);
+        if (($reader instanceof Csv || $reader instanceof Html || $reader instanceof Slk) && null !== $encoding) {
+            $reader->setInputEncoding($encoding);
+        }
 
         $data = $reader->load($filename)->getActiveSheet()->toArray();
 

EMS/common-bundle/src/Contracts/File/FileReaderInterface.php

@@ -9,5 +9,5 @@ interface FileReaderInterface
     /**
      * @return array<mixed>
      */
-    public function getData(string $filename, bool $skipFirstRow = false): array;
+    public function getData(string $filename, bool $skipFirstRow = false, string $encoding = null): array;
 }

elasticms-cli/src/Command/FileReader/FileReaderImportCommand.php

@@ -25,12 +25,16 @@ final class FileReaderImportCommand extends AbstractCommand
     private const OPTION_DRY_RUN = 'dry-run';
     private const OPTION_GENERATE_HASH = 'generate-hash';
     private const OPTION_DELETE_MISSING_DOCUMENTS = 'delete-missing-document';
+    private const OPTION_HASH_FILE = 'hash-file';
+    private const OPTION_ENCODING = 'encoding';
     private string $ouuidExpression;
     private string $contentType;
     private string $file;
     private bool $dryRun;
     private bool $hashOuuid;
     private bool $deleteMissingDocuments;
+    private bool $hashFile;
+    private ?string $encoding;
 
     public function __construct(private readonly AdminHelper $adminHelper, private readonly FileReaderInterface $fileReader)
     {
@@ -47,6 +51,8 @@ protected function configure(): void
             ->addOption(self::OPTION_GENERATE_HASH, null, InputOption::VALUE_NONE, 'Use the OUUID column and the content type name in order to generate a "better" ouuid')
             ->addOption(self::OPTION_DELETE_MISSING_DOCUMENTS, null, InputOption::VALUE_NONE, 'The command will delete content type document that are missing in the import file')
             ->addOption(self::OPTION_OUUID_EXPRESSION, null, InputOption::VALUE_OPTIONAL, 'Expression language apply to excel rows in order to identify the document by its ouuid. If equal to null new document will be created', "row['ouuid']")
+            ->addOption(self::OPTION_HASH_FILE, null, InputOption::VALUE_NONE, 'Specify that the file argument is a file hash not a file path.')
+            ->addOption(self::OPTION_ENCODING, null, InputOption::VALUE_OPTIONAL, 'Specify the file\'s encoding for csv, html and Slk file')
         ;
     }
 
@@ -59,6 +65,8 @@ protected function initialize(InputInterface $input, OutputInterface $output): v
         $this->dryRun = $this->getOptionBool(self::OPTION_DRY_RUN);
         $this->hashOuuid = $this->getOptionBool(self::OPTION_GENERATE_HASH);
         $this->deleteMissingDocuments = $this->getOptionBool(self::OPTION_DELETE_MISSING_DOCUMENTS);
+        $this->hashFile = $this->getOptionBool(self::OPTION_HASH_FILE);
+        $this->encoding = $this->getOptionStringNull(self::OPTION_ENCODING);
     }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
@@ -72,9 +80,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
             return self::EXECUTE_ERROR;
         }
+        $file = $this->hashFile ? $this->getFileByHash($this->file) : $this->file;
 
         $expressionLanguage = new ExpressionLanguage();
-        $rows = $this->fileReader->getData($this->file);
+        $rows = $this->fileReader->getData($file, false, $this->encoding);
         $header = $rows[0] ?? [];
 
         $ouuids = [];
@@ -89,14 +98,21 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             }
         }
 
+        $counter = 0;
         $progressBar = $this->io->createProgressBar(\count($rows) - 1);
         foreach ($rows as $key => $value) {
             if (0 === $key) {
                 continue;
             }
             $row = [];
+            $empty = true;
             foreach ($value as $key => $cell) {
                 $row[$header[$key] ?? $key] = $cell;
+                $empty = $empty && (null === $cell);
+            }
+            if ($empty) {
+                $progressBar->advance();
+                continue;
             }
 
             $ouuid = 'null' === $this->ouuidExpression ? null : $expressionLanguage->evaluate($this->ouuidExpression, [
@@ -127,8 +143,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             }
             $contentTypeApi->finalize($draft->getRevisionId());
             $progressBar->advance();
+            ++$counter;
         }
         $progressBar->finish();
+        $this->io->newLine(2);
+        $this->io->text(\sprintf('%d lines have been imported', $counter));
 
         if ($this->dryRun && \count($ouuids) > 0) {
             $this->io->newLine(2);
@@ -146,4 +165,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         return self::EXECUTE_SUCCESS;
     }
+
+    private function getFileByHash(string $hash): string
+    {
+        if (!$this->adminHelper->getCoreApi()->file()->headHash($hash)) {
+            throw new \RuntimeException(\sprintf('File with hash "%s" not found', $hash));
+        }
+
+        return $this->adminHelper->getCoreApi()->file()->downloadFile($hash);
+    }
 }