From 97a1e5ff0d42e0da2ab19cd4d83ffa7c4e5ff43f Mon Sep 17 00:00:00 2001 From: Mathieu De Keyzer Date: Mon, 11 Sep 2023 09:19:13 +0200 Subject: [PATCH] feat(common/fileReader): hash file support and implementation in cli import (#601) Co-authored-by: David mattei --- .../src/Common/File/FileReader.php | 8 ++++- .../Contracts/File/FileReaderInterface.php | 2 +- .../FileReader/FileReaderImportCommand.php | 30 ++++++++++++++++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/EMS/common-bundle/src/Common/File/FileReader.php b/EMS/common-bundle/src/Common/File/FileReader.php index e2aa902f1..a5bd958dc 100644 --- a/EMS/common-bundle/src/Common/File/FileReader.php +++ b/EMS/common-bundle/src/Common/File/FileReader.php @@ -6,15 +6,21 @@ use EMS\CommonBundle\Contracts\File\FileReaderInterface; use PhpOffice\PhpSpreadsheet\IOFactory; +use PhpOffice\PhpSpreadsheet\Reader\Csv; +use PhpOffice\PhpSpreadsheet\Reader\Html; +use PhpOffice\PhpSpreadsheet\Reader\Slk; final class FileReader implements FileReaderInterface { /** * {@inheritDoc} */ - public function getData(string $filename, bool $skipFirstRow = false): array + public function getData(string $filename, bool $skipFirstRow = false, string $encoding = null): array { $reader = IOFactory::createReaderForFile($filename); + if (($reader instanceof Csv || $reader instanceof Html || $reader instanceof Slk) && null !== $encoding) { + $reader->setInputEncoding($encoding); + } $data = $reader->load($filename)->getActiveSheet()->toArray(); diff --git a/EMS/common-bundle/src/Contracts/File/FileReaderInterface.php b/EMS/common-bundle/src/Contracts/File/FileReaderInterface.php index fed2c0c47..8ee982038 100644 --- a/EMS/common-bundle/src/Contracts/File/FileReaderInterface.php +++ b/EMS/common-bundle/src/Contracts/File/FileReaderInterface.php @@ -9,5 +9,5 @@ interface FileReaderInterface /** * @return array */ - public function getData(string $filename, bool $skipFirstRow = false): array; + public function getData(string $filename, bool $skipFirstRow = false, string $encoding = null): array; } diff --git a/elasticms-cli/src/Command/FileReader/FileReaderImportCommand.php b/elasticms-cli/src/Command/FileReader/FileReaderImportCommand.php index 602ccc0a0..80e5046f5 100644 --- a/elasticms-cli/src/Command/FileReader/FileReaderImportCommand.php +++ b/elasticms-cli/src/Command/FileReader/FileReaderImportCommand.php @@ -25,12 +25,16 @@ final class FileReaderImportCommand extends AbstractCommand private const OPTION_DRY_RUN = 'dry-run'; private const OPTION_GENERATE_HASH = 'generate-hash'; private const OPTION_DELETE_MISSING_DOCUMENTS = 'delete-missing-document'; + private const OPTION_HASH_FILE = 'hash-file'; + private const OPTION_ENCODING = 'encoding'; private string $ouuidExpression; private string $contentType; private string $file; private bool $dryRun; private bool $hashOuuid; private bool $deleteMissingDocuments; + private bool $hashFile; + private ?string $encoding; public function __construct(private readonly AdminHelper $adminHelper, private readonly FileReaderInterface $fileReader) { @@ -47,6 +51,8 @@ protected function configure(): void ->addOption(self::OPTION_GENERATE_HASH, null, InputOption::VALUE_NONE, 'Use the OUUID column and the content type name in order to generate a "better" ouuid') ->addOption(self::OPTION_DELETE_MISSING_DOCUMENTS, null, InputOption::VALUE_NONE, 'The command will delete content type document that are missing in the import file') ->addOption(self::OPTION_OUUID_EXPRESSION, null, InputOption::VALUE_OPTIONAL, 'Expression language apply to excel rows in order to identify the document by its ouuid. If equal to null new document will be created', "row['ouuid']") + ->addOption(self::OPTION_HASH_FILE, null, InputOption::VALUE_NONE, 'Specify that the file argument is a file hash not a file path.') + ->addOption(self::OPTION_ENCODING, null, InputOption::VALUE_OPTIONAL, 'Specify the file\'s encoding for csv, html and Slk file') ; } @@ -59,6 +65,8 @@ protected function initialize(InputInterface $input, OutputInterface $output): v $this->dryRun = $this->getOptionBool(self::OPTION_DRY_RUN); $this->hashOuuid = $this->getOptionBool(self::OPTION_GENERATE_HASH); $this->deleteMissingDocuments = $this->getOptionBool(self::OPTION_DELETE_MISSING_DOCUMENTS); + $this->hashFile = $this->getOptionBool(self::OPTION_HASH_FILE); + $this->encoding = $this->getOptionStringNull(self::OPTION_ENCODING); } protected function execute(InputInterface $input, OutputInterface $output): int @@ -72,9 +80,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int return self::EXECUTE_ERROR; } + $file = $this->hashFile ? $this->getFileByHash($this->file) : $this->file; $expressionLanguage = new ExpressionLanguage(); - $rows = $this->fileReader->getData($this->file); + $rows = $this->fileReader->getData($file, false, $this->encoding); $header = $rows[0] ?? []; $ouuids = []; @@ -89,14 +98,21 @@ protected function execute(InputInterface $input, OutputInterface $output): int } } + $counter = 0; $progressBar = $this->io->createProgressBar(\count($rows) - 1); foreach ($rows as $key => $value) { if (0 === $key) { continue; } $row = []; + $empty = true; foreach ($value as $key => $cell) { $row[$header[$key] ?? $key] = $cell; + $empty = $empty && (null === $cell); + } + if ($empty) { + $progressBar->advance(); + continue; } $ouuid = 'null' === $this->ouuidExpression ? null : $expressionLanguage->evaluate($this->ouuidExpression, [ @@ -127,8 +143,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int } $contentTypeApi->finalize($draft->getRevisionId()); $progressBar->advance(); + ++$counter; } $progressBar->finish(); + $this->io->newLine(2); + $this->io->text(\sprintf('%d lines have been imported', $counter)); if ($this->dryRun && \count($ouuids) > 0) { $this->io->newLine(2); @@ -146,4 +165,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int return self::EXECUTE_SUCCESS; } + + private function getFileByHash(string $hash): string + { + if (!$this->adminHelper->getCoreApi()->file()->headHash($hash)) { + throw new \RuntimeException(\sprintf('File with hash "%s" not found', $hash)); + } + + return $this->adminHelper->getCoreApi()->file()->downloadFile($hash); + } }