diff --git a/src/Backend/Teradata/Helper/StorageABSHelper.php b/src/Backend/Teradata/Helper/StorageABSHelper.php index 787082a1..6416bfe1 100644 --- a/src/Backend/Teradata/Helper/StorageABSHelper.php +++ b/src/Backend/Teradata/Helper/StorageABSHelper.php @@ -5,6 +5,7 @@ namespace Keboola\Db\ImportExport\Backend\Teradata\Helper; use Keboola\Db\ImportExport\Backend\Helper\BackendHelper as BaseHelper; +use Keboola\Db\ImportExport\Storage\ABS\BaseFile; use Keboola\Db\ImportExport\Storage\ABS\SourceFile; use LogicException; @@ -19,14 +20,14 @@ final class StorageABSHelper extends BaseHelper */ public static function getMask(SourceFile $source): string { - throw new LogicException('TODO Not implemented yet'); - /*$entries = $source->getManifestEntries(); + $entries = $source->getManifestEntries(); if (count($entries) === 0) { // no entries -> no data to load return ''; } - // SourceDirectory returns fileName as directory/file.csv but SourceFile returns s3://bucket/directory/file.csv - $toRemove = $source->getS3Prefix() . '/'; + // SourceDirectory returns fileName as directory/file.csv + // but SourceFile returns azure://myaccount...windows.net/bucket/directory/file.csv + $toRemove = $source->getContainerUrl(BaseFile::PROTOCOL_AZURE); $entriesAsArrays = []; $min = 99999; $minIndex = 0; @@ -53,7 +54,7 @@ public static function getMask(SourceFile $source): string } $out[$index] = $match ? $letter : '*'; } - return implode('', $out);*/ + return implode('', $out); } public static function isMultipartFile(SourceFile $source): bool @@ -69,18 +70,19 @@ public static function isMultipartFile(SourceFile $source): bool } /** - * extracts filename and prefix from s3 url - removing bucket, protocol and Fxxx suffix + * extracts filename and prefix from ABS url - removing bucket, protocol and Fxxx suffix * @return string[] */ public static function buildPrefixAndObject(SourceFile $source): array { - throw new LogicException('TODO Not implemented yet'); - /*// docs say 6, but my files are created with 5 + // docs say 6, but my files are created with 5 $entries = $source->getManifestEntries(); preg_match('/(?.*)\/F(?[0-9]{5,6})/', $entries[0], $out); $filePath = $out['filePath'] ?? ''; - $filePath = str_replace(($source->getS3Prefix() . '/'), '', $filePath); + // SourceDirectory returns fileName as directory/file.csv + // but SourceFile returns azure://myaccount...windows.net/bucket/directory/file.csv + $filePath = str_replace(($source->getContainerUrl(BaseFile::PROTOCOL_AZURE)), '', $filePath); $exploded = explode('/', $filePath); $object = end($exploded); @@ -88,6 +90,6 @@ public static function buildPrefixAndObject(SourceFile $source): array $prefix = implode('/', array_slice($exploded, 0, -1)); // prefix should end with / but only if it exists $prefix = $prefix ? ($prefix . '/') : ''; - return [$prefix, $object];*/ + return [$prefix, $object]; } } diff --git a/src/Backend/Teradata/ToStage/FromABSTPTAdapter.php b/src/Backend/Teradata/ToStage/FromABSTPTAdapter.php index 4a60c0be..294dc978 100644 --- a/src/Backend/Teradata/ToStage/FromABSTPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromABSTPTAdapter.php @@ -15,6 +15,7 @@ use Keboola\Db\ImportExport\ImportOptions; use Keboola\Db\ImportExport\ImportOptionsInterface; use Keboola\Db\ImportExport\Storage; +use Keboola\FileStorage\Abs\AbsProvider; use Keboola\FileStorage\Path\RelativePath; use Keboola\FileStorage\S3\S3Provider; use Keboola\TableBackendUtils\Escaping\Teradata\TeradataQuote; @@ -205,35 +206,31 @@ private function generateTPTScript( // file is abs://container/path/.../file.csv.gz/F00000 // extract real filepath from path -> remove F00000 and do just one load // load with SPF = false - throw new LogicException('Unsupported import type - multipart'); - /*[$prefix, $object] = StorageS3Helper::buildPrefixAndObject($source); + [$prefix, $object] = StorageABSHelper::buildPrefixAndObject($source); $moduleStr = sprintf( // phpcs:ignore - 'AccessModuleInitStr = \'S3Region="%s" S3Bucket="%s" S3Prefix="%s" S3Object="%s" S3SinglePartFile=False S3ConfigDir=%s\'', - $source->getRegion(), - $source->getBucket(), + 'AccessModuleInitStr = \'-ConfigDir "%s" -Container "%s" -Prefix "%s" -Object "%s" -SinglePartFile False\'', + $absConfigDir, + $source->getContainer(), $prefix, $object, - $absConfigDir - );*/ + ); } else { if ($source->isSliced()) { // load with wildcard // scenario b - throw new LogicException('Unsupported import type - sliced'); - /*$mask = StorageABSHelper::getMask($source); - $path = RelativePath::createFromRootAndPath(new S3Provider(), $source->getBucket(), $mask); + $mask = StorageABSHelper::getMask($source); + $path = RelativePath::createFromRootAndPath(new AbsProvider(), $source->getContainer(), $mask); $moduleStr = sprintf( // phpcs:ignore - 'AccessModuleInitStr = \'S3Region="%s" S3Bucket="%s" S3Prefix="%s" S3Object="%s" S3SinglePartFile=True S3ConfigDir=%s\'', - $source->getRegion(), + 'AccessModuleInitStr = \'-ConfigDir "%s" -Container "%s" -Prefix "%s" -Object "%s" -SinglePartFile True\'', + $absConfigDir, $path->getRoot(), $path->getPathWithoutRoot() . '/', $path->getFileName(), - $absConfigDir - );*/ + ); } else { // direct load with // scenario a @@ -243,9 +240,8 @@ private function generateTPTScript( 'AccessModuleInitStr = \'-ConfigDir "%s" -Container "%s" -Prefix "%s" -Object "%s" -SinglePartFile True\'', $absConfigDir, $source->getContainer(), - // TODO use parser - dirname($source->getFilePath()) === '.' ? '' : dirname($source->getFilePath()), - basename($source->getFilePath()), + $source->getPrefix(), + $source->getFileName(), ); } } diff --git a/src/Storage/ABS/SourceFile.php b/src/Storage/ABS/SourceFile.php index 3e1b238a..e3b6dd74 100644 --- a/src/Storage/ABS/SourceFile.php +++ b/src/Storage/ABS/SourceFile.php @@ -4,6 +4,7 @@ namespace Keboola\Db\ImportExport\Storage\ABS; +use Exception as InternalException; use Keboola\CsvOptions\CsvOptions; use Keboola\Db\Import\Exception; use Keboola\Db\ImportExport\Storage\FileNotFoundException; @@ -204,4 +205,42 @@ public function isSliced(): bool { return $this->isSliced; } + + /** + * from path data/shared/file.csv to file.csv + * + * @throws InternalException + */ + public function getFileName(): string + { + if ($this->isSliced) { + throw new InternalException('Not supported getFileName for sliced files.'); + } + $fileName = $this->filePath; + if (strrpos($fileName, '/') !== false) { + // there is dir in the path + return substr($fileName, strrpos($fileName, '/') + 1); + } + // there is no dir in the path, just the filename + return $fileName; + } + + /** + * from path data/shared/file.csv to data/shared/ + * + * @throws InternalException + */ + public function getPrefix(): string + { + $filePath = $this->getFilePath(); + // SourceDirectory returns fileName as directory/file.csv + // but SourceFile returns azure://myaccount...windows.net/bucket/directory/file.csv + $filePath = str_replace(($this->getContainerUrl(BaseFile::PROTOCOL_AZURE)), '', $filePath); + + $exploded = explode('/', $filePath); + // get all the parts of exploded path but without the last thing - the filename + $prefix = implode('/', array_slice($exploded, 0, -1)); + // prefix should end with / but only if it exists + return $prefix ? ($prefix . '/') : ''; + } } diff --git a/src/Storage/ABS/TeradataExportTPTAdapter.php b/src/Storage/ABS/TeradataExportTPTAdapter.php index 56867eb7..366fe156 100644 --- a/src/Storage/ABS/TeradataExportTPTAdapter.php +++ b/src/Storage/ABS/TeradataExportTPTAdapter.php @@ -114,6 +114,8 @@ private function generateTPTExportScript( ); file_put_contents($absConfigDir . '/credentials', $credentials); + $path = $destination->getRelativePath(); + $tptScript = sprintf( /** @lang SQL */<<getContainer(), - // TODO use parser - dirname($destination->getFilePath()), - basename($destination->getFilePath()) . ($exportOptions->isCompressed() ? '.gz' : ''), + $path->getPathWithoutRoot(), + $path->getFileName() . ($exportOptions->isCompressed() ? '.gz' : ''), $exportOptions->generateABSSizeOptions(), $source->getFromStatement() ); diff --git a/tests/functional/Teradata/ExportTest.php b/tests/functional/Teradata/ExportTest.php index f6ae2a2f..deeea31b 100644 --- a/tests/functional/Teradata/ExportTest.php +++ b/tests/functional/Teradata/ExportTest.php @@ -430,9 +430,9 @@ public function exportOptionsProvider(): array public function pipelineOptions(): array { if (getenv('STORAGE_TYPE') === 'S3') { - $generatedSliceName = '.gz/F000000'; + $generatedSliceName = '/F000000'; } elseif (getenv('STORAGE_TYPE') === 'ABS') { - $generatedSliceName = '.gz/F00000'; + $generatedSliceName = '/F00000'; } else { $this->fail('Unsupported file storage in this test!'); } @@ -441,7 +441,7 @@ public function pipelineOptions(): array 'compressed singleFile=false' => [ true, // gz false, // use SinglePartFile - $generatedSliceName, // generated file name based on ^^ + '.gz' . $generatedSliceName, // generated file name based on ^^ ], 'compressed singleFile=true' => [ true,