From c4b7e84647d7573d829201cdd5f46f2b12a0ea9a Mon Sep 17 00:00:00 2001 From: zajca Date: Thu, 17 Feb 2022 13:51:08 +0100 Subject: [PATCH 01/62] KBC-2391 install teradata odbc driver and tools --- Dockerfile | 31 +++++++++++++++++++++++++++++++ docker-compose.yml | 6 +++++- docker/teradata/odbc.ini | 14 ++++++++++++++ docker/teradata/odbcinst.ini | 9 +++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 docker/teradata/odbc.ini create mode 100644 docker/teradata/odbcinst.ini diff --git a/Dockerfile b/Dockerfile index d4d85c11..f60f58e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,9 @@ +FROM quay.io/keboola/aws-cli +ARG AWS_SECRET_ACCESS_KEY +ARG AWS_ACCESS_KEY_ID +RUN /usr/bin/aws s3 cp s3://keboola-drivers/teradata/tdodbc1710-17.10.00.08-1.x86_64.deb /tmp/teradata/tdodbc.deb +RUN /usr/bin/aws s3 cp s3://keboola-drivers/teradata/utils/TeradataToolsAndUtilitiesBase__ubuntu_x8664.17.00.34.00.tar.gz /tmp/teradata/tdutils.tar.gz + FROM php:7.4-cli ARG COMPOSER_FLAGS="--prefer-dist --no-interaction" @@ -89,6 +95,31 @@ RUN set -ex; \ echo "\n[exasol]\nDriver=/opt/exasol/libexaodbc-uo2214lv2.so\n" >> /etc/odbcinst.ini;\ rm -rf /tmp/exasol; +# Teradata ODBC +COPY --from=0 /tmp/teradata/tdodbc.deb /tmp/teradata/tdodbc.deb +COPY docker/teradata/odbc.ini /tmp/teradata/odbc_td.ini +COPY docker/teradata/odbcinst.ini /tmp/teradata/odbcinst_td.ini + +RUN dpkg -i /tmp/teradata/tdodbc.deb \ + && cat /tmp/teradata/odbc_td.ini >> /etc/odbc.ini \ + && cat /tmp/teradata/odbcinst_td.ini >> /etc/odbcinst.ini \ + && rm -r /tmp/teradata \ + && docker-php-ext-configure pdo_odbc --with-pdo-odbc=unixODBC,/usr \ + && docker-php-ext-install pdo_odbc + +ENV ODBCHOME = /opt/teradata/client/ODBC_64/ +ENV ODBCINI = /opt/teradata/client/ODBC_64/odbc.ini +ENV ODBCINST = /opt/teradata/client/ODBC_64/odbcinst.ini +ENV LD_LIBRARY_PATH = /opt/teradata/client/ODBC_64/lib + +# Teradata Utils +COPY --from=0 /tmp/teradata/tdutils.tar.gz /tmp/teradata/tdutils.tar.gz +RUN cd /tmp/teradata \ + && tar -xvaf tdutils.tar.gz \ + && sh /tmp/teradata/TeradataToolsAndUtilitiesBase/.setup.sh tptbase \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /tmp/teradata + #php odbc RUN docker-php-ext-configure pdo_odbc --with-pdo-odbc=unixODBC,/usr \ && docker-php-ext-install pdo_odbc diff --git a/docker-compose.yml b/docker-compose.yml index 153c5249..732d2b29 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,11 @@ version: '3' services: production: &prod - build: . + build: + context: . + args: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY environment: - ABS_ACCOUNT_NAME - ABS_ACCOUNT_KEY diff --git a/docker/teradata/odbc.ini b/docker/teradata/odbc.ini new file mode 100644 index 00000000..151d655f --- /dev/null +++ b/docker/teradata/odbc.ini @@ -0,0 +1,14 @@ +[ODBC] +InstallDir=/opt/teradata/client/ODBC_64 +Trace=0 +TraceDll=/opt/teradata/client/ODBC_64/lib/odbctrac.so +TraceFile=/usr/odbcusr/trace.log +TraceAutoStop=0 + +[Teradata] +Driver = /opt/teradata/client/ODBC_64/lib/tdataodbc_sb64.so +UsageCount = 2 +APILevel = CORE +ConnectFunctions = YYY +DriverODBCVer = 3.51 +SQLLevel = 1 diff --git a/docker/teradata/odbcinst.ini b/docker/teradata/odbcinst.ini new file mode 100644 index 00000000..af1a3dc2 --- /dev/null +++ b/docker/teradata/odbcinst.ini @@ -0,0 +1,9 @@ +[ODBC DRIVERS] +Teradata=Installed + +[Teradata] +Driver=/opt/teradata/client/ODBC_64/lib/tdataodbc_sb64.so +APILevel=CORE +ConnectFunctions=YYY +DriverODBCVer=3.51 +SQLLevel=1 From 4aec5292c1237ac6b1aa16ca53cb65ef5711af91 Mon Sep 17 00:00:00 2001 From: zajca Date: Fri, 18 Feb 2022 13:17:19 +0100 Subject: [PATCH 02/62] KBC-2391 add dependencies, init TD suites --- Dockerfile | 2 +- composer.json | 12 +- docker-compose.yml | 4 + phpunit.xml.dist | 4 + .../Teradata/TeradataBaseTestCase.php | 176 ++++++++++++++++++ 5 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 tests/functional/Teradata/TeradataBaseTestCase.php diff --git a/Dockerfile b/Dockerfile index f60f58e0..83090e76 100644 --- a/Dockerfile +++ b/Dockerfile @@ -116,7 +116,7 @@ ENV LD_LIBRARY_PATH = /opt/teradata/client/ODBC_64/lib COPY --from=0 /tmp/teradata/tdutils.tar.gz /tmp/teradata/tdutils.tar.gz RUN cd /tmp/teradata \ && tar -xvaf tdutils.tar.gz \ - && sh /tmp/teradata/TeradataToolsAndUtilitiesBase/.setup.sh tptbase \ + && sh /tmp/teradata/TeradataToolsAndUtilitiesBase/.setup.sh tptbase s3axsmod \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /tmp/teradata diff --git a/composer.json b/composer.json index 87937230..a58fd70a 100644 --- a/composer.json +++ b/composer.json @@ -10,7 +10,9 @@ "keboola/php-csv-db-import": "^5.0", "keboola/php-file-storage-utils": ">=0.2", "keboola/table-backend-utils": "^0.18", - "microsoft/azure-storage-blob": "^1.4" + "microsoft/azure-storage-blob": "^1.4", + "symfony/process": "^4.4|^5.0", + "keboola/php-temp": "^1.0" }, "require-dev": { "phpstan/phpstan": "^0.12.54", @@ -18,9 +20,7 @@ "keboola/coding-standard": "^9.0", "php-parallel-lint/php-parallel-lint": "^1.2", "phpstan/extension-installer": "^1.0", - "keboola/datadir-tests": "^2.0", - "keboola/php-temp": "^1.0", - "symfony/process": "^4.4|^5.0" + "keboola/datadir-tests": "^2.0" }, "autoload": { "psr-4": { @@ -49,6 +49,7 @@ "tests-synapse-clusterdindextemp": "SUITE=tests-synapse-clusterdindextemp CREDENTIALS_IMPORT_TYPE=SAS CREDENTIALS_EXPORT_TYPE=MASTER_KEY TEMP_TABLE_TYPE=CLUSTERED_INDEX DEDUP_TYPE=TMP_TABLE phpunit --colors=always --testsuite tests-synapse-clusterdindextemp", "tests-synapse-mi": "SUITE=tests-synapse-mi CREDENTIALS_IMPORT_TYPE=MANAGED_IDENTITY CREDENTIALS_EXPORT_TYPE=MANAGED_IDENTITY TEMP_TABLE_TYPE=HEAP DEDUP_TYPE=TMP_TABLE phpunit --colors=always --testsuite synapse-mi", "tests-exasol": "SUITE=tests-exasol STORAGE_TYPE=S3 phpunit --colors=always --testsuite exasol", + "tests-teradata": "SUITE=tests-teradata STORAGE_TYPE=S3 phpunit --colors=always --testsuite teradata", "tests-functional": [ "@tests-snowflake-abs", "@tests-snowflake-s3", @@ -58,7 +59,8 @@ "@tests-synapse-clusterdindextemp", "@tests-synapse-heap4000temp", "@tests-synapse-heap4000temp-optimized", - "@tests-exasol" + "@tests-exasol", + "@tests-teradata" ], "tests": [ "@tests-unit", diff --git a/docker-compose.yml b/docker-compose.yml index 732d2b29..1870e4b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,10 @@ services: - EXASOL_USERNAME - EXASOL_PASSWORD - BUILD_PREFIX + - TERADATA_HOST + - TERADATA_USERNAME + - TERADATA_PASSWORD + - TERADATA_PORT dev: &dev <<: *prod image: keboola/php-db-import-export diff --git a/phpunit.xml.dist b/phpunit.xml.dist index d737710a..4a0eb67b 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -61,6 +61,10 @@ tests/functional/Synapse/SqlCommandBuilderTest.php + + tests/functional/Teradata + + tests/unit diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php new file mode 100644 index 00000000..8afc66e2 --- /dev/null +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -0,0 +1,176 @@ +connection = $this->getTeradataConnection(); + } + + private function getTeradataConnection(): Connection + { + return TeradataConnection::getConnection([ + 'host' => (string) getenv('TERADATA_HOST'), + 'user' => (string) getenv('TERADATA_USERNAME'), + 'password' => (string) getenv('TERADATA_PASSWORD'), + 'port' => (int) getenv('TERADATA_PORT'), + 'dbname' => '', + ]); + } + + /** + * @param string[] $columnsNames + */ + public function getColumnsWithoutTypes(array $columnsNames): ColumnCollection + { + $columns = array_map(function ($colName) { + return new TeradataColumn( + $colName, + new Teradata( + Teradata::TYPE_VARCHAR, + ['length' => 4000] + ) + ); + }, $columnsNames); + return new ColumnCollection($columns); + } + + /** + * @param string[] $columns + * @param string[] $pks + */ + public function getGenericTableDefinition( + string $schemaName, + string $tableName, + array $columns, + array $pks = [] + ): TeradataTableDefinition { + return new TeradataTableDefinition ( + $schemaName, + $tableName, + false, + $this->getColumnsWithoutTypes($columns), + $pks + ); + } + + protected function cleanDatabase(string $dbname): void + { + if (!$this->dbExists($dbname)) { + return; + } + + // delete all objects in the DB + $this->connection->executeQuery(sprintf('DELETE DATABASE %s ALL', $dbname)); + // drop the empty db + $this->connection->executeQuery(sprintf('DROP DATABASE %s', $dbname)); + } + + public function createDatabase(string $dbName): void + { + $this->connection->executeQuery(sprintf(' +CREATE DATABASE %s AS + PERM = 1e9; + ', $dbName)); + } + + protected function dbExists(string $dbname): bool + { + try { + $this->connection->executeQuery(sprintf('HELP DATABASE %s', $dbname)); + return true; + } catch (\Doctrine\DBAL\Exception $e) { + return false; + } + } + + /** + * @param int|string $sortKey + * @param array $expected + * @param string|int $sortKey + */ + protected function assertSynapseTableEqualsExpected( + SourceInterface $source, + TeradataTableDefinition $destination, + ImportOptions $options, + array $expected, + $sortKey, + string $message = 'Imported tables are not the same as expected' + ): void { + $tableColumns = (new TeradataTableReflection( + $this->connection, + $destination->getSchemaName(), + $destination->getTableName() + ))->getColumnsNames(); + + if ($options->useTimestamp()) { + self::assertContains('_timestamp', $tableColumns); + } else { + self::assertNotContains('_timestamp', $tableColumns); + } + + if (!in_array('_timestamp', $source->getColumnsNames())) { + $tableColumns = array_filter($tableColumns, function ($column) { + return $column !== '_timestamp'; + }); + } + + $tableColumns = array_map(function ($column) { + return sprintf('[%s]', $column); + }, $tableColumns); + + $sql = sprintf( + 'SELECT %s FROM [%s].[%s]', + implode(', ', $tableColumns), + $destination->getSchemaName(), + $destination->getTableName() + ); + + $queryResult = array_map(function ($row) { + return array_map(function ($column) { + return $column; + }, array_values($row)); + }, $this->connection->fetchAll($sql)); + + $this->assertArrayEqualsSorted( + $expected, + $queryResult, + $sortKey, + $message + ); + } + + protected function getDestinationSchemaName(): string + { + return self::EXASOL_SOURCE_SCHEMA_NAME + . '-' + . getenv('SUITE'); + } +} From 4276550ec6712e806bd20226473230a871c2e17d Mon Sep 17 00:00:00 2001 From: zajca Date: Fri, 18 Feb 2022 13:17:45 +0100 Subject: [PATCH 03/62] KBC-2391 init TD implementation --- src/Backend/Teradata/Helper/BackendHelper.php | 18 ++ .../Teradata/TeradataImportOptions.php | 63 +++++ .../Exception/FailedTPTLoadException.php | 78 ++++++ .../Teradata/ToStage/FromS3TPTAdapter.php | 261 ++++++++++++++++++ .../ToStage/StageTableDefinitionFactory.php | 92 ++++++ .../Teradata/ToStage/ToStageImporter.php | 77 ++++++ src/Exception/Exception.php | 9 + src/Storage/S3/BaseFile.php | 5 + src/Storage/S3/SourceDirectory.php | 17 +- src/Storage/S3/SourceFile.php | 20 ++ tests/data/csv/simple/a_b_c-1row.csv | 2 + tests/functional/Teradata/StageImportTest.php | 61 ++++ 12 files changed, 698 insertions(+), 5 deletions(-) create mode 100644 src/Backend/Teradata/Helper/BackendHelper.php create mode 100644 src/Backend/Teradata/TeradataImportOptions.php create mode 100644 src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php create mode 100644 src/Backend/Teradata/ToStage/FromS3TPTAdapter.php create mode 100644 src/Backend/Teradata/ToStage/StageTableDefinitionFactory.php create mode 100644 src/Backend/Teradata/ToStage/ToStageImporter.php create mode 100644 src/Exception/Exception.php create mode 100644 tests/data/csv/simple/a_b_c-1row.csv create mode 100644 tests/functional/Teradata/StageImportTest.php diff --git a/src/Backend/Teradata/Helper/BackendHelper.php b/src/Backend/Teradata/Helper/BackendHelper.php new file mode 100644 index 00000000..22bdc23f --- /dev/null +++ b/src/Backend/Teradata/Helper/BackendHelper.php @@ -0,0 +1,18 @@ +teradataHost = $teradataHost; + $this->teradataUser = $teradataUser; + $this->teradataPassword = $teradataPassword; + $this->teradataPort = $teradataPort; + } + + public function getTeradataHost(): string + { + return $this->teradataHost; + } + + public function getTeradataUser(): string + { + return $this->teradataUser; + } + + public function getTeradataPassword(): string + { + return $this->teradataPassword; + } + + public function getTeradataPort(): int + { + return $this->teradataPort; + } +} diff --git a/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php b/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php new file mode 100644 index 00000000..cff2e7ab --- /dev/null +++ b/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php @@ -0,0 +1,78 @@ +stdErr = $stdErr; + $this->stdOut = $stdOut; + $this->logContent = $logContent; + $this->exitCode = $exitCode; + $this->logTableContent = $logTableContent; + $this->errTableContent = $errTableContent; + $this->errTable2Content = $errTable2Content; + } + + public function getStdErr(): string + { + return $this->stdErr; + } + + public function getStdOut(): string + { + return $this->stdOut; + } + + public function getLogContent(): ?string + { + return $this->logContent; + } + + public function getExitCode(): ?int + { + return $this->exitCode; + } + + public function getLogTableContent(): ?array + { + return $this->logTableContent; + } + + public function getErrTableContent(): ?array + { + return $this->errTableContent; + } + + public function getErrTable2Content(): ?array + { + return $this->errTable2Content; + } +} diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php new file mode 100644 index 00000000..c4ae4168 --- /dev/null +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -0,0 +1,261 @@ +connection = $connection; + } + + /** + * @param Storage\S3\SourceFile $source + * @param TeradataTableDefinition $destination + * @param ImportOptions $importOptions + */ + public function runCopyCommand( + Storage\SourceInterface $source, + TableDefinitionInterface $destination, + ImportOptionsInterface $importOptions + ): int { + assert($source instanceof Storage\S3\SourceFile); + assert($destination instanceof TeradataTableDefinition); + assert($importOptions instanceof TeradataImportOptions); + + /** + * @var Temp $temp + */ + [ + $temp, + $logTable, + $errTable, + $errTable2, + $processCmd, + ] = $this->generateTPTScript($source, $destination, $importOptions); + + $process = new Process( + $processCmd, + null, + [ + 'AWS_ACCESS_KEY_ID' => $source->getKey(), + 'AWS_SECRET_ACCESS_KEY' => $source->getSecret(), + ] + ); + $process->start(); + + //// debug stuff + //foreach ($process as $type => $data) { + // if ($process::OUT === $type) { + // echo "\nRead from stdout: " . $data; + // } else { // $process::ERR === $type + // echo "\nRead from stderr: " . $data; + // } + //} + $isTableExists = fn(string $tableName + ) => (bool) $this->connection->fetchOne(sprintf('SELECT 1 FROM dbc.TablesV WHERE TableName = %s', TeradataQuote::quote($tableName))); + + $logContent = null; + if ($isTableExists($logTable)) { + $logContent = $this->connection->fetchAllAssociative('SELECT * FROM ' . TeradataQuote::quoteSingleIdentifier($logTable)); + $this->connection->executeStatement('DROP TABLE ' . $logTable); + } + $errContent = null; + if ($isTableExists($errTable)) { + $errContent = $this->connection->fetchAllAssociative('SELECT * FROM ' . TeradataQuote::quoteSingleIdentifier($errTable)); + $this->connection->executeStatement('DROP TABLE ' . $errTable); + } + $err2Content = null; + if ($isTableExists($errTable2)) { + $err2Content = $this->connection->fetchAllAssociative('SELECT * FROM ' . TeradataQuote::quoteSingleIdentifier($errTable2)); + $this->connection->executeStatement('DROP TABLE ' . $errTable2); + } + // find the way how to get this out + + if ($process->getExitCode() !== 0) { + $qb = new TeradataTableQueryBuilder(); + // drop destination table it's not usable + $this->connection->executeStatement($qb->getDropTableCommand( + $destination->getSchemaName(), + $destination->getTableName() + )); + + throw new FailedTPTLoadException( + $process->getErrorOutput(), + $process->getOutput(), + $process->getExitCode(), + file_get_contents($temp->getTmpFolder() . '/import-1.out'), + $logContent, + $errContent, + $err2Content + ); + } + + $ref = new TeradataTableReflection( + $this->connection, + $destination->getSchemaName(), + $destination->getTableName() + ); + + return $ref->getRowsCount(); + } + + /** + * @return array{0: Temp, 1:string, 2:string, 3:string, 4: string[]} + */ + private function generateTPTScript( + Storage\S3\SourceFile $source, + TeradataTableDefinition $destination, + TeradataImportOptions $importOptions + ): array { + $temp = new Temp(); + $temp->initRunFolder(); + $folder = $temp->getTmpFolder(); + + $target = sprintf( + "%s.%s", + TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()), + TeradataQuote::quoteSingleIdentifier($destination->getTableName()), + ); + + if ($source->isSliced()) { + $moduleStr = sprintf( + 'AccessModuleInitStr = \'S3Region=%s S3Bucket=%s S3Prefix="%s/" S3SinglePartFile=False\'', + $source->getRegion(), + $source->getBucket(), + $source->getPrefix() + ); + } else { + $moduleStr = sprintf( + 'AccessModuleInitStr = \'S3Region=%s S3Bucket=%s S3Prefix="%s/" S3Object=%s S3SinglePartFile=True\'', + $source->getRegion(), + $source->getBucket(), + $source->getPrefix(), + $source->getFileName() + ); + } + $tptScript = <<getTeradataHost(); + $user = $importOptions->getTeradataUser(); + $pass = $importOptions->getTeradataPassword(); + $csvOptions = $source->getCsvOptions(); + $delimiter = $csvOptions->getDelimiter(); + $enclosure = $csvOptions->getEnclosure(); + if ($enclosure === '\'') { + $enclosure = '\\\''; + } + $escapedBy = $csvOptions->getEscapedBy(); + if ($escapedBy !== '') { + $escapedBy = sprintf(',FileReaderEscapeQuoteDelimiter = \'%s\'', $escapedBy); + } + $ignoredLines = $importOptions->getNumberOfIgnoredLines(); + + $tablesPrefix = BackendHelper::generateTempTableName(); + $logTable = $tablesPrefix . '_log'; + $logTableQuoted = TeradataQuote::quoteSingleIdentifier($logTable); + $errTable1 = $tablesPrefix . '_e1'; + $errTableQuoted = TeradataQuote::quoteSingleIdentifier($errTable1); + $errTable2 = $tablesPrefix . '_e2'; + $errTable2Quoted = TeradataQuote::quoteSingleIdentifier($errTable2); + + $jobVariableFile = <<getColumnsDefinitions() as $definition) { + if ($definition->getColumnName() === $columnName) { + // if column exists in destination set destination type + $newDefinitions[] = new TeradataColumn( + $columnName, + new Teradata( + $definition->getColumnDefinition()->getType(), + [ + 'length' => $definition->getColumnDefinition()->getLength(), + 'nullable' => true, + 'default' => $definition->getColumnDefinition()->getDefault(), + ] + ) + ); + continue 2; + } + } + // if column doesn't exists in destination set default type + $newDefinitions[] = self::createVarcharColumn($columnName); + } + + return new TeradataTableDefinition( + $destination->getSchemaName(), + BackendHelper::generateTempTableName(), + true, + new ColumnCollection($newDefinitions), + $destination->getPrimaryKeysNames() + ); + } + + private static function createVarcharColumn(string $columnName): TeradataColumn + { + return new TeradataColumn( + $columnName, + new Teradata( + Teradata::TYPE_VARCHAR, + [ + 'length' => 32000, + 'nullable' => true, + ] + ) + ); + } + + /** + * @param string[] $sourceColumnsNames + */ + public static function createStagingTableDefinitionWithText( + TeradataTableDefinition $destination, + array $sourceColumnsNames + ): TeradataTableDefinition { + $newDefinitions = []; + foreach ($sourceColumnsNames as $columnName) { + $newDefinitions[] = self::createVarcharColumn($columnName); + } + + return new TeradataTableDefinition( + $destination->getSchemaName(), + BackendHelper::generateTempTableName(), + true, + new ColumnCollection($newDefinitions), + $destination->getPrimaryKeysNames() + ); + } +} diff --git a/src/Backend/Teradata/ToStage/ToStageImporter.php b/src/Backend/Teradata/ToStage/ToStageImporter.php new file mode 100644 index 00000000..5c6cc4de --- /dev/null +++ b/src/Backend/Teradata/ToStage/ToStageImporter.php @@ -0,0 +1,77 @@ +connection = $connection; + } + + public function importToStagingTable( + Storage\SourceInterface $source, + TableDefinitionInterface $destinationDefinition, + ImportOptionsInterface $options + ): ImportState { + assert($destinationDefinition instanceof TeradataTableDefinition); + $state = new ImportState($destinationDefinition->getTableName()); + + $adapter = $this->getAdapter($source); + + $state->startTimer(self::TIMER_TABLE_IMPORT); + try { + $state->addImportedRowsCount( + $adapter->runCopyCommand( + $source, + $destinationDefinition, + $options + ) + ); + } catch (\Doctrine\DBAL\Exception $e) { + throw SynapseException::covertException($e); + } + $state->stopTimer(self::TIMER_TABLE_IMPORT); + + return $state; + } + + private function getAdapter(Storage\SourceInterface $source): CopyAdapterInterface + { + switch (true) { + case $source instanceof Storage\S3\SourceFile: + $adapter = new FromS3TPTAdapter($this->connection); + //$adapter = new FromLocalTPTAdapter($this->connection); + break; + //case $source instanceof Storage\SqlSourceInterface: + // $adapter = new FromTableInsertIntoAdapter($this->connection); + // break; + default: + throw new LogicException( + sprintf( + 'No suitable adapter found for source: "%s".', + get_class($source) + ) + ); + } + return $adapter; + } +} diff --git a/src/Exception/Exception.php b/src/Exception/Exception.php new file mode 100644 index 00000000..20411319 --- /dev/null +++ b/src/Exception/Exception.php @@ -0,0 +1,9 @@ +filePath; } + public function getBucket(): string + { + return $this->bucket; + } + public function getBucketURL(): string { return sprintf('https://%s.s3.%s.amazonaws.com', $this->bucket, $this->region); diff --git a/src/Storage/S3/SourceDirectory.php b/src/Storage/S3/SourceDirectory.php index f3188880..ab93988c 100644 --- a/src/Storage/S3/SourceDirectory.php +++ b/src/Storage/S3/SourceDirectory.php @@ -12,11 +12,7 @@ class SourceDirectory extends SourceFile public function getManifestEntries(): array { $client = $this->getClient(); - $prefix = $this->filePath; - if (substr($prefix, -1) !== '/') { - // add trailing slash if not set to list only blobs in folder - $prefix .= '/'; - } + $prefix = $this->getPrefix(); $response = $client->listObjectsV2([ 'Bucket' => $this->bucket, 'Delimiter' => '/', @@ -27,4 +23,15 @@ public function getManifestEntries(): array return $file['Key']; }, $response->get('Contents')); } + + public function getPrefix(): string + { + $prefix = $this->filePath; + if (substr($prefix, -1) !== '/') { + // add trailing slash if not set to list only blobs in folder + $prefix .= '/'; + } + + return $prefix; + } } diff --git a/src/Storage/S3/SourceFile.php b/src/Storage/S3/SourceFile.php index 09defe95..72bd21e3 100644 --- a/src/Storage/S3/SourceFile.php +++ b/src/Storage/S3/SourceFile.php @@ -94,6 +94,11 @@ public function getManifestEntries(): array }, $manifest['entries']); } + public function isSliced(): bool + { + return $this->isSliced; + } + /** * @return string[]|null */ @@ -101,4 +106,19 @@ public function getPrimaryKeysNames(): ?array { return $this->primaryKeysNames; } + + public function getFileName(): string + { + if ($this->isSliced) { + throw new \Exception('Not supported getFileName for sliced files.'); + } + $fileName = $this->filePath; + return substr($fileName, strrpos($fileName, '/') + 1); + } + + public function getPrefix(): string + { + $prefix = $this->filePath; + return substr($prefix, 0, strrpos($prefix, '/')); + } } diff --git a/tests/data/csv/simple/a_b_c-1row.csv b/tests/data/csv/simple/a_b_c-1row.csv new file mode 100644 index 00000000..0eadb693 --- /dev/null +++ b/tests/data/csv/simple/a_b_c-1row.csv @@ -0,0 +1,2 @@ +a,b,c +1,2,3 \ No newline at end of file diff --git a/tests/functional/Teradata/StageImportTest.php b/tests/functional/Teradata/StageImportTest.php new file mode 100644 index 00000000..0505a331 --- /dev/null +++ b/tests/functional/Teradata/StageImportTest.php @@ -0,0 +1,61 @@ +cleanDatabase(self::TEST_DATABASE); + $this->createDatabase(self::TEST_DATABASE); + } + + public function testSimpleStageImport(): void + { + $this->connection->executeQuery( + sprintf( + 'CREATE MULTISET TABLE %s.%s ,NO FALLBACK + ( + "id" INTEGER NOT NULL, + "first_name" CHAR(32000), + "last_name" CHAR(32000) + );', + TeradataQuote::quoteSingleIdentifier(self::TEST_DATABASE), + TeradataQuote::quoteSingleIdentifier(self::TABLE_GENERIC) + ) + ); + + $importer = new ToStageImporter($this->connection); + $ref = new TeradataTableReflection( + $this->connection, + self::TEST_DATABASE, + self::TABLE_GENERIC + ); + + $state = $importer->importToStagingTable( + $this->createS3SourceInstanceFromCsv('csv/simple/a_b_c-1row.csv', new CsvOptions()), + $ref->getTableDefinition(), + new TeradataImportOptions( + (string) getenv('TERADATA_HOST'), + (string) getenv('TERADATA_USERNAME'), + (string) getenv('TERADATA_PASSWORD'), + (int) getenv('TERADATA_PORT'), + [], + false, + false, + 1 + ) + ); + + $this->assertEquals(1, $state->getResult()->getImportedRowsCount()); + } +} From f4b16a1eb6947f1bdb3147b21b1adbe2356e0859 Mon Sep 17 00:00:00 2001 From: zajca Date: Thu, 10 Mar 2022 15:37:37 +0100 Subject: [PATCH 04/62] fix error tables database --- .../Teradata/ToStage/FromS3TPTAdapter.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index c4ae4168..c6847586 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -77,21 +77,22 @@ public function runCopyCommand( // echo "\nRead from stderr: " . $data; // } //} - $isTableExists = fn(string $tableName - ) => (bool) $this->connection->fetchOne(sprintf('SELECT 1 FROM dbc.TablesV WHERE TableName = %s', TeradataQuote::quote($tableName))); + $isTableExists = function (string $tableName, string $databaseName) { + return (bool) $this->connection->fetchOne(sprintf('SELECT 1 FROM dbc.TablesV WHERE TableName = %s AND DataBaseName = %s', TeradataQuote::quote($tableName), TeradataQuote::quote($databaseName))); + }; $logContent = null; - if ($isTableExists($logTable)) { + if ($isTableExists($logTable, $destination->getSchemaName())) { $logContent = $this->connection->fetchAllAssociative('SELECT * FROM ' . TeradataQuote::quoteSingleIdentifier($logTable)); $this->connection->executeStatement('DROP TABLE ' . $logTable); } $errContent = null; - if ($isTableExists($errTable)) { + if ($isTableExists($errTable, $destination->getSchemaName())) { $errContent = $this->connection->fetchAllAssociative('SELECT * FROM ' . TeradataQuote::quoteSingleIdentifier($errTable)); $this->connection->executeStatement('DROP TABLE ' . $errTable); } $err2Content = null; - if ($isTableExists($errTable2)) { + if ($isTableExists($errTable2, $destination->getSchemaName())) { $err2Content = $this->connection->fetchAllAssociative('SELECT * FROM ' . TeradataQuote::quoteSingleIdentifier($errTable2)); $this->connection->executeStatement('DROP TABLE ' . $errTable2); } @@ -200,11 +201,11 @@ private function generateTPTScript( $tablesPrefix = BackendHelper::generateTempTableName(); $logTable = $tablesPrefix . '_log'; - $logTableQuoted = TeradataQuote::quoteSingleIdentifier($logTable); + $logTableQuoted = TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()) . '.' . TeradataQuote::quoteSingleIdentifier($logTable); $errTable1 = $tablesPrefix . '_e1'; - $errTableQuoted = TeradataQuote::quoteSingleIdentifier($errTable1); + $errTableQuoted = TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()) . '.' . TeradataQuote::quoteSingleIdentifier($errTable1); $errTable2 = $tablesPrefix . '_e2'; - $errTable2Quoted = TeradataQuote::quoteSingleIdentifier($errTable2); + $errTable2Quoted = TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()) . '.' . TeradataQuote::quoteSingleIdentifier($errTable2); $jobVariableFile = << Date: Thu, 10 Mar 2022 16:39:02 +0100 Subject: [PATCH 05/62] limit perm space for testing db --- .../Teradata/ToStage/FromS3TPTAdapter.php | 16 ++++++++-------- .../functional/Teradata/TeradataBaseTestCase.php | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index c6847586..fe1abdde 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -69,14 +69,14 @@ public function runCopyCommand( ); $process->start(); - //// debug stuff - //foreach ($process as $type => $data) { - // if ($process::OUT === $type) { - // echo "\nRead from stdout: " . $data; - // } else { // $process::ERR === $type - // echo "\nRead from stderr: " . $data; - // } - //} + // debug stuff + foreach ($process as $type => $data) { + if ($process::OUT === $type) { + echo "\nRead from stdout: " . $data; + } else { // $process::ERR === $type + echo "\nRead from stderr: " . $data; + } + } $isTableExists = function (string $tableName, string $databaseName) { return (bool) $this->connection->fetchOne(sprintf('SELECT 1 FROM dbc.TablesV WHERE TableName = %s AND DataBaseName = %s', TeradataQuote::quote($tableName), TeradataQuote::quote($databaseName))); }; diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index 8afc66e2..aa94202e 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -97,7 +97,7 @@ public function createDatabase(string $dbName): void { $this->connection->executeQuery(sprintf(' CREATE DATABASE %s AS - PERM = 1e9; + PERM = 5e6; ', $dbName)); } From 75496bbfbf12753fddc7a107a80e5811c35f9c9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Thu, 10 Mar 2022 16:39:13 +0100 Subject: [PATCH 06/62] env and docs --- .env.dist | 5 +++++ README.md | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/.env.dist b/.env.dist index 3652e8ea..ae59a480 100644 --- a/.env.dist +++ b/.env.dist @@ -25,3 +25,8 @@ EXASOL_PASSWORD=exasol # ID of build used in CI only BUILD_PREFIX= + +TERADATA_HOST= +TERADATA_USERNAME= +TERADATA_PASSWORD= +TERADATA_PORT= diff --git a/README.md b/README.md index 12869eae..0ae94b92 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,13 @@ ## Development +### Docker + +Prepare `.env` (copy of `.env.dist`) and set up AWS keys which has access to `keboola-drivers` bucket in order to build this image. Then run `docker-compose --env-file=.env.local build` + +The AWS credentials have to also have access to bucket specified in `AWS_S3_BUCKET`. This bucket has to contain testing data. Run `docker-compose run --rm dev composer loadS3` to load them up. + + ### Preparation #### Azure @@ -96,6 +103,15 @@ EXASOL_PASSWORD= Obtain host (with port), username and password from Exasol SaaS for your testing DB and fill it in `.env` as desribed above. Make sure, that your account has enabled network for your IP. +#### Teradata +```bash +TERADATA_HOST= +TERADATA_USERNAME= +TERADATA_PASSWORD=JirkaTdPassword+ +TERADATA_PORT= +``` + + ### Tests Run tests with following command. From 11a673496897ed93a2f63da6f54d31c7ba4b6681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Thu, 10 Mar 2022 17:20:23 +0100 Subject: [PATCH 07/62] init classes and tests for full import --- src/Backend/Teradata/TeradataException.php | 16 ++ .../Teradata/ToFinalTable/FullImporter.php | 117 +++++++++++ .../Teradata/ToFinalTable/SqlBuilder.php | 44 ++++ tests/functional/Teradata/FullImportTest.php | 196 ++++++++++++++++++ tests/functional/Teradata/StageImportTest.php | 22 +- .../Teradata/TeradataBaseTestCase.php | 79 ++++++- 6 files changed, 455 insertions(+), 19 deletions(-) create mode 100644 src/Backend/Teradata/TeradataException.php create mode 100644 src/Backend/Teradata/ToFinalTable/FullImporter.php create mode 100644 src/Backend/Teradata/ToFinalTable/SqlBuilder.php create mode 100644 tests/functional/Teradata/FullImportTest.php diff --git a/src/Backend/Teradata/TeradataException.php b/src/Backend/Teradata/TeradataException.php new file mode 100644 index 00000000..23557f9b --- /dev/null +++ b/src/Backend/Teradata/TeradataException.php @@ -0,0 +1,16 @@ +connection = $connection; + $this->sqlBuilder = new SqlBuilder(); + } + + private function doLoadFullWithoutDedup( + TeradataTableDefinition $stagingTableDefinition, + TeradataTableDefinition $destinationTableDefinition, + TeradataImportOptions $options, + ImportState $state + ) { + // truncate destination table + $this->connection->executeStatement( + $this->sqlBuilder->getTruncateTableWithDeleteCommand( + $destinationTableDefinition->getSchemaName(), + $destinationTableDefinition->getTableName() + ) + ); + $state->startTimer(self::TIMER_COPY_TO_TARGET); + + // move data with INSERT INTO + $this->connection->executeStatement( + $this->sqlBuilder->getInsertAllIntoTargetTableCommand( + $stagingTableDefinition, + $destinationTableDefinition, + $options, + DateTimeHelper::getNowFormatted() + ) + ); + $state->stopTimer(self::TIMER_COPY_TO_TARGET); + + $this->connection->executeStatement( + $this->sqlBuilder->getCommitTransaction() + ); + } + + public function importToTable( + TableDefinitionInterface $stagingTableDefinition, + TableDefinitionInterface $destinationTableDefinition, + ImportOptionsInterface $options, + ImportState $state + ): Result { + assert($stagingTableDefinition instanceof TeradataTableDefinition); + assert($destinationTableDefinition instanceof TeradataTableDefinition); + assert($options instanceof TeradataImportOptions); + + + /** @var TeradataTableDefinition $destinationTableDefinition */ + try { + //import files to staging table + if (!empty($destinationTableDefinition->getPrimaryKeysNames())) { + throw new \Exception('not implemented yet'); + } else { + $this->doLoadFullWithoutDedup( + $stagingTableDefinition, + $destinationTableDefinition, + $options, + $state + ); + } + } catch (Exception $e) { + throw TeradataException::covertException($e); + } finally { + // drop optimized load tmp table if exists + $this->connection->executeStatement( + $this->sqlBuilder->getDropTableIfExistsCommand( + $destinationTableDefinition->getSchemaName(), + $destinationTableDefinition->getTableName() . self::OPTIMIZED_LOAD_TMP_TABLE_SUFFIX + ) + ); + // drop optimized load rename table if exists + $this->connection->executeStatement( + $this->sqlBuilder->getDropTableIfExistsCommand( + $destinationTableDefinition->getSchemaName(), + $destinationTableDefinition->getTableName() . self::OPTIMIZED_LOAD_RENAME_TABLE_SUFFIX + ) + ); + } + + $state->setImportedColumns($stagingTableDefinition->getColumnsNames()); + + return $state->getResult(); + } +} diff --git a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php new file mode 100644 index 00000000..6555428b --- /dev/null +++ b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php @@ -0,0 +1,44 @@ +cleanDatabase($this->getDestinationDbName()); + $this->createDatabase($this->getDestinationDbName()); + + $this->cleanDatabase($this->getSourceDbName()); + $this->createDatabase($this->getSourceDbName()); + } + + public function testLoadToFinalTableWithoutDedup(): void + { + $this->initTable(self::TABLE_COLUMN_NAME_ROW_NUMBER); + + // skipping header + $options = $this->getImportOptions(); + $source = $this->createS3SourceInstance( + 'column-name-row-number.csv', + [ + 'id', + 'row_number', + ], + false, + false, + [] + ); + + $importer = new ToStageImporter($this->connection); + $destinationRef = new TeradataTableReflection( + $this->connection, + $this->getDestinationDbName(), + self::TABLE_COLUMN_NAME_ROW_NUMBER + ); + $destination = $destinationRef->getTableDefinition(); + $stagingTable = StageTableDefinitionFactory::createStagingTableDefinition($destination, [ + 'id', + 'row_number', + ]); + $qb = new TeradataTableQueryBuilder(); + $this->connection->executeStatement( + $qb->getCreateTableCommandFromDefinition($stagingTable) + ); + $importState = $importer->importToStagingTable( + $source, + $stagingTable, + $options + ); + $toFinalTableImporter = new FullImporter($this->connection); + $result = $toFinalTableImporter->importToTable( + $stagingTable, + $destination, + $options, + $importState + ); + + self::assertEquals(2, $destinationRef->getRowsCount()); + } + + public function testLoadToTableWithDedupWithSinglePK(): void + { + $this->initTable(self::TABLE_SINGLE_PK); + + // skipping header + $options = $this->getTeradataImportOptions(1, false); + $source = $this->createS3SourceInstance( + 'multi-pk.csv', + [ + 'VisitID', + 'Value', + 'MenuItem', + 'Something', + 'Other', + ], + false, + false, + ['VisitID'] + ); + + $importer = new ToStageImporter($this->connection); + $destinationRef = new TeradataTableReflection( + $this->connection, + $this->getDestinationDbName(), + self::TABLE_SINGLE_PK + ); + $destination = $destinationRef->getTableDefinition(); + $stagingTable = StageTableDefinitionFactory::createStagingTableDefinition($destination, [ + 'VisitID', + 'Value', + 'MenuItem', + 'Something', + 'Other', + ]); + $qb = new TeradataTableQueryBuilder(); + $this->connection->executeStatement( + $qb->getCreateTableCommandFromDefinition($stagingTable) + ); + $importState = $importer->importToStagingTable( + $source, + $stagingTable, + $options + ); + $toFinalTableImporter = new FullImporter($this->connection); + $result = $toFinalTableImporter->importToTable( + $stagingTable, + $destination, + $options, + $importState + ); + + self::assertEquals(4, $destinationRef->getRowsCount()); + } + + public function testLoadToTableWithDedupWithMultiPK(): void + { + $this->initTable(self::TABLE_MULTI_PK); + + // skipping header + $options = $this->getTeradataImportOptions(1, false); + $source = $this->createS3SourceInstance( + 'multi-pk.csv', + [ + 'VisitID', + 'Value', + 'MenuItem', + 'Something', + 'Other', + ], + false, + false, + ['VisitID', 'Something'] + ); + + $importer = new ToStageImporter($this->connection); + $destinationRef = new TeradataTableReflection( + $this->connection, + $this->getDestinationDbName(), + self::TABLE_MULTI_PK + ); + $destination = $destinationRef->getTableDefinition(); + $stagingTable = StageTableDefinitionFactory::createStagingTableDefinition($destination, [ + 'VisitID', + 'Value', + 'MenuItem', + 'Something', + 'Other', + ]); + $qb = new TeradataTableQueryBuilder(); + $this->connection->executeStatement( + $qb->getCreateTableCommandFromDefinition($stagingTable) + ); + $importState = $importer->importToStagingTable( + $source, + $stagingTable, + $options + ); + + // now 6 lines. Add one with same VisitId and Something as an existing line has + // -> expecting that this line will be skipped when DEDUP + $this->connection->executeQuery( + sprintf( + "INSERT INTO %s.%s VALUES ('134', 'xx', 'yy', 'abc', 'def');", + TeradataQuote::quoteSingleIdentifier($stagingTable->getSchemaName()), + TeradataQuote::quoteSingleIdentifier($stagingTable->getTableName()) + ) + ); + $toFinalTableImporter = new FullImporter($this->connection); + $result = $toFinalTableImporter->importToTable( + $stagingTable, + $destination, + $options, + $importState + ); + + self::assertEquals(6, $destinationRef->getRowsCount()); + } + + +} diff --git a/tests/functional/Teradata/StageImportTest.php b/tests/functional/Teradata/StageImportTest.php index 0505a331..6471cd9c 100644 --- a/tests/functional/Teradata/StageImportTest.php +++ b/tests/functional/Teradata/StageImportTest.php @@ -42,19 +42,15 @@ public function testSimpleStageImport(): void ); $state = $importer->importToStagingTable( - $this->createS3SourceInstanceFromCsv('csv/simple/a_b_c-1row.csv', new CsvOptions()), - $ref->getTableDefinition(), - new TeradataImportOptions( - (string) getenv('TERADATA_HOST'), - (string) getenv('TERADATA_USERNAME'), - (string) getenv('TERADATA_PASSWORD'), - (int) getenv('TERADATA_PORT'), - [], - false, - false, - 1 - ) - ); + $this->createS3SourceInstanceFromCsv('csv/simple/a_b_c-1row.csv', new CsvOptions()), + $ref->getTableDefinition(), + $this->getImportOptions( + [], + false, + false, + 1 + ) + ); $this->assertEquals(1, $state->getResult()->getImportedRowsCount()); } diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index aa94202e..d1eefc0c 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -6,11 +6,13 @@ use Doctrine\DBAL\Connection; use Keboola\Datatype\Definition\Teradata; +use Keboola\Db\ImportExport\Backend\Teradata\TeradataImportOptions; use Keboola\Db\ImportExport\ImportOptions; use Keboola\Db\ImportExport\Storage\SourceInterface; use Keboola\TableBackendUtils\Column\ColumnCollection; use Keboola\TableBackendUtils\Column\Teradata\TeradataColumn; use Keboola\TableBackendUtils\Connection\Teradata\TeradataConnection; +use Keboola\TableBackendUtils\Escaping\Teradata\TeradataQuote; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableDefinition; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableReflection; use Tests\Keboola\Db\ImportExportFunctional\ImportExportBaseTest; @@ -18,13 +20,27 @@ class TeradataBaseTestCase extends ImportExportBaseTest { public const TESTS_PREFIX = 'ieLibTest_'; - public const TEST_DATABASE = self::TESTS_PREFIX . 'refTableDatabase'; public const TABLE_GENERIC = self::TESTS_PREFIX . 'refTab'; public const VIEW_GENERIC = self::TESTS_PREFIX . 'refView'; - protected const TERADATA_SOURCE_DATABASE_NAME = 'tests_source'; protected const TERADATA_DESTINATION_DATABASE_NAME = 'tests_source'; + // TODO move somewhere else + protected const EXASOL_DEST_SCHEMA_NAME = 'in_c-tests'; + protected const EXASOL_SOURCE_SCHEMA_NAME = 'some_tests'; + public const TABLE_ACCOUNTS_3 = 'accounts-3'; + public const TABLE_ACCOUNTS_BEZ_TS = 'accounts-bez-ts'; + public const TABLE_COLUMN_NAME_ROW_NUMBER = 'column-name-row-number'; + public const TABLE_MULTI_PK = 'multi-pk'; + public const TABLE_MULTI_PK_WITH_TS = 'multi-pk_ts'; + public const TABLE_SINGLE_PK = 'single-pk'; + public const TABLE_OUT_CSV_2COLS = 'out_csv_2Cols'; + public const TABLE_OUT_CSV_2COLS_WITHOUT_TS = 'out_csv_2Cols_without_ts'; + public const TABLE_NULLIFY = 'nullify'; + public const TABLE_OUT_LEMMA = 'out_lemma'; + public const TABLE_OUT_NO_TIMESTAMP_TABLE = 'out_no_timestamp_table'; + public const TABLE_TABLE = 'table'; + public const TABLE_TYPES = 'types'; protected Connection $connection; @@ -34,6 +50,20 @@ protected function setUp(): void $this->connection = $this->getTeradataConnection(); } + protected function getSourceDbName(): string + { + return self::TERADATA_SOURCE_DATABASE_NAME + . '-' + . getenv('SUITE'); + } + + protected function getDestinationDbName(): string + { + return self::TERADATA_DESTINATION_DATABASE_NAME + . '-' + . getenv('SUITE'); + } + private function getTeradataConnection(): Connection { return TeradataConnection::getConnection([ @@ -167,10 +197,47 @@ protected function assertSynapseTableEqualsExpected( ); } - protected function getDestinationSchemaName(): string + protected function initTable(string $tableName): void { - return self::EXASOL_SOURCE_SCHEMA_NAME - . '-' - . getenv('SUITE'); + switch ($tableName) { + case self::TABLE_OUT_CSV_2COLS_WITHOUT_TS: + $this->connection->executeQuery( + sprintf( + 'CREATE MULTISET TABLE %s.%s, NO FALLBACK + ( +"VisitID" VARCHAR(2000000) , +"Value" VARCHAR(2000000), +"MenuItem" VARCHAR(2000000), +"Something" VARCHAR(2000000), +"Other" VARCHAR(2000000), + ) +PRIMARY INDEX ("VisitID"); + );', + TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), + TeradataQuote::quoteSingleIdentifier($tableName) + ) + ); + break; + } + } + + protected function getImportOptions( + array $convertEmptyValuesToNull = [], + bool $isIncremental = false, + bool $useTimestamp = false, + int $numberOfIgnoredLines = 0 + + ): TeradataImportOptions { + return + new TeradataImportOptions( + (string) getenv('TERADATA_HOST'), + (string) getenv('TERADATA_USERNAME'), + (string) getenv('TERADATA_PASSWORD'), + (int) getenv('TERADATA_PORT'), + $convertEmptyValuesToNull, + $isIncremental, + $useTimestamp, + $numberOfIgnoredLines, + ); } } From ffd99ecb528878ba753736c145ecf2d5300858f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 11 Mar 2022 10:16:11 +0100 Subject: [PATCH 08/62] escape db names --- tests/functional/Teradata/TeradataBaseTestCase.php | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index d1eefc0c..41c9bbb3 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -118,9 +118,9 @@ protected function cleanDatabase(string $dbname): void } // delete all objects in the DB - $this->connection->executeQuery(sprintf('DELETE DATABASE %s ALL', $dbname)); + $this->connection->executeQuery(sprintf('DELETE DATABASE %s ALL', TeradataQuote::quoteSingleIdentifier($dbname))); // drop the empty db - $this->connection->executeQuery(sprintf('DROP DATABASE %s', $dbname)); + $this->connection->executeQuery(sprintf('DROP DATABASE %s', TeradataQuote::quoteSingleIdentifier($dbname))); } public function createDatabase(string $dbName): void @@ -128,16 +128,20 @@ public function createDatabase(string $dbName): void $this->connection->executeQuery(sprintf(' CREATE DATABASE %s AS PERM = 5e6; - ', $dbName)); + ', TeradataQuote::quoteSingleIdentifier($dbName))); } protected function dbExists(string $dbname): bool { try { - $this->connection->executeQuery(sprintf('HELP DATABASE %s', $dbname)); + $this->connection->executeQuery(sprintf('HELP DATABASE %s', TeradataQuote::quoteSingleIdentifier($dbname))); return true; } catch (\Doctrine\DBAL\Exception $e) { - return false; + // https://docs.teradata.com/r/GVKfXcemJFkTJh_89R34UQ/j2TdlzqRJ9LpndY3efMdlw + if (strpos($e->getMessage(), "3802")) { + return false; + } + throw $e; } } From 11935ce775de6dbf157027c7a0bbd5246cfe82fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 11 Mar 2022 10:16:28 +0100 Subject: [PATCH 09/62] init TABLE_COLUMN_NAME_ROW_NUMBER --- tests/functional/Teradata/TeradataBaseTestCase.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index 41c9bbb3..1a67a4e4 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -222,6 +222,18 @@ protected function initTable(string $tableName): void ) ); break; + case self::TABLE_COLUMN_NAME_ROW_NUMBER: + $this->connection->executeQuery(sprintf( + 'CREATE MULTISET TABLE %s.%s, NO FALLBACK + ( + "id" VARCHAR(4000) , + "row_number" VARCHAR(4000) , + "_timestamp" TIMESTAMP + )', + TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), + TeradataQuote::quoteSingleIdentifier($tableName) + )); + break; } } From 4f1dc278f5da83d13faf367fee0fab0e0452b432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 11 Mar 2022 10:17:01 +0100 Subject: [PATCH 10/62] first testLoadToFinalTableWithoutDedup ends on not implemented --- tests/functional/Teradata/FullImportTest.php | 121 +------------------ 1 file changed, 1 insertion(+), 120 deletions(-) diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index de6d4036..44a8b98d 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -33,7 +33,7 @@ public function testLoadToFinalTableWithoutDedup(): void // skipping header $options = $this->getImportOptions(); $source = $this->createS3SourceInstance( - 'column-name-row-number.csv', + self::TABLE_COLUMN_NAME_ROW_NUMBER . '.csv', [ 'id', 'row_number', @@ -74,123 +74,4 @@ public function testLoadToFinalTableWithoutDedup(): void self::assertEquals(2, $destinationRef->getRowsCount()); } - public function testLoadToTableWithDedupWithSinglePK(): void - { - $this->initTable(self::TABLE_SINGLE_PK); - - // skipping header - $options = $this->getTeradataImportOptions(1, false); - $source = $this->createS3SourceInstance( - 'multi-pk.csv', - [ - 'VisitID', - 'Value', - 'MenuItem', - 'Something', - 'Other', - ], - false, - false, - ['VisitID'] - ); - - $importer = new ToStageImporter($this->connection); - $destinationRef = new TeradataTableReflection( - $this->connection, - $this->getDestinationDbName(), - self::TABLE_SINGLE_PK - ); - $destination = $destinationRef->getTableDefinition(); - $stagingTable = StageTableDefinitionFactory::createStagingTableDefinition($destination, [ - 'VisitID', - 'Value', - 'MenuItem', - 'Something', - 'Other', - ]); - $qb = new TeradataTableQueryBuilder(); - $this->connection->executeStatement( - $qb->getCreateTableCommandFromDefinition($stagingTable) - ); - $importState = $importer->importToStagingTable( - $source, - $stagingTable, - $options - ); - $toFinalTableImporter = new FullImporter($this->connection); - $result = $toFinalTableImporter->importToTable( - $stagingTable, - $destination, - $options, - $importState - ); - - self::assertEquals(4, $destinationRef->getRowsCount()); - } - - public function testLoadToTableWithDedupWithMultiPK(): void - { - $this->initTable(self::TABLE_MULTI_PK); - - // skipping header - $options = $this->getTeradataImportOptions(1, false); - $source = $this->createS3SourceInstance( - 'multi-pk.csv', - [ - 'VisitID', - 'Value', - 'MenuItem', - 'Something', - 'Other', - ], - false, - false, - ['VisitID', 'Something'] - ); - - $importer = new ToStageImporter($this->connection); - $destinationRef = new TeradataTableReflection( - $this->connection, - $this->getDestinationDbName(), - self::TABLE_MULTI_PK - ); - $destination = $destinationRef->getTableDefinition(); - $stagingTable = StageTableDefinitionFactory::createStagingTableDefinition($destination, [ - 'VisitID', - 'Value', - 'MenuItem', - 'Something', - 'Other', - ]); - $qb = new TeradataTableQueryBuilder(); - $this->connection->executeStatement( - $qb->getCreateTableCommandFromDefinition($stagingTable) - ); - $importState = $importer->importToStagingTable( - $source, - $stagingTable, - $options - ); - - // now 6 lines. Add one with same VisitId and Something as an existing line has - // -> expecting that this line will be skipped when DEDUP - $this->connection->executeQuery( - sprintf( - "INSERT INTO %s.%s VALUES ('134', 'xx', 'yy', 'abc', 'def');", - TeradataQuote::quoteSingleIdentifier($stagingTable->getSchemaName()), - TeradataQuote::quoteSingleIdentifier($stagingTable->getTableName()) - ) - ); - $toFinalTableImporter = new FullImporter($this->connection); - $result = $toFinalTableImporter->importToTable( - $stagingTable, - $destination, - $options, - $importState - ); - - self::assertEquals(6, $destinationRef->getRowsCount()); - } - - } From bb5f406b47b164a6e2a81adf3075488f4e099c12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 11 Mar 2022 13:52:17 +0100 Subject: [PATCH 11/62] sourcefile supports files which are not in a folder --- src/Storage/S3/SourceFile.php | 23 +++++++++++++++++++++-- tests/unit/Storage/S3/SourceFileTest.php | 7 +++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/Storage/S3/SourceFile.php b/src/Storage/S3/SourceFile.php index 72bd21e3..faee739e 100644 --- a/src/Storage/S3/SourceFile.php +++ b/src/Storage/S3/SourceFile.php @@ -107,18 +107,37 @@ public function getPrimaryKeysNames(): ?array return $this->primaryKeysNames; } + /** + * from path data/shared/file.csv to file.csv + * @return string + * @throws \Exception + */ public function getFileName(): string { if ($this->isSliced) { throw new \Exception('Not supported getFileName for sliced files.'); } $fileName = $this->filePath; - return substr($fileName, strrpos($fileName, '/') + 1); + if (strrpos($fileName, '/') !== false) { + // there is dir in the path + return substr($fileName, strrpos($fileName, '/') + 1); + } + // there is no dir in the path, just the filename + return $fileName; } + /** + * from path data/shared/file.csv to data/shared/ + * @return string + * @throws \Exception + */ public function getPrefix(): string { $prefix = $this->filePath; - return substr($prefix, 0, strrpos($prefix, '/')); + if (($prefixLength = strrpos($prefix, '/')) !== false) { + // include / at the end + return substr($prefix, 0, $prefixLength + 1); + } + return ''; } } diff --git a/tests/unit/Storage/S3/SourceFileTest.php b/tests/unit/Storage/S3/SourceFileTest.php index dad7e174..e429111f 100644 --- a/tests/unit/Storage/S3/SourceFileTest.php +++ b/tests/unit/Storage/S3/SourceFileTest.php @@ -24,4 +24,11 @@ public function testDefaultValues(): void self::assertEquals([], $source->getColumnsNames()); self::assertNull($source->getPrimaryKeysNames()); } + + public function testGetFilepathParts() + { + $source = $this->createDummyS3SourceInstance('data/shared/file.csv'); + self::assertEquals('data/shared/', $source->getPrefix()); + self::assertEquals('file.csv', $source->getFileName()); + } } From f85d5ac123af9a2768df2c730abae682d7955abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 11 Mar 2022 13:57:52 +0100 Subject: [PATCH 12/62] tpt expects path with slash --- src/Backend/Teradata/ToStage/FromS3TPTAdapter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index fe1abdde..40da9320 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -153,7 +153,7 @@ private function generateTPTScript( ); } else { $moduleStr = sprintf( - 'AccessModuleInitStr = \'S3Region=%s S3Bucket=%s S3Prefix="%s/" S3Object=%s S3SinglePartFile=True\'', + 'AccessModuleInitStr = \'S3Region=%s S3Bucket=%s S3Prefix="%s" S3Object=%s S3SinglePartFile=True\'', $source->getRegion(), $source->getBucket(), $source->getPrefix(), From 982e24c2e307dcf71e9e36bcc79c694c1393729e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 11 Mar 2022 13:58:05 +0100 Subject: [PATCH 13/62] init test for full load --- src/Backend/Teradata/ToFinalTable/SqlBuilder.php | 11 ++++++----- tests/functional/Teradata/FullImportTest.php | 1 - tests/functional/Teradata/TeradataBaseTestCase.php | 3 +-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php index 6555428b..38b6f80f 100644 --- a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php +++ b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php @@ -12,14 +12,14 @@ class SqlBuilder { public function getCommitTransaction(): string { - // TODO - return ''; + //TODO + throw new \Exception('not implemented yet'); } public function getDropTableIfExistsCommand(string $getSchemaName, string $string): string { - // TODO - return ''; + //TODO + throw new \Exception('not implemented yet'); } public function getInsertAllIntoTargetTableCommand( @@ -28,7 +28,8 @@ public function getInsertAllIntoTargetTableCommand( TeradataImportOptions $options, string $getNowFormatted ): string { - return ''; + //TODO + throw new \Exception('not implemented yet'); } public function getTruncateTableWithDeleteCommand( diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index 44a8b98d..f63b552e 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -7,7 +7,6 @@ use Keboola\Db\ImportExport\Backend\Teradata\ToFinalTable\FullImporter; use Keboola\Db\ImportExport\Backend\Teradata\ToStage\StageTableDefinitionFactory; use Keboola\Db\ImportExport\Backend\Teradata\ToStage\ToStageImporter; -use Keboola\TableBackendUtils\Escaping\Teradata\TeradataQuote; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableQueryBuilder; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableReflection; use Tests\Keboola\Db\ImportExport\S3SourceTrait; diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index 1a67a4e4..8b80926e 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -227,8 +227,7 @@ protected function initTable(string $tableName): void 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( "id" VARCHAR(4000) , - "row_number" VARCHAR(4000) , - "_timestamp" TIMESTAMP + "row_number" VARCHAR(4000) )', TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), TeradataQuote::quoteSingleIdentifier($tableName) From 529a6e8ac676025fbc101011ca185f86012c87b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 11 Mar 2022 14:28:27 +0100 Subject: [PATCH 14/62] fix droping table --- .../Teradata/ToFinalTable/FullImporter.php | 43 +++++++++++++------ .../Teradata/ToFinalTable/SqlBuilder.php | 25 +++++++++-- 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/src/Backend/Teradata/ToFinalTable/FullImporter.php b/src/Backend/Teradata/ToFinalTable/FullImporter.php index c59a98be..69307257 100644 --- a/src/Backend/Teradata/ToFinalTable/FullImporter.php +++ b/src/Backend/Teradata/ToFinalTable/FullImporter.php @@ -82,6 +82,7 @@ public function importToTable( try { //import files to staging table if (!empty($destinationTableDefinition->getPrimaryKeysNames())) { + // dedup throw new \Exception('not implemented yet'); } else { $this->doLoadFullWithoutDedup( @@ -94,24 +95,38 @@ public function importToTable( } catch (Exception $e) { throw TeradataException::covertException($e); } finally { - // drop optimized load tmp table if exists - $this->connection->executeStatement( - $this->sqlBuilder->getDropTableIfExistsCommand( - $destinationTableDefinition->getSchemaName(), - $destinationTableDefinition->getTableName() . self::OPTIMIZED_LOAD_TMP_TABLE_SUFFIX - ) - ); - // drop optimized load rename table if exists - $this->connection->executeStatement( - $this->sqlBuilder->getDropTableIfExistsCommand( - $destinationTableDefinition->getSchemaName(), - $destinationTableDefinition->getTableName() . self::OPTIMIZED_LOAD_RENAME_TABLE_SUFFIX - ) - ); + $tmpTableName = $destinationTableDefinition->getTableName() . self::OPTIMIZED_LOAD_TMP_TABLE_SUFFIX; + $renameTableName = $destinationTableDefinition->getTableName() . self::OPTIMIZED_LOAD_RENAME_TABLE_SUFFIX; + + if ($this->tableExists($destinationTableDefinition->getSchemaName(), $renameTableName)) { + // drop optimized load tmp table if exists + $this->connection->executeStatement( + $this->sqlBuilder->getDropTableUnsafe( + $destinationTableDefinition->getSchemaName(), + $tmpTableName + ) + ); + } + + if ($this->tableExists($destinationTableDefinition->getSchemaName(), $renameTableName)) { + // drop optimized load rename table if exists + $this->connection->executeStatement( + $this->sqlBuilder->getDropTableUnsafe( + $destinationTableDefinition->getSchemaName(), + $renameTableName + ) + ); + } } $state->setImportedColumns($stagingTableDefinition->getColumnsNames()); return $state->getResult(); } + + protected function tableExists($dbName, $tableName): bool + { + $data = $this->connection->fetchOne($this->sqlBuilder->getTableExistsCommand($dbName, $tableName)); + return ((int) $data) > 0; + } } diff --git a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php index 38b6f80f..718e5869 100644 --- a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php +++ b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php @@ -16,10 +16,29 @@ public function getCommitTransaction(): string throw new \Exception('not implemented yet'); } - public function getDropTableIfExistsCommand(string $getSchemaName, string $string): string + /** + * SQL to drop table. DOES NOT check existence of table + * + * @param string $dbName + * @param string $tableName + * @return string + */ + public function getDropTableUnsafe(string $dbName, string $tableName): string { - //TODO - throw new \Exception('not implemented yet'); + return sprintf( + 'DROP TABLE %s.%s', + TeradataQuote::quoteSingleIdentifier($dbName), + TeradataQuote::quoteSingleIdentifier($tableName) + ); + } + + public function getTableExistsCommand(string $dbName, string $tableName): string + { + return sprintf( + 'SELECT COUNT(*) FROM DBC.Tables WHERE DatabaseName = %s AND TableName = %s;', + TeradataQuote::quote($dbName), + TeradataQuote::quote($tableName) + ); } public function getInsertAllIntoTargetTableCommand( From 831dd6cabff76589795571571aa1f11fb730d660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 11 Mar 2022 15:10:01 +0100 Subject: [PATCH 15/62] first implementation of full load --- .../Teradata/ToFinalTable/FullImporter.php | 19 +++-- .../Teradata/ToFinalTable/SqlBuilder.php | 83 +++++++++++++++++-- tests/functional/Teradata/FullImportTest.php | 2 +- 3 files changed, 89 insertions(+), 15 deletions(-) diff --git a/src/Backend/Teradata/ToFinalTable/FullImporter.php b/src/Backend/Teradata/ToFinalTable/FullImporter.php index 69307257..4af2f87c 100644 --- a/src/Backend/Teradata/ToFinalTable/FullImporter.php +++ b/src/Backend/Teradata/ToFinalTable/FullImporter.php @@ -52,19 +52,20 @@ private function doLoadFullWithoutDedup( $state->startTimer(self::TIMER_COPY_TO_TARGET); // move data with INSERT INTO + $sql = $this->sqlBuilder->getInsertAllIntoTargetTableCommand( + $stagingTableDefinition, + $destinationTableDefinition, + $options, + DateTimeHelper::getNowFormatted() + ); $this->connection->executeStatement( - $this->sqlBuilder->getInsertAllIntoTargetTableCommand( - $stagingTableDefinition, - $destinationTableDefinition, - $options, - DateTimeHelper::getNowFormatted() - ) + $sql ); $state->stopTimer(self::TIMER_COPY_TO_TARGET); - $this->connection->executeStatement( - $this->sqlBuilder->getCommitTransaction() - ); +// $this->connection->executeStatement( +// $this->sqlBuilder->getCommitTransaction() +// ); } public function importToTable( diff --git a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php index 718e5869..2d32abdc 100644 --- a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php +++ b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php @@ -4,12 +4,17 @@ namespace Keboola\Db\ImportExport\Backend\Teradata\ToFinalTable; +use Keboola\Datatype\Definition\BaseType; use Keboola\Db\ImportExport\Backend\Teradata\TeradataImportOptions; +use Keboola\Db\ImportExport\Backend\ToStageImporterInterface; +use Keboola\TableBackendUtils\Column\Teradata\TeradataColumn; use Keboola\TableBackendUtils\Escaping\Teradata\TeradataQuote; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableDefinition; class SqlBuilder { + const SRC_ALIAS = 'src'; + public function getCommitTransaction(): string { //TODO @@ -42,13 +47,65 @@ public function getTableExistsCommand(string $dbName, string $tableName): string } public function getInsertAllIntoTargetTableCommand( - TeradataTableDefinition $stagingTableDefinition, + TeradataTableDefinition $sourceTableDefinition, TeradataTableDefinition $destinationTableDefinition, - TeradataImportOptions $options, - string $getNowFormatted + TeradataImportOptions $importOptions, + string $timestamp ): string { - //TODO - throw new \Exception('not implemented yet'); + $destinationTable = sprintf( + '%s.%s', + TeradataQuote::quoteSingleIdentifier($destinationTableDefinition->getSchemaName()), + TeradataQuote::quoteSingleIdentifier($destinationTableDefinition->getTableName()) + ); + + $columnsToInsert = $sourceTableDefinition->getColumnsNames(); + $useTimestamp = !in_array(ToStageImporterInterface::TIMESTAMP_COLUMN_NAME, $columnsToInsert, true) + && $importOptions->useTimestamp(); + + if ($useTimestamp) { + $columnsToInsert = array_merge( + $sourceTableDefinition->getColumnsNames(), + [ToStageImporterInterface::TIMESTAMP_COLUMN_NAME] + ); + } + + $columnsSetSql = []; + + /** @var TeradataColumn $columnDefinition */ + foreach ($sourceTableDefinition->getColumnsDefinitions() as $columnDefinition) { + if (in_array($columnDefinition->getColumnName(), $importOptions->getConvertEmptyValuesToNull(), true)) { + // use nullif only for string base type + if ($columnDefinition->getColumnDefinition()->getBasetype() === BaseType::STRING) { + $columnsSetSql[] = sprintf( + 'NULLIF(%s, \'\')', + TeradataQuote::quoteSingleIdentifier($columnDefinition->getColumnName()) + ); + } else { + $columnsSetSql[] = TeradataQuote::quoteSingleIdentifier($columnDefinition->getColumnName()); + } + } else { + $columnsSetSql[] = sprintf( + 'CAST(COALESCE(%s, \'\') AS %s) AS %s', + TeradataQuote::quoteSingleIdentifier($columnDefinition->getColumnName()), + $columnDefinition->getColumnDefinition()->getSQLDefinition(), + TeradataQuote::quoteSingleIdentifier($columnDefinition->getColumnName()) + ); + } + } + + if ($useTimestamp) { + $columnsSetSql[] = TeradataQuote::quote($timestamp); + } + + return sprintf( + 'INSERT INTO %s (%s) SELECT %s FROM %s.%s AS %s', + $destinationTable, + $this->getColumnsString($columnsToInsert), + implode(',', $columnsSetSql), + TeradataQuote::quoteSingleIdentifier($sourceTableDefinition->getSchemaName()), + TeradataQuote::quoteSingleIdentifier($sourceTableDefinition->getTableName()), + TeradataQuote::quoteSingleIdentifier(self::SRC_ALIAS) + ); } public function getTruncateTableWithDeleteCommand( @@ -61,4 +118,20 @@ public function getTruncateTableWithDeleteCommand( TeradataQuote::quoteSingleIdentifier($tableName) ); } + + /** + * @param string[] $columns + */ + public function getColumnsString( + array $columns, + string $delimiter = ', ', + ?string $tableAlias = null + ): string { + return implode($delimiter, array_map(static function ($columns) use ( + $tableAlias + ) { + $alias = $tableAlias === null ? '' : $tableAlias . '.'; + return $alias . TeradataQuote::quoteSingleIdentifier($columns); + }, $columns)); + } } diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index f63b552e..79058b74 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -30,7 +30,7 @@ public function testLoadToFinalTableWithoutDedup(): void $this->initTable(self::TABLE_COLUMN_NAME_ROW_NUMBER); // skipping header - $options = $this->getImportOptions(); + $options = $this->getImportOptions([], false, false, 1); $source = $this->createS3SourceInstance( self::TABLE_COLUMN_NAME_ROW_NUMBER . '.csv', [ From 278817bc16c05ec8a556219c9e776bbd647aac71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 11 Mar 2022 18:25:34 +0100 Subject: [PATCH 16/62] test with string-like data --- src/Backend/Teradata/TeradataException.php | 1 + tests/functional/Teradata/FullImportTest.php | 17 +++++++++++------ .../Teradata/TeradataBaseTestCase.php | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/Backend/Teradata/TeradataException.php b/src/Backend/Teradata/TeradataException.php index 23557f9b..9057c5e4 100644 --- a/src/Backend/Teradata/TeradataException.php +++ b/src/Backend/Teradata/TeradataException.php @@ -11,6 +11,7 @@ class TeradataException extends Exception public static function covertException(\Doctrine\DBAL\Exception $e): \Throwable { + // TODO return $e; } } diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index 79058b74..efde2dac 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -27,15 +27,18 @@ protected function setUp(): void public function testLoadToFinalTableWithoutDedup(): void { - $this->initTable(self::TABLE_COLUMN_NAME_ROW_NUMBER); + // table translations checks numeric and string-ish data + $this->initTable(self::TABLE_TRANSLATIONS); // skipping header $options = $this->getImportOptions([], false, false, 1); $source = $this->createS3SourceInstance( - self::TABLE_COLUMN_NAME_ROW_NUMBER . '.csv', + self::TABLE_TRANSLATIONS . '.csv', [ 'id', - 'row_number', + 'name', + 'price', + 'isDeleted', ], false, false, @@ -46,12 +49,14 @@ public function testLoadToFinalTableWithoutDedup(): void $destinationRef = new TeradataTableReflection( $this->connection, $this->getDestinationDbName(), - self::TABLE_COLUMN_NAME_ROW_NUMBER + self::TABLE_TRANSLATIONS ); $destination = $destinationRef->getTableDefinition(); $stagingTable = StageTableDefinitionFactory::createStagingTableDefinition($destination, [ 'id', - 'row_number', + 'name', + 'price', + 'isDeleted', ]); $qb = new TeradataTableQueryBuilder(); $this->connection->executeStatement( @@ -70,7 +75,7 @@ public function testLoadToFinalTableWithoutDedup(): void $importState ); - self::assertEquals(2, $destinationRef->getRowsCount()); + self::assertEquals(3, $destinationRef->getRowsCount()); } } diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index 8b80926e..e12aa4f3 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -41,6 +41,7 @@ class TeradataBaseTestCase extends ImportExportBaseTest public const TABLE_OUT_NO_TIMESTAMP_TABLE = 'out_no_timestamp_table'; public const TABLE_TABLE = 'table'; public const TABLE_TYPES = 'types'; + public const TABLE_TRANSLATIONS = 'transactions'; protected Connection $connection; @@ -233,6 +234,19 @@ protected function initTable(string $tableName): void TeradataQuote::quoteSingleIdentifier($tableName) )); break; + case self::TABLE_TRANSLATIONS: + $this->connection->executeQuery(sprintf( + 'CREATE MULTISET TABLE %s.%s, NO FALLBACK + ( + "id" INT , + "name" VARCHAR(4000) , + "price" INT , + "isDeleted" INT + )', + TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), + TeradataQuote::quoteSingleIdentifier($tableName) + )); + break; } } From df0bc8d22173f5167da870ff64cb06dc7e59389b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Mon, 14 Mar 2022 15:02:22 +0100 Subject: [PATCH 17/62] init sqlbuilder test --- .../Teradata/ToFinalTable/FullImporter.php | 4 - .../Teradata/ToFinalTable/SqlBuilder.php | 5 + tests/functional/Teradata/SqlBuilderTest.php | 552 ++++++++++++++++++ .../Teradata/TeradataBaseTestCase.php | 22 +- 4 files changed, 577 insertions(+), 6 deletions(-) create mode 100644 tests/functional/Teradata/SqlBuilderTest.php diff --git a/src/Backend/Teradata/ToFinalTable/FullImporter.php b/src/Backend/Teradata/ToFinalTable/FullImporter.php index 4af2f87c..75057ffd 100644 --- a/src/Backend/Teradata/ToFinalTable/FullImporter.php +++ b/src/Backend/Teradata/ToFinalTable/FullImporter.php @@ -62,10 +62,6 @@ private function doLoadFullWithoutDedup( $sql ); $state->stopTimer(self::TIMER_COPY_TO_TARGET); - -// $this->connection->executeStatement( -// $this->sqlBuilder->getCommitTransaction() -// ); } public function importToTable( diff --git a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php index 2d32abdc..0b5995bf 100644 --- a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php +++ b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php @@ -134,4 +134,9 @@ public function getColumnsString( return $alias . TeradataQuote::quoteSingleIdentifier($columns); }, $columns)); } + + public function getDeleteOldItemsCommand() + { + throw new \Exception('not implemented yet'); + } } diff --git a/tests/functional/Teradata/SqlBuilderTest.php b/tests/functional/Teradata/SqlBuilderTest.php new file mode 100644 index 00000000..1771055f --- /dev/null +++ b/tests/functional/Teradata/SqlBuilderTest.php @@ -0,0 +1,552 @@ +cleanDatabase(self::TEST_DB); + } + + protected function getBuilder(): SqlBuilder + { + return new SqlBuilder(); + } + + protected function setUp(): void + { + parent::setUp(); + $this->dropTestDb(); + } + + protected function createTestDb(): void + { + $this->createDatabase(self::TEST_DB); + } + + public function testGetDedupCommand(): void + { + $this->markTestSkipped('not impl'); + $this->createTestDb(); + $stageDef = $this->createStagingTableWithData(); + + $deduplicationDef = new TeradataTableDefinition( + self::TEST_DB, + 'tempTable', + true, + new ColumnCollection([ + TeradataColumn::createGenericColumn('col1'), + TeradataColumn::createGenericColumn('col2'), + ]), + [ + 'pk1', + 'pk2', + ] + ); + $qb = new TeradataTableQueryBuilder(); + $this->connection->executeStatement($qb->getCreateTableCommandFromDefinition($deduplicationDef)); + + $sql = $this->getBuilder()->getDedupCommand( + $stageDef, + $deduplicationDef, + $deduplicationDef->getPrimaryKeysNames() + ); + self::assertEquals( + // phpcs:ignore + 'INSERT INTO "import-export-test_schema"."tempTable" ("col1", "col2") SELECT a."col1",a."col2" FROM (SELECT "col1", "col2", ROW_NUMBER() OVER (PARTITION BY "pk1","pk2" ORDER BY "pk1","pk2") AS "_row_number_" FROM "import-export-test_schema"."stagingTable") AS a WHERE a."_row_number_" = 1', + $sql + ); + $this->connection->executeStatement($sql); + $result = $this->connection->fetchAllAssociative(sprintf( + 'SELECT * FROM %s.%s', + TeradataQuote::quoteSingleIdentifier(self::TEST_DB), + TeradataQuote::quoteSingleIdentifier($deduplicationDef->getTableName()) + )); + + self::assertCount(2, $result); + } + + private function createStagingTableWithData(bool $includeEmptyValues = false): TeradataTableDefinition + { + $def = $this->getStagingTableDefinition(); + $qb = new TeradataTableQueryBuilder(); + $this->connection->executeStatement($qb->getCreateTableCommandFromDefinition($def)); + + $this->connection->executeStatement( + sprintf( + 'INSERT INTO %s.%s("pk1","pk2","col1","col2") VALUES (1,1,\'1\',\'1\')', + self::TEST_DB_QUOTED, + self::TEST_STAGING_TABLE_QUOTED + ) + ); + $this->connection->executeStatement( + sprintf( + 'INSERT INTO %s.%s("pk1","pk2","col1","col2") VALUES (1,1,\'1\',\'1\')', + self::TEST_DB_QUOTED, + self::TEST_STAGING_TABLE_QUOTED + ) + ); + $this->connection->executeStatement( + sprintf( + 'INSERT INTO %s.%s("pk1","pk2","col1","col2") VALUES (2,2,\'2\',\'2\')', + self::TEST_DB_QUOTED, + self::TEST_STAGING_TABLE_QUOTED + ) + ); + + if ($includeEmptyValues) { + $this->connection->executeStatement( + sprintf( + 'INSERT INTO %s.%s("pk1","pk2","col1","col2") VALUES (2,2,\'\',NULL)', + self::TEST_DB_QUOTED, + self::TEST_STAGING_TABLE_QUOTED + ) + ); + } + + return $def; + } + + private function getDummyImportOptions(): TeradataImportOptions + { + return new TeradataImportOptions(); + } + + public function testGetDeleteOldItemsCommand(): void + { + $this->createTestDb(); + + $tableDefinition = new TeradataTableDefinition( + self::TEST_DB, + self::TEST_TABLE, + false, + new ColumnCollection([ + new TeradataColumn( + 'id', + new Teradata( + Teradata::TYPE_INT + ) + ), + TeradataColumn::createGenericColumn('pk1'), + TeradataColumn::createGenericColumn('pk2'), + TeradataColumn::createGenericColumn('col1'), + TeradataColumn::createGenericColumn('col2'), + ]), + ['pk1', 'pk2'] + ); + $tableSql = sprintf( + '%s.%s', + TeradataQuote::quoteSingleIdentifier(self::TEST_DB), + TeradataQuote::quoteSingleIdentifier($tableDefinition->getTableName()) + ); + $qb = new TeradataTableQueryBuilder(); + $this->connection->executeStatement($qb->getCreateTableCommandFromDefinition($tableDefinition)); + $this->connection->executeStatement( + sprintf( + 'INSERT INTO %s("id","pk1","pk2","col1","col2") VALUES (1,1,1,\'1\',\'1\')', + $tableSql + ) + ); + $stagingTableDefinition = new TeradataTableDefinition( + self::TEST_DB, + self::TEST_STAGING_TABLE, + false, + new ColumnCollection([ + TeradataColumn::createGenericColumn('pk1'), + TeradataColumn::createGenericColumn('pk2'), + TeradataColumn::createGenericColumn('col1'), + TeradataColumn::createGenericColumn('col2'), + ]), + ['pk1', 'pk2'] + ); + $this->connection->executeStatement($qb->getCreateTableCommandFromDefinition($stagingTableDefinition)); + $stagingTableSql = sprintf( + '%s.%s', + // TODO where the getDbName came from + TeradataQuote::quoteSingleIdentifier(self::TEST_DB), + TeradataQuote::quoteSingleIdentifier($stagingTableDefinition->getTableName()) + ); + $this->connection->executeStatement( + sprintf( + 'INSERT INTO %s("pk1","pk2","col1","col2") VALUES (1,1,\'1\',\'1\')', + $stagingTableSql + ) + ); + $this->connection->executeStatement( + sprintf( + 'INSERT INTO %s("pk1","pk2","col1","col2") VALUES (2,1,\'1\',\'1\')', + $stagingTableSql + ) + ); + + $sql = $this->getBuilder()->getDeleteOldItemsCommand( + $stagingTableDefinition, + $tableDefinition + ); + + self::assertEquals( + // phpcs:ignore + 'DELETE FROM "import-export-test_schema"."stagingTable" WHERE EXISTS (SELECT * FROM "import-export-test_schema"."import-export-test_test" WHERE COALESCE("import-export-test_schema"."import-export-test_test"."pk1", \'KBC_$#\') = COALESCE("import-export-test_schema"."stagingTable"."pk1", \'KBC_$#\') AND COALESCE("import-export-test_schema"."import-export-test_test"."pk2", \'KBC_$#\') = COALESCE("import-export-test_schema"."stagingTable"."pk2", \'KBC_$#\'))', + $sql + ); + $this->connection->executeStatement($sql); + + $result = $this->connection->fetchAllAssociative(sprintf( + 'SELECT * FROM %s', + $stagingTableSql + )); + + self::assertCount(1, $result); + self::assertSame([ + [ + 'pk1' => '2', + 'pk2' => '1', + 'col1' => '1', + 'col2' => '1', + ], + ], $result); + } + + private function assertTableNotExists(string $schemaName, string $tableName): void + { + try { + (new TeradataTableReflection($this->connection, $schemaName, $tableName))->getTableStats(); + self::fail(sprintf( + 'Table "%s.%s" is expected to not exist.', + $schemaName, + $tableName + )); + } catch (Exception $e) { + } + } + + public function testGetDropTableIfExistsCommand(): void + { + $this->createTestDb(); + $this->assertTableNotExists(self::TEST_DB, self::TEST_TABLE); + + // check that it cannot find non-existing table + $sql = $this->getBuilder()->getTableExistsCommand(self::TEST_DB, self::TEST_TABLE); + self::assertEquals( + // phpcs:ignore + "SELECT COUNT(*) FROM DBC.Tables WHERE DatabaseName = 'import-export-test_schema' AND TableName = 'import-export-test_test';", $sql + ); + $this->assertEquals(0, $this->connection->fetchOne($sql)); + + // try to drop not existing table + try { + $sql = $this->getBuilder()->getDropTableUnsafe(self::TEST_DB, self::TEST_TABLE); + self::assertEquals( + // phpcs:ignore + 'DROP TABLE "import-export-test_schema"."import-export-test_test"', + $sql + ); + $this->connection->executeStatement($sql); + } catch (DriverException $e) { + $this->assertContains('Base table or view not found', $e->getMessage()); + } + + // create table + $this->initSingleTable(self::TEST_DB, self::TEST_TABLE); + + // check that the table exists already + $sql = $this->getBuilder()->getTableExistsCommand(self::TEST_DB, self::TEST_TABLE); + $this->assertEquals(1, $this->connection->fetchOne($sql)); + + // drop existing table + $sql = $this->getBuilder()->getDropTableUnsafe(self::TEST_DB, self::TEST_TABLE); + $this->connection->executeStatement($sql); + + // check that the table doesn't exist anymore + $sql = $this->getBuilder()->getTableExistsCommand(self::TEST_DB, self::TEST_TABLE); + $this->assertEquals(0, $this->connection->fetchOne($sql)); + } + + public function testGetInsertAllIntoTargetTableCommand(): void + { + $this->createTestDb(); + $destination = $this->createTestTableWithColumns(); + $this->createStagingTableWithData(true); + + // create fake stage and say that there is less columns + $fakeStage = new TeradataTableDefinition( + self::TEST_DB, + self::TEST_STAGING_TABLE, + true, + new ColumnCollection([ + $this->createNullableGenericColumn('col1'), + $this->createNullableGenericColumn('col2'), + ]), + [] + ); + + // no convert values no timestamp + $sql = $this->getBuilder()->getInsertAllIntoTargetTableCommand( + $fakeStage, + $destination, + $this->getImportOptions(), + '2020-01-01 00:00:00' + ); + + self::assertEquals( + // phpcs:ignore + 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2") SELECT CAST(COALESCE("col1", \'\') AS VARCHAR (4000)) AS "col1",CAST(COALESCE("col2", \'\') AS VARCHAR (4000)) AS "col2" FROM "import-export-test_schema"."stagingTable" AS "src"', + $sql + ); + + $out = $this->connection->executeStatement($sql); + self::assertEquals(4, $out); + + + $result = $this->connection->fetchAllAssociative(sprintf( + 'SELECT * FROM %s.%s', + TeradataQuote::quoteSingleIdentifier(self::TEST_DB), + TeradataQuote::quoteSingleIdentifier(self::TEST_TABLE), + )); + + self::assertEqualsCanonicalizing([ + [ + 'id' => null, + 'col1' => '1', + 'col2' => '1', + ], + [ + 'id' => null, + 'col1' => '1', + 'col2' => '1', + ], + [ + 'id' => null, + 'col1' => '2', + 'col2' => '2', + ], + [ + 'id' => null, + 'col1' => '', + 'col2' => '', + ], + ], $result); + } + + protected function createTestTableWithColumns( + bool $includeTimestamp = false, + bool $includePrimaryKey = false + ): TeradataTableDefinition { + $columns = []; + $pks = []; + if ($includePrimaryKey) { + $pks[] = 'id'; + $columns[] = new TeradataColumn( + 'id', + new Teradata(Teradata::TYPE_INT) + ); + } else { + $columns[] = $this->createNullableGenericColumn('id'); + } + $columns[] = $this->createNullableGenericColumn('col1'); + $columns[] = $this->createNullableGenericColumn('col2'); + + if ($includeTimestamp) { + $columns[] = new TeradataColumn( + '_timestamp', + new Teradata(Teradata::TYPE_TIMESTAMP) + ); + } + + $tableDefinition = new TeradataTableDefinition( + self::TEST_DB, + self::TEST_TABLE, + false, + new ColumnCollection($columns), + $pks + ); + $this->connection->executeStatement( + (new TeradataTableQueryBuilder())->getCreateTableCommandFromDefinition($tableDefinition) + ); + + return $tableDefinition; + } + + private function createNullableGenericColumn(string $columnName): TeradataColumn + { + $definition = new Teradata( + Teradata::TYPE_VARCHAR, + [ + 'length' => '4000', // should be changed to max in future + 'nullable' => true, + ] + ); + + return new TeradataColumn( + $columnName, + $definition + ); + } + + public function testGetInsertAllIntoTargetTableCommandConvertToNull(): void + { + $this->createTestDb(); + $destination = $this->createTestTableWithColumns(); + + $this->createStagingTableWithData(true); + // create fake stage and say that there is less columns + $fakeStage = new TeradataTableDefinition( + self::TEST_DB, + self::TEST_STAGING_TABLE, + true, + new ColumnCollection([ + $this->createNullableGenericColumn('col1'), + $this->createNullableGenericColumn('col2'), + ]), + [] + ); + + // convert col1 to null + $options = $this->getImportOptions(['col1']); + $sql = $this->getBuilder()->getInsertAllIntoTargetTableCommand( + $fakeStage, + $destination, + $options, + '2020-01-01 00:00:00' + ); + self::assertEquals( + // phpcs:ignore + 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2") SELECT NULLIF("col1", \'\'),CAST(COALESCE("col2", \'\') AS VARCHAR (4000)) AS "col2" FROM "import-export-test_schema"."stagingTable" AS "src"', + $sql + ); + $out = $this->connection->executeStatement($sql); + self::assertEquals(4, $out); + + $result = $this->connection->fetchAllAssociative(sprintf( + 'SELECT * FROM %s', + self::TEST_TABLE_IN_DB + )); + + self::assertEqualsCanonicalizing([ + [ + 'id' => null, + 'col1' => '1', + 'col2' => '1', + ], + [ + 'id' => null, + 'col1' => '1', + 'col2' => '1', + ], + [ + 'id' => null, + 'col1' => '2', + 'col2' => '2', + ], + [ + 'id' => null, + 'col1' => null, + 'col2' => '', + ], + ], $result); + } + + public function testGetInsertAllIntoTargetTableCommandConvertToNullWithTimestamp(): void + { + $this->createTestDb(); + $destination = $this->createTestTableWithColumns(true); + $this->createStagingTableWithData(true); + // create fake stage and say that there is less columns + $fakeStage = new TeradataTableDefinition( + self::TEST_DB, + self::TEST_STAGING_TABLE, + true, + new ColumnCollection([ + $this->createNullableGenericColumn('col1'), + $this->createNullableGenericColumn('col2'), + ]), + [] + ); + + // use timestamp + $options = $this->getImportOptions(['col1'], false, true); + $sql = $this->getBuilder()->getInsertAllIntoTargetTableCommand( + $fakeStage, + $destination, + $options, + '2020-01-01 00:00:00' + ); + self::assertEquals( + // phpcs:ignore + 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2", "_timestamp") SELECT NULLIF("col1", \'\'),CAST(COALESCE("col2", \'\') AS VARCHAR (4000)) AS "col2",\'2020-01-01 00:00:00\' FROM "import-export-test_schema"."stagingTable" AS "src"', + $sql + ); + $out = $this->connection->executeStatement($sql); + self::assertEquals(4, $out); + + $result = $this->connection->fetchAllAssociative(sprintf( + 'SELECT * FROM %s', + self::TEST_TABLE_IN_DB + )); + + foreach ($result as $item) { + self::assertArrayHasKey('id', $item); + self::assertArrayHasKey('col1', $item); + self::assertArrayHasKey('col2', $item); + self::assertArrayHasKey('_timestamp', $item); + } + } + + public function testGetTruncateTableWithDeleteCommand(): void + { + $this->createTestDb(); + $this->createStagingTableWithData(); + + $ref = new TeradataTableReflection($this->connection, self::TEST_DB, self::TEST_STAGING_TABLE); + self::assertEquals(3, $ref->getRowsCount()); + + $sql = $this->getBuilder()->getTruncateTableWithDeleteCommand(self::TEST_DB, self::TEST_STAGING_TABLE); + self::assertEquals( + 'DELETE FROM "import-export-test_schema"."stagingTable"', + $sql + ); + $this->connection->executeStatement($sql); + self::assertEquals(0, $ref->getRowsCount()); + } + + private function getStagingTableDefinition(): TeradataTableDefinition + { + return new TeradataTableDefinition( + self::TEST_DB, + self::TEST_STAGING_TABLE, + true, + new ColumnCollection([ + $this->createNullableGenericColumn('pk1'), + $this->createNullableGenericColumn('pk2'), + $this->createNullableGenericColumn('col1'), + $this->createNullableGenericColumn('col2'), + ]), + [] + ); + } +} diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index e12aa4f3..7957cafe 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -26,8 +26,6 @@ class TeradataBaseTestCase extends ImportExportBaseTest protected const TERADATA_SOURCE_DATABASE_NAME = 'tests_source'; protected const TERADATA_DESTINATION_DATABASE_NAME = 'tests_source'; // TODO move somewhere else - protected const EXASOL_DEST_SCHEMA_NAME = 'in_c-tests'; - protected const EXASOL_SOURCE_SCHEMA_NAME = 'some_tests'; public const TABLE_ACCOUNTS_3 = 'accounts-3'; public const TABLE_ACCOUNTS_BEZ_TS = 'accounts-bez-ts'; public const TABLE_COLUMN_NAME_ROW_NUMBER = 'column-name-row-number'; @@ -202,6 +200,26 @@ protected function assertSynapseTableEqualsExpected( ); } + protected function initSingleTable( + string $db = self::TERADATA_SOURCE_DATABASE_NAME, + string $table = self::TABLE_TABLE + ): void { + if (!$this->dbExists($db)) { + $this->createDatabase($db); + } + // char because of Stats test + $this->connection->executeQuery( + sprintf( + 'CREATE MULTISET TABLE %s.%s, NO FALLBACK + ( +"Other" VARCHAR(4000) + );', + TeradataQuote::quoteSingleIdentifier($db), + TeradataQuote::quoteSingleIdentifier($table) + ) + ); + } + protected function initTable(string $tableName): void { switch ($tableName) { From 4215cf5436213ffcfdbbaf1e65038f7f286106dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Mon, 14 Mar 2022 15:59:09 +0100 Subject: [PATCH 18/62] init tables for fullload test --- .../Teradata/ToFinalTable/FullImporter.php | 2 +- src/Storage/Teradata/SelectSource.php | 89 ++++ src/Storage/Teradata/Table.php | 96 +++++ tests/functional/Teradata/FullImportTest.php | 397 ++++++++++++++++++ .../Teradata/TeradataBaseTestCase.php | 143 ++++++- 5 files changed, 718 insertions(+), 9 deletions(-) create mode 100644 src/Storage/Teradata/SelectSource.php create mode 100644 src/Storage/Teradata/Table.php diff --git a/src/Backend/Teradata/ToFinalTable/FullImporter.php b/src/Backend/Teradata/ToFinalTable/FullImporter.php index 75057ffd..fdcad35b 100644 --- a/src/Backend/Teradata/ToFinalTable/FullImporter.php +++ b/src/Backend/Teradata/ToFinalTable/FullImporter.php @@ -41,7 +41,7 @@ private function doLoadFullWithoutDedup( TeradataTableDefinition $destinationTableDefinition, TeradataImportOptions $options, ImportState $state - ) { + ): void { // truncate destination table $this->connection->executeStatement( $this->sqlBuilder->getTruncateTableWithDeleteCommand( diff --git a/src/Storage/Teradata/SelectSource.php b/src/Storage/Teradata/SelectSource.php new file mode 100644 index 00000000..6bcf22cb --- /dev/null +++ b/src/Storage/Teradata/SelectSource.php @@ -0,0 +1,89 @@ +query = $query; + $this->queryBindings = $queryBindings; + $this->dataTypes = $dataTypes; + $this->columnsNames = $columnsNames; + $this->primaryKeysNames = $primaryKeysNames; + } + + /** + * @return string[] + */ + public function getColumnsNames(): array + { + return $this->columnsNames; + } + + /** + * @return string[] + */ + public function getDataTypes(): array + { + return $this->dataTypes; + } + + public function getFromStatement(): string + { + return sprintf('%s', $this->getQuery()); + } + + public function getQuery(): string + { + return $this->query; + } + + /** + * @return string[]|null + */ + public function getPrimaryKeysNames(): ?array + { + return $this->primaryKeysNames; + } + + /** + * @return array|string[] + */ + public function getQueryBindings(): array + { + return $this->queryBindings; + } +} diff --git a/src/Storage/Teradata/Table.php b/src/Storage/Teradata/Table.php new file mode 100644 index 00000000..6182d59e --- /dev/null +++ b/src/Storage/Teradata/Table.php @@ -0,0 +1,96 @@ +schema = $schema; + $this->tableName = $tableName; + $this->columnsNames = $columns; + $this->primaryKeysNames = $primaryKeysNames; + } + + public function getFromStatement(): string + { + $select = '*'; + $colums = $this->getColumnsNames(); + if ($colums !== []) { + $quotedColumns = array_map(static function ($column) { + return ExasolQuote::quoteSingleIdentifier($column); + }, $colums); + $select = implode(', ', $quotedColumns); + } + + return sprintf('SELECT %s FROM %s', $select, $this->getQuotedTableWithScheme()); + } + + /** + * @return string[] + */ + public function getColumnsNames(): array + { + return $this->columnsNames; + } + + public function getQuotedTableWithScheme(): string + { + return sprintf( + '%s.%s', + ExasolQuote::quoteSingleIdentifier($this->getSchema()), + ExasolQuote::quoteSingleIdentifier($this->getTableName()) + ); + } + + public function getPrimaryKeysNames(): ?array + { + return $this->primaryKeysNames; + } + + /** + * @return string[] + */ + public function getQueryBindings(): array + { + // TODO + return []; + } + + public function getSchema(): string + { + return $this->schema; + } + + public function getTableName(): string + { + return $this->tableName; + } +} diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index efde2dac..dbce8527 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -4,9 +4,18 @@ namespace Tests\Keboola\Db\ImportExportFunctional\Teradata; +use Generator; +use Keboola\Csv\CsvFile; +use Keboola\CsvOptions\CsvOptions; +use Keboola\Db\ImportExport\Backend\Teradata\TeradataImportOptions; +use Keboola\Db\ImportExport\Backend\Teradata\ToFinalTable\SqlBuilder; use Keboola\Db\ImportExport\Backend\Teradata\ToFinalTable\FullImporter; use Keboola\Db\ImportExport\Backend\Teradata\ToStage\StageTableDefinitionFactory; use Keboola\Db\ImportExport\Backend\Teradata\ToStage\ToStageImporter; +use Keboola\Db\ImportExport\ImportOptions; +use Keboola\Db\ImportExport\Storage\SourceInterface; +use Keboola\Db\ImportExport\Storage\Teradata\Table; +use Keboola\TableBackendUtils\Table\Teradata\TeradataTableDefinition; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableQueryBuilder; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableReflection; use Tests\Keboola\Db\ImportExport\S3SourceTrait; @@ -78,4 +87,392 @@ public function testLoadToFinalTableWithoutDedup(): void self::assertEquals(3, $destinationRef->getRowsCount()); } + /** + * @return Generator> + */ + public function fullImportData(): Generator + { + $expectedEscaping = []; + $file = new CsvFile(self::DATA_DIR . 'escaping/standard-with-enclosures.csv'); + foreach ($file as $row) { + $expectedEscaping[] = $row; + } + $escapingHeader = array_shift($expectedEscaping); // remove header + $expectedEscaping = array_values($expectedEscaping); + + $expectedAccounts = []; + $file = new CsvFile(self::DATA_DIR . 'tw_accounts.csv'); + foreach ($file as $row) { + $expectedAccounts[] = $row; + } + $accountsHeader = array_shift($expectedAccounts); // remove header + $expectedAccounts = array_values($expectedAccounts); + + $file = new CsvFile(self::DATA_DIR . 'tw_accounts.changedColumnsOrder.csv'); + $accountChangedColumnsOrderHeader = $file->getHeader(); + + $file = new CsvFile(self::DATA_DIR . 'lemma.csv'); + $expectedLemma = []; + foreach ($file as $row) { + $expectedLemma[] = $row; + } + $lemmaHeader = array_shift($expectedLemma); + $expectedLemma = array_values($expectedLemma); + + // large sliced manifest + $expectedLargeSlicedManifest = []; + for ($i = 0; $i <= 1500; $i++) { + $expectedLargeSlicedManifest[] = ['a', 'b']; + } + + yield 'large manifest' => [ + $this->createS3SourceInstance( + 'sliced/2cols-large/S3.2cols-large.csvmanifest', + $escapingHeader, + true, + false, + [] + ), + [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], + $this->getImportOptions(), + $expectedLargeSlicedManifest, + 1501, + self::TABLE_OUT_CSV_2COLS, + ]; + + yield 'empty manifest' => [ + $this->createS3SourceInstance( + 'empty.manifest', + $escapingHeader, + true, + false, + [] + ), + [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], + $this->getImportOptions(), + [], + 0, + self::TABLE_OUT_CSV_2COLS, + ]; + + yield 'lemma' => [ + $this->createS3SourceInstance( + 'lemma.csv', + $lemmaHeader, + false, + false, + [] + ), + [$this->getDestinationDbName(), self::TABLE_OUT_LEMMA], + $this->getImportOptions(), + $expectedLemma, + 5, + self::TABLE_OUT_LEMMA, + ]; + + yield 'standard with enclosures' => [ + $this->createS3SourceInstance( + 'standard-with-enclosures.csv', + $escapingHeader, + false, + false, + [] + ), + [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], + $this->getImportOptions(), + $expectedEscaping, + 7, + self::TABLE_OUT_CSV_2COLS, + ]; + + yield 'gzipped standard with enclosure' => [ + $this->createS3SourceInstance( + 'gzipped-standard-with-enclosures.csv.gz', + $escapingHeader, + false, + false, + [] + ), + [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], + $this->getImportOptions(), + $expectedEscaping, + 7, + self::TABLE_OUT_CSV_2COLS, + ]; + + yield 'standard with enclosures tabs' => [ + $this->createS3SourceInstanceFromCsv( + 'standard-with-enclosures.tabs.csv', + new CsvOptions("\t"), + $escapingHeader, + false, + false, + [] + ), + [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], + $this->getImportOptions(), + $expectedEscaping, + 7, + self::TABLE_OUT_CSV_2COLS, + ]; + + yield 'accounts changedColumnsOrder' => [ + $this->createS3SourceInstance( + 'tw_accounts.changedColumnsOrder.csv', + $accountChangedColumnsOrderHeader, + false, + false, + ['id'] + ), + [ + $this->getDestinationDbName(), + self::TABLE_ACCOUNTS_3, + ], + $this->getImportOptions(), + $expectedAccounts, + 3, + self::TABLE_ACCOUNTS_3, + ]; + yield 'accounts' => [ + $this->createS3SourceInstance( + 'tw_accounts.csv', + $accountsHeader, + false, + false, + ['id'] + ), + [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], + $this->getImportOptions(), + $expectedAccounts, + 3, + self::TABLE_ACCOUNTS_3, + ]; + + // line ending detection is not supported yet for S3 + //yield 'accounts crlf' => [ + // $this->createS3SourceInstance( + // 'tw_accounts.crlf.csv', + // $accountsHeader, + // false, + // false, + // ['id'] + // ), + // [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], + // $this->getImportOptions(), + // $expectedAccounts, + // 3, + // self::TABLE_ACCOUNTS_3, + //]; + + // manifests + yield 'accounts sliced' => [ + $this->createS3SourceInstance( + 'sliced/accounts/S3.accounts.csvmanifest', + $accountsHeader, + true, + false, + ['id'] + ), + [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], + $this->getImportOptions(), + $expectedAccounts, + 3, + self::TABLE_ACCOUNTS_3, + ]; + + yield 'accounts sliced gzip' => [ + $this->createS3SourceInstance( + 'sliced/accounts-gzip/S3.accounts-gzip.csvmanifest', + $accountsHeader, + true, + false, + ['id'] + ), + [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], + $this->getImportOptions(), + $expectedAccounts, + 3, + self::TABLE_ACCOUNTS_3, + ]; + + // folder + yield 'accounts sliced folder import' => [ + $this->createS3SourceInstance( + 'sliced_accounts_no_manifest/', + $accountsHeader, + true, + true, + ['id'] + ), + [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], + $this->getImportOptions(), + $expectedAccounts, + 3, + self::TABLE_ACCOUNTS_3, + ]; + + // reserved words + yield 'reserved words' => [ + $this->createS3SourceInstance( + 'reserved-words.csv', + ['column', 'table'], + false, + false, + [] + ), + [$this->getDestinationDbName(), self::TABLE_TABLE], + $this->getImportOptions(), + [['table', 'column', null]], + 1, + self::TABLE_TABLE, + ]; + // import table with _timestamp columns - used by snapshots + yield 'import with _timestamp columns' => [ + $this->createS3SourceInstance( + 'with-ts.csv', + [ + 'col1', + 'col2', + '_timestamp', + ], + false, + false, + [] + ), + [ + $this->getDestinationDbName(), + self::TABLE_OUT_CSV_2COLS, + ], + $this->getImportOptions(), + [ + ['a', 'b', '2014-11-10 13:12:06.000000'], + ['c', 'd', '2014-11-10 14:12:06.000000'], + ], + 2, + self::TABLE_OUT_CSV_2COLS, + ]; + // test creating table without _timestamp column + yield 'table without _timestamp column' => [ + $this->createS3SourceInstance( + 'standard-with-enclosures.csv', + $escapingHeader, + false, + false, + [] + ), + [ + $this->getDestinationDbName(), + self::TABLE_OUT_NO_TIMESTAMP_TABLE, + ], + $this->getImportOptions( + [], + false, + false, // don't use timestamp + ImportOptions::SKIP_FIRST_LINE + ), + $expectedEscaping, + 7, + self::TABLE_OUT_NO_TIMESTAMP_TABLE, + ]; + // copy from table + yield 'copy from table' => [ + new Table($this->getSourceDbName(), self::TABLE_OUT_CSV_2COLS, $escapingHeader), + [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], + $this->getImportOptions(), + [['a', 'b'], ['c', 'd']], + 2, + self::TABLE_OUT_CSV_2COLS, + ]; + yield 'copy from table 2' => [ + new Table( + $this->getSourceDbName(), + self::TABLE_TYPES, + [ + 'charCol', + 'numCol', + 'floatCol', + 'boolCol', + ] + ), + [ + $this->getDestinationDbName(), + self::TABLE_TYPES, + ], + $this->getImportOptions(), + [['a', '10.5', '0.3', '1']], + 1, + self::TABLE_TYPES, + ]; + } + + /** + * @dataProvider fullImportData + * @param array $table + * @param array $expected + * @param string $tablesToInit + */ + public function testFullImportWithDataSet( + SourceInterface $source, + array $table, + TeradataImportOptions $options, + array $expected, + int $expectedImportedRowCount, + string $tablesToInit + ): void { + $this->initTable($tablesToInit); + + [$schemaName, $tableName] = $table; + /** @var TeradataTableDefinition $destination */ + $destination = (new TeradataTableReflection( + $this->connection, + $schemaName, + $tableName + ))->getTableDefinition(); + + $stagingTable = \Keboola\Db\ImportExport\Backend\Teradata\ToStage\StageTableDefinitionFactory::createStagingTableDefinition( + $destination, + $source->getColumnsNames() + ); + $qb = new TeradataTableQueryBuilder(); + $this->connection->executeStatement( + $qb->getCreateTableCommandFromDefinition($stagingTable) + ); + $toStageImporter = new \Keboola\Db\ImportExport\Backend\Teradata\ToStage\ToStageImporter($this->connection); + $toFinalTableImporter = new \Keboola\Db\ImportExport\Backend\Teradata\ToFinalTable\FullImporter($this->connection); + try { + $importState = $toStageImporter->importToStagingTable( + $source, + $stagingTable, + $options + ); + $result = $toFinalTableImporter->importToTable( + $stagingTable, + $destination, + $options, + $importState + ); + } finally { + if ($this->connection->fetchOne( + (new SqlBuilder())->getTableExistsCommand( + $stagingTable->getSchemaName(), + $stagingTable->getTableName() + ) + ) > 0) { + $this->connection->executeStatement((new SqlBuilder())->getDropTableUnsafe( + $stagingTable->getSchemaName(), + $stagingTable->getTableName() + )); + } + } + + self::assertEquals($expectedImportedRowCount, $result->getImportedRowsCount()); + +// $this->assertTeradataTableEqualsExpected( +// $source, +// $destination, +// $options, +// $expected, +// 0 +// ); + } } diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index 7957cafe..728e41c7 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -24,7 +24,7 @@ class TeradataBaseTestCase extends ImportExportBaseTest public const TABLE_GENERIC = self::TESTS_PREFIX . 'refTab'; public const VIEW_GENERIC = self::TESTS_PREFIX . 'refView'; protected const TERADATA_SOURCE_DATABASE_NAME = 'tests_source'; - protected const TERADATA_DESTINATION_DATABASE_NAME = 'tests_source'; + protected const TERADATA_DESTINATION_DATABASE_NAME = 'tests_destination'; // TODO move somewhere else public const TABLE_ACCOUNTS_3 = 'accounts-3'; public const TABLE_ACCOUNTS_BEZ_TS = 'accounts-bez-ts'; @@ -159,7 +159,7 @@ protected function assertSynapseTableEqualsExpected( ): void { $tableColumns = (new TeradataTableReflection( $this->connection, - $destination->getSchemaName(), + $destination->getDbName(), $destination->getTableName() ))->getColumnsNames(); @@ -182,7 +182,7 @@ protected function assertSynapseTableEqualsExpected( $sql = sprintf( 'SELECT %s FROM [%s].[%s]', implode(', ', $tableColumns), - $destination->getSchemaName(), + $destination->getDbName(), $destination->getTableName() ); @@ -228,11 +228,11 @@ protected function initTable(string $tableName): void sprintf( 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( -"VisitID" VARCHAR(2000000) , -"Value" VARCHAR(2000000), -"MenuItem" VARCHAR(2000000), -"Something" VARCHAR(2000000), -"Other" VARCHAR(2000000), +"VisitID" VARCHAR(4000) , +"Value" VARCHAR(4000), +"MenuItem" VARCHAR(4000), +"Something" VARCHAR(4000), +"Other" VARCHAR(4000), ) PRIMARY INDEX ("VisitID"); );', @@ -265,6 +265,133 @@ protected function initTable(string $tableName): void TeradataQuote::quoteSingleIdentifier($tableName) )); break; + case self::TABLE_OUT_CSV_2COLS: + $this->connection->executeQuery( + sprintf( + 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( + "col1" VARCHAR(20000) , + "col2" VARCHAR(20000) , + "_timestamp" TIMESTAMP + );', + TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), + TeradataQuote::quoteSingleIdentifier($tableName) + ) + ); + + $this->connection->executeQuery(sprintf( + 'INSERT INTO %s.%s VALUES (\'x\', \'y\', NOW());', + TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), + TeradataQuote::quoteSingleIdentifier($tableName) + )); + + $this->connection->executeQuery(sprintf( + 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( + "col1" VARCHAR(4000) , + "col2" VARCHAR(4000) + );', + TeradataQuote::quoteSingleIdentifier($this->getSourceDbName()), + TeradataQuote::quoteSingleIdentifier($tableName) + )); + + $this->connection->executeQuery(sprintf( + 'INSERT INTO %s.%s VALUES (\'a\', \'b\');', + TeradataQuote::quoteSingleIdentifier($this->getSourceDbName()), + TeradataQuote::quoteSingleIdentifier($tableName) + )); + + $this->connection->executeQuery(sprintf( + 'INSERT INTO %s.%s VALUES (\'c\', \'d\');', + TeradataQuote::quoteSingleIdentifier($this->getSourceDbName()), + TeradataQuote::quoteSingleIdentifier($tableName) + )); + break; + case self::TABLE_OUT_LEMMA: + $this->connection->executeQuery(sprintf( + 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( + "ts" VARCHAR(4000) , + "lemma" VARCHAR(4000) , + "lemmaIndex" VARCHAR(4000) , + "_timestamp" TIMESTAMP + );', + TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), + TeradataQuote::quoteSingleIdentifier($tableName) + )); + break; + case self::TABLE_ACCOUNTS_3: + $this->connection->executeQuery(sprintf( + 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( + "id" VARCHAR(4000) , + "idTwitter" VARCHAR(4000) , + "name" VARCHAR(4000) , + "import" VARCHAR(4000) , + "isImported" VARCHAR(4000) , + "apiLimitExceededDatetime" VARCHAR(4000) , + "analyzeSentiment" VARCHAR(4000) , + "importKloutScore" VARCHAR(4000) , + "timestamp" VARCHAR(4000) , + "oauthToken" VARCHAR(4000) , + "oauthSecret" VARCHAR(4000) , + "idApp" VARCHAR(4000) , + "_timestamp" TIMESTAMP, + CONSTRAINT PRIMARY KEY ("id") + );', + TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), + TeradataQuote::quoteSingleIdentifier($tableName) + )); + break; + case self::TABLE_TABLE: + $this->connection->executeQuery(sprintf( + 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( + "column" VARCHAR(4000) , + "table" VARCHAR(4000) , + "lemmaIndex" VARCHAR(4000) , + "_timestamp" TIMESTAMP + );', + TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), + TeradataQuote::quoteSingleIdentifier($tableName) + )); + break; + case self::TABLE_OUT_NO_TIMESTAMP_TABLE: + $this->connection->executeQuery(sprintf( + 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( + "col1" VARCHAR(4000) , + "col2" VARCHAR(4000) + );', + TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), + TeradataQuote::quoteSingleIdentifier($tableName) + )); + break; + + case self::TABLE_TYPES: + $this->connection->executeQuery(sprintf( + 'CREATE TABLE %s."types" ( + "charCol" VARCHAR(4000) , + "numCol" VARCHAR(4000) , + "floatCol" VARCHAR(4000) , + "boolCol" VARCHAR(4000) , + "_timestamp" TIMESTAMP + );', + TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()) + )); + + $this->connection->executeQuery(sprintf( + 'CREATE TABLE %s."types" ( + "charCol" VARCHAR(4000) , + "numCol" decimal(10,1) , + "floatCol" float , + "boolCol" tinyint + );', + TeradataQuote::quoteSingleIdentifier($this->getSourceDbName()) + )); + $this->connection->executeQuery(sprintf( + 'INSERT INTO %s."types" VALUES + (\'a\', \'10.5\', \'0.3\', 1) + ;', + TeradataQuote::quoteSingleIdentifier($this->getSourceDbName()) + )); + break; + default: + throw new \Exception('unknown table'); } } From 9ec44df2d2160e472c94c999b9ccd69d4a446fcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Tue, 15 Mar 2022 09:49:37 +0100 Subject: [PATCH 19/62] tmp solution for sliced files --- src/Backend/Teradata/Helper/BackendHelper.php | 44 +++++++++++++++++ .../Teradata/ToStage/FromS3TPTAdapter.php | 7 +-- tests/unit/Backend/Teradata/HelperTest.php | 47 +++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 tests/unit/Backend/Teradata/HelperTest.php diff --git a/src/Backend/Teradata/Helper/BackendHelper.php b/src/Backend/Teradata/Helper/BackendHelper.php index 22bdc23f..a2d69e95 100644 --- a/src/Backend/Teradata/Helper/BackendHelper.php +++ b/src/Backend/Teradata/Helper/BackendHelper.php @@ -4,6 +4,8 @@ namespace Keboola\Db\ImportExport\Backend\Teradata\Helper; +use Keboola\Db\ImportExport\Storage\S3\SourceFile; + final class BackendHelper { public static function generateTempTableName(): string @@ -15,4 +17,46 @@ public static function generateTempDedupTableName(): string { return '__temp_DEDUP_' . str_replace('.', '_', uniqid('csvimport', true)); } + + /** + * creates a wildcard string which should match all files in manifest + * [file01.csv, file01.csv] => file0* + * TODO + * - has to fix edgecases a) [1_file.csv, 2_file.csv] b) not all the files matched in WC have to be on s3 + * @param SourceFile $source + * @return string + * @throws \Keboola\Db\Import\Exception + */ + public static function getMask(SourceFile $source): string + { + $entries = $source->getManifestEntries(); + $toRemove = $source->getS3Prefix() . '/' . $source->getPrefix(); + $entriesAsArrays = []; + $min = 99999; + $minIndex = 0; + foreach ($entries as $i => $entry) { + $entry = str_replace($toRemove, '', $entry); + $asArray = str_split($entry); + $entriesAsArrays[] = $asArray; + $thisSize = count($asArray); + if ($thisSize < $min) { + $min = $thisSize; + $minIndex = $i; + } + } + $out = []; + + foreach ($entriesAsArrays[$minIndex] as $index => $letter) { + $match = true; + + foreach ($entriesAsArrays as $fileName) { + if ($fileName[$index] !== $letter) { + $match = false; + break; + } + } + $out[$index] = $match ? $letter : '*'; + } + return implode('', $out); + } } diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index 40da9320..4fc9086a 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -137,7 +137,6 @@ private function generateTPTScript( $temp = new Temp(); $temp->initRunFolder(); $folder = $temp->getTmpFolder(); - $target = sprintf( "%s.%s", TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()), @@ -145,11 +144,13 @@ private function generateTPTScript( ); if ($source->isSliced()) { + $moduleStr = sprintf( - 'AccessModuleInitStr = \'S3Region=%s S3Bucket=%s S3Prefix="%s/" S3SinglePartFile=False\'', + 'AccessModuleInitStr = \'S3Region=%s S3Bucket=%s S3Prefix="%s" S3Object=%s S3SinglePartFile=True\'', $source->getRegion(), $source->getBucket(), - $source->getPrefix() + $source->getPrefix(), + BackendHelper::getMask($source) ); } else { $moduleStr = sprintf( diff --git a/tests/unit/Backend/Teradata/HelperTest.php b/tests/unit/Backend/Teradata/HelperTest.php new file mode 100644 index 00000000..add7241c --- /dev/null +++ b/tests/unit/Backend/Teradata/HelperTest.php @@ -0,0 +1,47 @@ +assertEquals($expected, BackendHelper::getMask($entries)); + } + + public function dataProvider(): array + { + return [ + [ + 'sliced.csv_*', + [ + 'sliced.csv_1001', + 'sliced.csv_0', + 'sliced.csv_1', + 'sliced.csv_10', + 'sliced.csv_100', + 'sliced.csv_1002', + ], + ], + [ + 'sliced0****', + [ + 'sliced0122.csv', + 'sliced012.csv', + 'sliced01.csv', + 'sliced01999.csv', + 'sliced0.csv', + 'sliced034.csv', + ], + ], + ]; + } +} From d9996e640e167e519f1c030c0cc147f4d263b6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Tue, 15 Mar 2022 10:27:32 +0100 Subject: [PATCH 20/62] allow to create TABLE_ACCOUNTS_3 --- tests/functional/Teradata/TeradataBaseTestCase.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index 728e41c7..e763aecc 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -332,9 +332,8 @@ protected function initTable(string $tableName): void "oauthToken" VARCHAR(4000) , "oauthSecret" VARCHAR(4000) , "idApp" VARCHAR(4000) , - "_timestamp" TIMESTAMP, - CONSTRAINT PRIMARY KEY ("id") - );', + "_timestamp" TIMESTAMP + ) PRIMARY INDEX ("id");', TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), TeradataQuote::quoteSingleIdentifier($tableName) )); From a29eb60e11a1686a52c3644e4b31463676657fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Tue, 15 Mar 2022 11:13:18 +0100 Subject: [PATCH 21/62] first fullload tests --- tests/functional/Teradata/FullImportTest.php | 478 +++++++++--------- .../Teradata/TeradataBaseTestCase.php | 16 + 2 files changed, 254 insertions(+), 240 deletions(-) diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index dbce8527..431b4898 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -6,7 +6,6 @@ use Generator; use Keboola\Csv\CsvFile; -use Keboola\CsvOptions\CsvOptions; use Keboola\Db\ImportExport\Backend\Teradata\TeradataImportOptions; use Keboola\Db\ImportExport\Backend\Teradata\ToFinalTable\SqlBuilder; use Keboola\Db\ImportExport\Backend\Teradata\ToFinalTable\FullImporter; @@ -14,7 +13,6 @@ use Keboola\Db\ImportExport\Backend\Teradata\ToStage\ToStageImporter; use Keboola\Db\ImportExport\ImportOptions; use Keboola\Db\ImportExport\Storage\SourceInterface; -use Keboola\Db\ImportExport\Storage\Teradata\Table; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableDefinition; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableQueryBuilder; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableReflection; @@ -139,100 +137,100 @@ public function fullImportData(): Generator 1501, self::TABLE_OUT_CSV_2COLS, ]; - - yield 'empty manifest' => [ - $this->createS3SourceInstance( - 'empty.manifest', - $escapingHeader, - true, - false, - [] - ), - [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], - $this->getImportOptions(), - [], - 0, - self::TABLE_OUT_CSV_2COLS, - ]; - - yield 'lemma' => [ - $this->createS3SourceInstance( - 'lemma.csv', - $lemmaHeader, - false, - false, - [] - ), - [$this->getDestinationDbName(), self::TABLE_OUT_LEMMA], - $this->getImportOptions(), - $expectedLemma, - 5, - self::TABLE_OUT_LEMMA, - ]; - - yield 'standard with enclosures' => [ - $this->createS3SourceInstance( - 'standard-with-enclosures.csv', - $escapingHeader, - false, - false, - [] - ), - [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], - $this->getImportOptions(), - $expectedEscaping, - 7, - self::TABLE_OUT_CSV_2COLS, - ]; - - yield 'gzipped standard with enclosure' => [ - $this->createS3SourceInstance( - 'gzipped-standard-with-enclosures.csv.gz', - $escapingHeader, - false, - false, - [] - ), - [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], - $this->getImportOptions(), - $expectedEscaping, - 7, - self::TABLE_OUT_CSV_2COLS, - ]; - - yield 'standard with enclosures tabs' => [ - $this->createS3SourceInstanceFromCsv( - 'standard-with-enclosures.tabs.csv', - new CsvOptions("\t"), - $escapingHeader, - false, - false, - [] - ), - [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], - $this->getImportOptions(), - $expectedEscaping, - 7, - self::TABLE_OUT_CSV_2COLS, - ]; - - yield 'accounts changedColumnsOrder' => [ - $this->createS3SourceInstance( - 'tw_accounts.changedColumnsOrder.csv', - $accountChangedColumnsOrderHeader, - false, - false, - ['id'] - ), - [ - $this->getDestinationDbName(), - self::TABLE_ACCOUNTS_3, - ], - $this->getImportOptions(), - $expectedAccounts, - 3, - self::TABLE_ACCOUNTS_3, - ]; +// +// yield 'empty manifest' => [ +// $this->createS3SourceInstance( +// 'empty.manifest', +// $escapingHeader, +// true, +// false, +// [] +// ), +// [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], +// $this->getImportOptions(), +// [], +// 0, +// self::TABLE_OUT_CSV_2COLS, +// ]; +// +// yield 'lemma' => [ +// $this->createS3SourceInstance( +// 'lemma.csv', +// $lemmaHeader, +// false, +// false, +// [] +// ), +// [$this->getDestinationDbName(), self::TABLE_OUT_LEMMA], +// $this->getImportOptions(), +// $expectedLemma, +// 5, +// self::TABLE_OUT_LEMMA, +// ]; +// +// yield 'standard with enclosures' => [ +// $this->createS3SourceInstance( +// 'standard-with-enclosures.csv', +// $escapingHeader, +// false, +// false, +// [] +// ), +// [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], +// $this->getImportOptions(), +// $expectedEscaping, +// 7, +// self::TABLE_OUT_CSV_2COLS, +// ]; +// +// yield 'gzipped standard with enclosure' => [ +// $this->createS3SourceInstance( +// 'gzipped-standard-with-enclosures.csv.gz', +// $escapingHeader, +// false, +// false, +// [] +// ), +// [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], +// $this->getImportOptions(), +// $expectedEscaping, +// 7, +// self::TABLE_OUT_CSV_2COLS, +// ]; +// +// yield 'standard with enclosures tabs' => [ +// $this->createS3SourceInstanceFromCsv( +// 'standard-with-enclosures.tabs.csv', +// new CsvOptions("\t"), +// $escapingHeader, +// false, +// false, +// [] +// ), +// [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], +// $this->getImportOptions(), +// $expectedEscaping, +// 7, +// self::TABLE_OUT_CSV_2COLS, +// ]; +// +// yield 'accounts changedColumnsOrder' => [ +// $this->createS3SourceInstance( +// 'tw_accounts.changedColumnsOrder.csv', +// $accountChangedColumnsOrderHeader, +// false, +// false, +// ['id'] +// ), +// [ +// $this->getDestinationDbName(), +// self::TABLE_ACCOUNTS_3, +// ], +// $this->getImportOptions(), +// $expectedAccounts, +// 3, +// self::TABLE_ACCOUNTS_3, +// ]; yield 'accounts' => [ $this->createS3SourceInstance( 'tw_accounts.csv', @@ -242,27 +240,27 @@ public function fullImportData(): Generator ['id'] ), [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], - $this->getImportOptions(), + $this->getSimpleImportOptions(ImportOptions::SKIP_FIRST_LINE), $expectedAccounts, 3, self::TABLE_ACCOUNTS_3, ]; - - // line ending detection is not supported yet for S3 - //yield 'accounts crlf' => [ - // $this->createS3SourceInstance( - // 'tw_accounts.crlf.csv', - // $accountsHeader, - // false, - // false, - // ['id'] - // ), - // [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], - // $this->getImportOptions(), - // $expectedAccounts, - // 3, - // self::TABLE_ACCOUNTS_3, - //]; +// +// // line ending detection is not supported yet for S3 +// //yield 'accounts crlf' => [ +// // $this->createS3SourceInstance( +// // 'tw_accounts.crlf.csv', +// // $accountsHeader, +// // false, +// // false, +// // ['id'] +// // ), +// // [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], +// // $this->getImportOptions(), +// // $expectedAccounts, +// // 3, +// // self::TABLE_ACCOUNTS_3, +// //]; // manifests yield 'accounts sliced' => [ @@ -279,130 +277,130 @@ public function fullImportData(): Generator 3, self::TABLE_ACCOUNTS_3, ]; - - yield 'accounts sliced gzip' => [ - $this->createS3SourceInstance( - 'sliced/accounts-gzip/S3.accounts-gzip.csvmanifest', - $accountsHeader, - true, - false, - ['id'] - ), - [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], - $this->getImportOptions(), - $expectedAccounts, - 3, - self::TABLE_ACCOUNTS_3, - ]; - - // folder - yield 'accounts sliced folder import' => [ - $this->createS3SourceInstance( - 'sliced_accounts_no_manifest/', - $accountsHeader, - true, - true, - ['id'] - ), - [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], - $this->getImportOptions(), - $expectedAccounts, - 3, - self::TABLE_ACCOUNTS_3, - ]; - - // reserved words - yield 'reserved words' => [ - $this->createS3SourceInstance( - 'reserved-words.csv', - ['column', 'table'], - false, - false, - [] - ), - [$this->getDestinationDbName(), self::TABLE_TABLE], - $this->getImportOptions(), - [['table', 'column', null]], - 1, - self::TABLE_TABLE, - ]; - // import table with _timestamp columns - used by snapshots - yield 'import with _timestamp columns' => [ - $this->createS3SourceInstance( - 'with-ts.csv', - [ - 'col1', - 'col2', - '_timestamp', - ], - false, - false, - [] - ), - [ - $this->getDestinationDbName(), - self::TABLE_OUT_CSV_2COLS, - ], - $this->getImportOptions(), - [ - ['a', 'b', '2014-11-10 13:12:06.000000'], - ['c', 'd', '2014-11-10 14:12:06.000000'], - ], - 2, - self::TABLE_OUT_CSV_2COLS, - ]; - // test creating table without _timestamp column - yield 'table without _timestamp column' => [ - $this->createS3SourceInstance( - 'standard-with-enclosures.csv', - $escapingHeader, - false, - false, - [] - ), - [ - $this->getDestinationDbName(), - self::TABLE_OUT_NO_TIMESTAMP_TABLE, - ], - $this->getImportOptions( - [], - false, - false, // don't use timestamp - ImportOptions::SKIP_FIRST_LINE - ), - $expectedEscaping, - 7, - self::TABLE_OUT_NO_TIMESTAMP_TABLE, - ]; - // copy from table - yield 'copy from table' => [ - new Table($this->getSourceDbName(), self::TABLE_OUT_CSV_2COLS, $escapingHeader), - [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], - $this->getImportOptions(), - [['a', 'b'], ['c', 'd']], - 2, - self::TABLE_OUT_CSV_2COLS, - ]; - yield 'copy from table 2' => [ - new Table( - $this->getSourceDbName(), - self::TABLE_TYPES, - [ - 'charCol', - 'numCol', - 'floatCol', - 'boolCol', - ] - ), - [ - $this->getDestinationDbName(), - self::TABLE_TYPES, - ], - $this->getImportOptions(), - [['a', '10.5', '0.3', '1']], - 1, - self::TABLE_TYPES, - ]; +// +// yield 'accounts sliced gzip' => [ +// $this->createS3SourceInstance( +// 'sliced/accounts-gzip/S3.accounts-gzip.csvmanifest', +// $accountsHeader, +// true, +// false, +// ['id'] +// ), +// [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], +// $this->getImportOptions(), +// $expectedAccounts, +// 3, +// self::TABLE_ACCOUNTS_3, +// ]; +// +// // folder +// yield 'accounts sliced folder import' => [ +// $this->createS3SourceInstance( +// 'sliced_accounts_no_manifest/', +// $accountsHeader, +// true, +// true, +// ['id'] +// ), +// [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], +// $this->getImportOptions(), +// $expectedAccounts, +// 3, +// self::TABLE_ACCOUNTS_3, +// ]; +// +// // reserved words +// yield 'reserved words' => [ +// $this->createS3SourceInstance( +// 'reserved-words.csv', +// ['column', 'table'], +// false, +// false, +// [] +// ), +// [$this->getDestinationDbName(), self::TABLE_TABLE], +// $this->getImportOptions(), +// [['table', 'column', null]], +// 1, +// self::TABLE_TABLE, +// ]; +// // import table with _timestamp columns - used by snapshots +// yield 'import with _timestamp columns' => [ +// $this->createS3SourceInstance( +// 'with-ts.csv', +// [ +// 'col1', +// 'col2', +// '_timestamp', +// ], +// false, +// false, +// [] +// ), +// [ +// $this->getDestinationDbName(), +// self::TABLE_OUT_CSV_2COLS, +// ], +// $this->getImportOptions(), +// [ +// ['a', 'b', '2014-11-10 13:12:06.000000'], +// ['c', 'd', '2014-11-10 14:12:06.000000'], +// ], +// 2, +// self::TABLE_OUT_CSV_2COLS, +// ]; +// // test creating table without _timestamp column +// yield 'table without _timestamp column' => [ +// $this->createS3SourceInstance( +// 'standard-with-enclosures.csv', +// $escapingHeader, +// false, +// false, +// [] +// ), +// [ +// $this->getDestinationDbName(), +// self::TABLE_OUT_NO_TIMESTAMP_TABLE, +// ], +// $this->getImportOptions( +// [], +// false, +// false, // don't use timestamp +// ImportOptions::SKIP_FIRST_LINE +// ), +// $expectedEscaping, +// 7, +// self::TABLE_OUT_NO_TIMESTAMP_TABLE, +// ]; +// // copy from table +// yield 'copy from table' => [ +// new Table($this->getSourceDbName(), self::TABLE_OUT_CSV_2COLS, $escapingHeader), +// [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], +// $this->getImportOptions(), +// [['a', 'b'], ['c', 'd']], +// 2, +// self::TABLE_OUT_CSV_2COLS, +// ]; +// yield 'copy from table 2' => [ +// new Table( +// $this->getSourceDbName(), +// self::TABLE_TYPES, +// [ +// 'charCol', +// 'numCol', +// 'floatCol', +// 'boolCol', +// ] +// ), +// [ +// $this->getDestinationDbName(), +// self::TABLE_TYPES, +// ], +// $this->getImportOptions(), +// [['a', '10.5', '0.3', '1']], +// 1, +// self::TABLE_TYPES, +// ]; } /** @@ -429,7 +427,7 @@ public function testFullImportWithDataSet( $tableName ))->getTableDefinition(); - $stagingTable = \Keboola\Db\ImportExport\Backend\Teradata\ToStage\StageTableDefinitionFactory::createStagingTableDefinition( + $stagingTable = StageTableDefinitionFactory::createStagingTableDefinition( $destination, $source->getColumnsNames() ); @@ -437,8 +435,8 @@ public function testFullImportWithDataSet( $this->connection->executeStatement( $qb->getCreateTableCommandFromDefinition($stagingTable) ); - $toStageImporter = new \Keboola\Db\ImportExport\Backend\Teradata\ToStage\ToStageImporter($this->connection); - $toFinalTableImporter = new \Keboola\Db\ImportExport\Backend\Teradata\ToFinalTable\FullImporter($this->connection); + $toStageImporter = new ToStageImporter($this->connection); + $toFinalTableImporter = new FullImporter($this->connection); try { $importState = $toStageImporter->importToStagingTable( $source, diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index e763aecc..7d2841a2 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -413,4 +413,20 @@ protected function getImportOptions( $numberOfIgnoredLines, ); } + + protected function getSimpleImportOptions( + int $skipLines = 0 + ): TeradataImportOptions { + return + new TeradataImportOptions( + (string) getenv('TERADATA_HOST'), + (string) getenv('TERADATA_USERNAME'), + (string) getenv('TERADATA_PASSWORD'), + (int) getenv('TERADATA_PORT'), + [], + false, + false, + $skipLines, + ); + } } From 938f4f35fd51d95e41823b4117ea2a4b54c05614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Tue, 15 Mar 2022 11:25:57 +0100 Subject: [PATCH 22/62] another tests --- tests/functional/Teradata/FullImportTest.php | 100 +++++++++--------- .../Teradata/TeradataBaseTestCase.php | 2 +- 2 files changed, 52 insertions(+), 50 deletions(-) diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index 431b4898..2864995a 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -6,6 +6,7 @@ use Generator; use Keboola\Csv\CsvFile; +use Keboola\CsvOptions\CsvOptions; use Keboola\Db\ImportExport\Backend\Teradata\TeradataImportOptions; use Keboola\Db\ImportExport\Backend\Teradata\ToFinalTable\SqlBuilder; use Keboola\Db\ImportExport\Backend\Teradata\ToFinalTable\FullImporter; @@ -153,35 +154,35 @@ public function fullImportData(): Generator // self::TABLE_OUT_CSV_2COLS, // ]; // -// yield 'lemma' => [ -// $this->createS3SourceInstance( -// 'lemma.csv', -// $lemmaHeader, -// false, -// false, -// [] -// ), -// [$this->getDestinationDbName(), self::TABLE_OUT_LEMMA], -// $this->getImportOptions(), -// $expectedLemma, -// 5, -// self::TABLE_OUT_LEMMA, -// ]; -// -// yield 'standard with enclosures' => [ -// $this->createS3SourceInstance( -// 'standard-with-enclosures.csv', -// $escapingHeader, -// false, -// false, -// [] -// ), -// [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], -// $this->getImportOptions(), -// $expectedEscaping, -// 7, -// self::TABLE_OUT_CSV_2COLS, -// ]; + yield 'lemma' => [ + $this->createS3SourceInstance( + 'lemma.csv', + $lemmaHeader, + false, + false, + [] + ), + [$this->getDestinationDbName(), self::TABLE_OUT_LEMMA], + $this->getSimpleImportOptions(), + $expectedLemma, + 5, + self::TABLE_OUT_LEMMA, + ]; + + yield 'standard with enclosures' => [ + $this->createS3SourceInstance( + 'standard-with-enclosures.csv', + $escapingHeader, + false, + false, + [] + ), + [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], + $this->getSimpleImportOptions(), + $expectedEscaping, + 7, + self::TABLE_OUT_CSV_2COLS, + ]; // // yield 'gzipped standard with enclosure' => [ // $this->createS3SourceInstance( @@ -208,29 +209,30 @@ public function fullImportData(): Generator // [] // ), // [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], -// $this->getImportOptions(), +// $this->getSimpleImportOptions(), // $expectedEscaping, // 7, // self::TABLE_OUT_CSV_2COLS, // ]; -// -// yield 'accounts changedColumnsOrder' => [ -// $this->createS3SourceInstance( -// 'tw_accounts.changedColumnsOrder.csv', -// $accountChangedColumnsOrderHeader, -// false, -// false, -// ['id'] -// ), -// [ -// $this->getDestinationDbName(), -// self::TABLE_ACCOUNTS_3, -// ], -// $this->getImportOptions(), -// $expectedAccounts, -// 3, -// self::TABLE_ACCOUNTS_3, -// ]; + + yield 'accounts changedColumnsOrder' => [ + $this->createS3SourceInstance( + 'tw_accounts.changedColumnsOrder.csv', + $accountChangedColumnsOrderHeader, + false, + false, + ['id'] + ), + [ + $this->getDestinationDbName(), + self::TABLE_ACCOUNTS_3, + ], + $this->getSimpleImportOptions(), + $expectedAccounts, + 3, + self::TABLE_ACCOUNTS_3, + ]; + yield 'accounts' => [ $this->createS3SourceInstance( 'tw_accounts.csv', @@ -240,7 +242,7 @@ public function fullImportData(): Generator ['id'] ), [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], - $this->getSimpleImportOptions(ImportOptions::SKIP_FIRST_LINE), + $this->getSimpleImportOptions(), $expectedAccounts, 3, self::TABLE_ACCOUNTS_3, diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index 7d2841a2..5d834518 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -415,7 +415,7 @@ protected function getImportOptions( } protected function getSimpleImportOptions( - int $skipLines = 0 + int $skipLines = ImportOptions::SKIP_FIRST_LINE ): TeradataImportOptions { return new TeradataImportOptions( From 63f4175a72fae662f88a439e72ab4df0326052ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Tue, 15 Mar 2022 13:08:26 +0100 Subject: [PATCH 23/62] enable and check most of the tests --- tests/functional/Teradata/FullImportTest.php | 295 +++++++++--------- .../Teradata/TeradataBaseTestCase.php | 63 +++- 2 files changed, 209 insertions(+), 149 deletions(-) diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index 2864995a..c8a82ea4 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -14,6 +14,7 @@ use Keboola\Db\ImportExport\Backend\Teradata\ToStage\ToStageImporter; use Keboola\Db\ImportExport\ImportOptions; use Keboola\Db\ImportExport\Storage\SourceInterface; +use Keboola\Db\ImportExport\Storage\Teradata\Table; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableDefinition; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableQueryBuilder; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableReflection; @@ -138,22 +139,22 @@ public function fullImportData(): Generator 1501, self::TABLE_OUT_CSV_2COLS, ]; -// -// yield 'empty manifest' => [ -// $this->createS3SourceInstance( -// 'empty.manifest', -// $escapingHeader, -// true, -// false, -// [] -// ), -// [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], -// $this->getImportOptions(), -// [], -// 0, -// self::TABLE_OUT_CSV_2COLS, -// ]; -// + + yield 'empty manifest' => [ + $this->createS3SourceInstance( + 'empty.manifest', + $escapingHeader, + true, + false, + [] + ), + [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], + $this->getImportOptions(), + [], + 0, + self::TABLE_OUT_CSV_2COLS, + ]; + yield 'lemma' => [ $this->createS3SourceInstance( 'lemma.csv', @@ -279,130 +280,130 @@ public function fullImportData(): Generator 3, self::TABLE_ACCOUNTS_3, ]; -// -// yield 'accounts sliced gzip' => [ -// $this->createS3SourceInstance( -// 'sliced/accounts-gzip/S3.accounts-gzip.csvmanifest', -// $accountsHeader, -// true, -// false, -// ['id'] -// ), -// [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], -// $this->getImportOptions(), -// $expectedAccounts, -// 3, -// self::TABLE_ACCOUNTS_3, -// ]; -// -// // folder -// yield 'accounts sliced folder import' => [ -// $this->createS3SourceInstance( -// 'sliced_accounts_no_manifest/', -// $accountsHeader, -// true, -// true, -// ['id'] -// ), -// [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], -// $this->getImportOptions(), -// $expectedAccounts, -// 3, -// self::TABLE_ACCOUNTS_3, -// ]; -// -// // reserved words -// yield 'reserved words' => [ -// $this->createS3SourceInstance( -// 'reserved-words.csv', -// ['column', 'table'], -// false, -// false, -// [] -// ), -// [$this->getDestinationDbName(), self::TABLE_TABLE], -// $this->getImportOptions(), -// [['table', 'column', null]], -// 1, -// self::TABLE_TABLE, -// ]; -// // import table with _timestamp columns - used by snapshots -// yield 'import with _timestamp columns' => [ -// $this->createS3SourceInstance( -// 'with-ts.csv', -// [ -// 'col1', -// 'col2', -// '_timestamp', -// ], -// false, -// false, -// [] -// ), -// [ -// $this->getDestinationDbName(), -// self::TABLE_OUT_CSV_2COLS, -// ], -// $this->getImportOptions(), -// [ -// ['a', 'b', '2014-11-10 13:12:06.000000'], -// ['c', 'd', '2014-11-10 14:12:06.000000'], -// ], -// 2, -// self::TABLE_OUT_CSV_2COLS, -// ]; -// // test creating table without _timestamp column -// yield 'table without _timestamp column' => [ -// $this->createS3SourceInstance( -// 'standard-with-enclosures.csv', -// $escapingHeader, -// false, -// false, -// [] -// ), -// [ -// $this->getDestinationDbName(), -// self::TABLE_OUT_NO_TIMESTAMP_TABLE, -// ], -// $this->getImportOptions( -// [], -// false, -// false, // don't use timestamp -// ImportOptions::SKIP_FIRST_LINE -// ), -// $expectedEscaping, -// 7, -// self::TABLE_OUT_NO_TIMESTAMP_TABLE, -// ]; -// // copy from table -// yield 'copy from table' => [ -// new Table($this->getSourceDbName(), self::TABLE_OUT_CSV_2COLS, $escapingHeader), -// [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], -// $this->getImportOptions(), -// [['a', 'b'], ['c', 'd']], -// 2, -// self::TABLE_OUT_CSV_2COLS, -// ]; -// yield 'copy from table 2' => [ -// new Table( -// $this->getSourceDbName(), -// self::TABLE_TYPES, -// [ -// 'charCol', -// 'numCol', -// 'floatCol', -// 'boolCol', -// ] -// ), -// [ -// $this->getDestinationDbName(), -// self::TABLE_TYPES, -// ], -// $this->getImportOptions(), -// [['a', '10.5', '0.3', '1']], -// 1, -// self::TABLE_TYPES, -// ]; + + yield 'accounts sliced gzip' => [ + $this->createS3SourceInstance( + 'sliced/accounts-gzip/S3.accounts-gzip.csvmanifest', + $accountsHeader, + true, + false, + ['id'] + ), + [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], + $this->getImportOptions(), + $expectedAccounts, + 3, + self::TABLE_ACCOUNTS_3, + ]; + + // folder + yield 'accounts sliced folder import' => [ + $this->createS3SourceInstance( + 'sliced_accounts_no_manifest/', + $accountsHeader, + true, + true, + ['id'] + ), + [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], + $this->getImportOptions(), + $expectedAccounts, + 3, + self::TABLE_ACCOUNTS_3, + ]; + + // reserved words + yield 'reserved words' => [ + $this->createS3SourceInstance( + 'reserved-words.csv', + ['column', 'table'], + false, + false, + [] + ), + [$this->getDestinationDbName(), self::TABLE_TABLE], + $this->getSimpleImportOptions(), + [['table', 'column', null]], + 1, + self::TABLE_TABLE, + ]; + // import table with _timestamp columns - used by snapshots + yield 'import with _timestamp columns' => [ + $this->createS3SourceInstance( + 'with-ts.csv', + [ + 'col1', + 'col2', + '_timestamp', + ], + false, + false, + [] + ), + [ + $this->getDestinationDbName(), + self::TABLE_OUT_CSV_2COLS, + ], + $this->getImportOptions(), + [ + ['a', 'b', '2014-11-10 13:12:06.000000'], + ['c', 'd', '2014-11-10 14:12:06.000000'], + ], + 2, + self::TABLE_OUT_CSV_2COLS, + ]; + // test creating table without _timestamp column + yield 'table without _timestamp column' => [ + $this->createS3SourceInstance( + 'standard-with-enclosures.csv', + $escapingHeader, + false, + false, + [] + ), + [ + $this->getDestinationDbName(), + self::TABLE_OUT_NO_TIMESTAMP_TABLE, + ], + $this->getImportOptions( + [], + false, + false, // don't use timestamp + ImportOptions::SKIP_FIRST_LINE + ), + $expectedEscaping, + 7, + self::TABLE_OUT_NO_TIMESTAMP_TABLE, + ]; + // copy from table + yield 'copy from table' => [ + new Table($this->getSourceDbName(), self::TABLE_OUT_CSV_2COLS, $escapingHeader), + [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], + $this->getImportOptions(), + [['a', 'b'], ['c', 'd']], + 2, + self::TABLE_OUT_CSV_2COLS, + ]; + yield 'copy from table 2' => [ + new Table( + $this->getSourceDbName(), + self::TABLE_TYPES, + [ + 'charCol', + 'numCol', + 'floatCol', + 'boolCol', + ] + ), + [ + $this->getDestinationDbName(), + self::TABLE_TYPES, + ], + $this->getImportOptions(), + [['a', '10.5', '0.3', '1']], + 1, + self::TABLE_TYPES, + ]; } /** @@ -467,12 +468,12 @@ public function testFullImportWithDataSet( self::assertEquals($expectedImportedRowCount, $result->getImportedRowsCount()); -// $this->assertTeradataTableEqualsExpected( -// $source, -// $destination, -// $options, -// $expected, -// 0 -// ); + $this->assertTeradataTableEqualsExpected( + $source, + $destination, + $options, + $expected, + 0 + ); } } diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index 5d834518..ef8e4eaa 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -415,7 +415,8 @@ protected function getImportOptions( } protected function getSimpleImportOptions( - int $skipLines = ImportOptions::SKIP_FIRST_LINE + int $skipLines = ImportOptions::SKIP_FIRST_LINE, + bool $useTimestamp = true ): TeradataImportOptions { return new TeradataImportOptions( @@ -425,8 +426,66 @@ protected function getSimpleImportOptions( (int) getenv('TERADATA_PORT'), [], false, - false, + $useTimestamp, $skipLines, ); } + + /** + * @param int|string $sortKey + * @param array $expected + * @param string|int $sortKey + */ + protected function assertTeradataTableEqualsExpected( + SourceInterface $source, + TeradataTableDefinition $destination, + TeradataImportOptions $options, + array $expected, + $sortKey, + string $message = 'Imported tables are not the same as expected' + ): void { + $tableColumns = (new TeradataTableReflection( + $this->connection, + $destination->getSchemaName(), + $destination->getTableName() + ))->getColumnsNames(); + + if ($options->useTimestamp()) { + self::assertContains('_timestamp', $tableColumns); + } else { + self::assertNotContains('_timestamp', $tableColumns); + } + + if (!in_array('_timestamp', $source->getColumnsNames(), true)) { + $tableColumns = array_filter($tableColumns, static function ($column) { + return $column !== '_timestamp'; + }); + } + + $tableColumns = array_map(static function ($column) { + return sprintf('%s', $column); + }, $tableColumns); + + $sql = sprintf( + 'SELECT %s FROM %s.%s', + implode(', ', array_map(static function ($item) { + return TeradataQuote::quoteSingleIdentifier($item); + }, $tableColumns)), + TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()), + TeradataQuote::quoteSingleIdentifier($destination->getTableName()) + ); + + $queryResult = array_map(static function ($row) { + return array_map(static function ($column) { + return $column; + }, array_values($row)); + }, $this->connection->fetchAllAssociative($sql)); + + $this->assertArrayEqualsSorted( + $expected, + $queryResult, + $sortKey, + $message + ); + } } From 34fe45b94d246fa4c8f585ca35d21dbfae11a82e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 18 Mar 2022 15:25:17 +0100 Subject: [PATCH 24/62] set unicode tables --- .../Teradata/ToStage/FromS3TPTAdapter.php | 5 +- tests/functional/Teradata/FullImportTest.php | 53 +++++++++---- .../Teradata/TeradataBaseTestCase.php | 77 ++++++++++--------- 3 files changed, 81 insertions(+), 54 deletions(-) diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index 4fc9086a..db5069d7 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -96,9 +96,9 @@ public function runCopyCommand( $err2Content = $this->connection->fetchAllAssociative('SELECT * FROM ' . TeradataQuote::quoteSingleIdentifier($errTable2)); $this->connection->executeStatement('DROP TABLE ' . $errTable2); } - // find the way how to get this out + // TODO find the way how to get this out - if ($process->getExitCode() !== 0) { + if ($process->getExitCode() !== 0 || $errContent || $err2Content) { $qb = new TeradataTableQueryBuilder(); // drop destination table it's not usable $this->connection->executeStatement($qb->getDropTableCommand( @@ -162,6 +162,7 @@ private function generateTPTScript( ); } $tptScript = <<getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], - $this->getImportOptions(), + $this->getSimpleImportOptions(), $expectedLargeSlicedManifest, 1501, self::TABLE_OUT_CSV_2COLS, @@ -275,7 +275,7 @@ public function fullImportData(): Generator ['id'] ), [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], - $this->getImportOptions(), + $this->getSimpleImportOptions(ImportOptions::SKIP_NO_LINE), $expectedAccounts, 3, self::TABLE_ACCOUNTS_3, @@ -290,7 +290,7 @@ public function fullImportData(): Generator ['id'] ), [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], - $this->getImportOptions(), + $this->getSimpleImportOptions(ImportOptions::SKIP_NO_LINE), $expectedAccounts, 3, self::TABLE_ACCOUNTS_3, @@ -306,7 +306,7 @@ public function fullImportData(): Generator ['id'] ), [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], - $this->getImportOptions(), + $this->getSimpleImportOptions(), $expectedAccounts, 3, self::TABLE_ACCOUNTS_3, @@ -453,17 +453,17 @@ public function testFullImportWithDataSet( $importState ); } finally { - if ($this->connection->fetchOne( - (new SqlBuilder())->getTableExistsCommand( - $stagingTable->getSchemaName(), - $stagingTable->getTableName() - ) - ) > 0) { - $this->connection->executeStatement((new SqlBuilder())->getDropTableUnsafe( - $stagingTable->getSchemaName(), - $stagingTable->getTableName() - )); - } +// if ($this->connection->fetchOne( +// (new SqlBuilder())->getTableExistsCommand( +// $stagingTable->getSchemaName(), +// $stagingTable->getTableName() +// ) +// ) > 0) { +// $this->connection->executeStatement((new SqlBuilder())->getDropTableUnsafe( +// $stagingTable->getSchemaName(), +// $stagingTable->getTableName() +// )); +// } } self::assertEquals($expectedImportedRowCount, $result->getImportedRowsCount()); @@ -476,4 +476,27 @@ public function testFullImportWithDataSet( 0 ); } +// +// public function testFetch() +// { +// $db = self::TERADATA_SOURCE_DATABASE_NAME; +// $name = "jirka" . time(); +// $this->initSingleTable($name, "transactions"); +// $this->connection->fetchAllAssociative(sprintf( +// 'SELECT * FROM "'.$name.'"."transactions";' +// )); +//// $this->connection->fetchAllAssociative(sprintf( +//// 'SELECT * FROM "jirkadatabazeaa"."jirkatabulka";' +//// )); +// +// +//// $this->cleanDatabase($db); +//// $this->createDatabase($db); +//// $table = self::TABLE_TABLE; +//// $this->initSingleTable(); +//// $count = $this->connection->fetchOne(sprintf("select count(*) from %s.%s", TeradataQuote::quoteSingleIdentifier($db), +//// TeradataQuote::quoteSingleIdentifier($table))); +//// $data = $this->connection->fetchAllAssociative(sprintf('select "Other" from %s.%s', TeradataQuote::quoteSingleIdentifier($db), +//// TeradataQuote::quoteSingleIdentifier($table))); +// } } diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index ef8e4eaa..61553b91 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -126,7 +126,9 @@ public function createDatabase(string $dbName): void { $this->connection->executeQuery(sprintf(' CREATE DATABASE %s AS - PERM = 5e6; + PERM = 5e7 + SPOOL = 5e7; + ', TeradataQuote::quoteSingleIdentifier($dbName))); } @@ -212,7 +214,7 @@ protected function initSingleTable( sprintf( 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( -"Other" VARCHAR(4000) +"Other" VARCHAR(50) );', TeradataQuote::quoteSingleIdentifier($db), TeradataQuote::quoteSingleIdentifier($table) @@ -228,11 +230,11 @@ protected function initTable(string $tableName): void sprintf( 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( -"VisitID" VARCHAR(4000) , -"Value" VARCHAR(4000), -"MenuItem" VARCHAR(4000), -"Something" VARCHAR(4000), -"Other" VARCHAR(4000), +"VisitID" VARCHAR(50) CHARACTER SET UNICODE, +"Value" VARCHAR(50), +"MenuItem" VARCHAR(50), +"Something" VARCHAR(50), +"Other" VARCHAR(50), ) PRIMARY INDEX ("VisitID"); );', @@ -245,8 +247,8 @@ protected function initTable(string $tableName): void $this->connection->executeQuery(sprintf( 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( - "id" VARCHAR(4000) , - "row_number" VARCHAR(4000) + "id" VARCHAR(50) CHARACTER SET UNICODE, + "row_number" VARCHAR(50) )', TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), TeradataQuote::quoteSingleIdentifier($tableName) @@ -257,7 +259,7 @@ protected function initTable(string $tableName): void 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( "id" INT , - "name" VARCHAR(4000) , + "name" VARCHAR(50) CHARACTER SET UNICODE, "price" INT , "isDeleted" INT )', @@ -286,8 +288,8 @@ protected function initTable(string $tableName): void $this->connection->executeQuery(sprintf( 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( - "col1" VARCHAR(4000) , - "col2" VARCHAR(4000) + "col1" VARCHAR(50) CHARACTER SET UNICODE, + "col2" VARCHAR(50) );', TeradataQuote::quoteSingleIdentifier($this->getSourceDbName()), TeradataQuote::quoteSingleIdentifier($tableName) @@ -308,9 +310,9 @@ protected function initTable(string $tableName): void case self::TABLE_OUT_LEMMA: $this->connection->executeQuery(sprintf( 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( - "ts" VARCHAR(4000) , - "lemma" VARCHAR(4000) , - "lemmaIndex" VARCHAR(4000) , + "ts" VARCHAR(50) , + "lemma" VARCHAR(50) , + "lemmaIndex" VARCHAR(50) CHARACTER SET UNICODE, "_timestamp" TIMESTAMP );', TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), @@ -320,18 +322,18 @@ protected function initTable(string $tableName): void case self::TABLE_ACCOUNTS_3: $this->connection->executeQuery(sprintf( 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( - "id" VARCHAR(4000) , - "idTwitter" VARCHAR(4000) , - "name" VARCHAR(4000) , - "import" VARCHAR(4000) , - "isImported" VARCHAR(4000) , - "apiLimitExceededDatetime" VARCHAR(4000) , - "analyzeSentiment" VARCHAR(4000) , - "importKloutScore" VARCHAR(4000) , - "timestamp" VARCHAR(4000) , - "oauthToken" VARCHAR(4000) , - "oauthSecret" VARCHAR(4000) , - "idApp" VARCHAR(4000) , + "id" VARCHAR(50) CHARACTER SET UNICODE, + "idTwitter" VARCHAR(50) CHARACTER SET UNICODE, + "name" VARCHAR(100) CHARACTER SET UNICODE, + "import" VARCHAR(50) CHARACTER SET UNICODE, + "isImported" VARCHAR(50) CHARACTER SET UNICODE, + "apiLimitExceededDatetime" VARCHAR(50) CHARACTER SET UNICODE, + "analyzeSentiment" VARCHAR(50) CHARACTER SET UNICODE, + "importKloutScore" VARCHAR(50) CHARACTER SET UNICODE, + "timestamp" VARCHAR(50) CHARACTER SET UNICODE, + "oauthToken" VARCHAR(50) CHARACTER SET UNICODE, + "oauthSecret" VARCHAR(50) CHARACTER SET UNICODE, + "idApp" VARCHAR(50) CHARACTER SET UNICODE, "_timestamp" TIMESTAMP ) PRIMARY INDEX ("id");', TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), @@ -341,9 +343,9 @@ protected function initTable(string $tableName): void case self::TABLE_TABLE: $this->connection->executeQuery(sprintf( 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( - "column" VARCHAR(4000) , - "table" VARCHAR(4000) , - "lemmaIndex" VARCHAR(4000) , + "column" VARCHAR(50) , + "table" VARCHAR(50) , + "lemmaIndex" VARCHAR(50) CHARACTER SET UNICODE, "_timestamp" TIMESTAMP );', TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), @@ -353,8 +355,8 @@ protected function initTable(string $tableName): void case self::TABLE_OUT_NO_TIMESTAMP_TABLE: $this->connection->executeQuery(sprintf( 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( - "col1" VARCHAR(4000) , - "col2" VARCHAR(4000) + "col1" VARCHAR(50) , + "col2" VARCHAR(50) );', TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), TeradataQuote::quoteSingleIdentifier($tableName) @@ -364,10 +366,10 @@ protected function initTable(string $tableName): void case self::TABLE_TYPES: $this->connection->executeQuery(sprintf( 'CREATE TABLE %s."types" ( - "charCol" VARCHAR(4000) , - "numCol" VARCHAR(4000) , - "floatCol" VARCHAR(4000) , - "boolCol" VARCHAR(4000) , + "charCol" VARCHAR(50) CHARACTER SET UNICODE, + "numCol" VARCHAR(50) CHARACTER SET UNICODE, + "floatCol" VARCHAR(50) CHARACTER SET UNICODE, + "boolCol" VARCHAR(50) CHARACTER SET UNICODE, "_timestamp" TIMESTAMP );', TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()) @@ -375,7 +377,7 @@ protected function initTable(string $tableName): void $this->connection->executeQuery(sprintf( 'CREATE TABLE %s."types" ( - "charCol" VARCHAR(4000) , + "charCol" VARCHAR(50) CHARACTER SET UNICODE, "numCol" decimal(10,1) , "floatCol" float , "boolCol" tinyint @@ -488,4 +490,5 @@ protected function assertTeradataTableEqualsExpected( $message ); } + } From 05d4223f04eb64a2b41805e790c27cbeb94c3115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Sun, 20 Mar 2022 23:51:21 +0100 Subject: [PATCH 25/62] load meaningful data from error tables --- .../Teradata/ToStage/FromS3TPTAdapter.php | 40 ++++++++++++++----- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index db5069d7..bf543c93 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -8,6 +8,7 @@ use Keboola\Db\ImportExport\Backend\CopyAdapterInterface; use Keboola\Db\ImportExport\Backend\Teradata\Helper\BackendHelper; use Keboola\Db\ImportExport\Backend\Teradata\TeradataImportOptions; +use Keboola\Db\ImportExport\Backend\Teradata\ToFinalTable\SqlBuilder; use Keboola\Db\ImportExport\Backend\Teradata\ToStage\Exception\FailedTPTLoadException; use Keboola\Db\ImportExport\ImportOptions; use Keboola\Db\ImportExport\ImportOptionsInterface; @@ -68,6 +69,8 @@ public function runCopyCommand( ] ); $process->start(); + // check end of process + $process->wait(); // debug stuff foreach ($process as $type => $data) { @@ -77,24 +80,39 @@ public function runCopyCommand( echo "\nRead from stderr: " . $data; } } - $isTableExists = function (string $tableName, string $databaseName) { - return (bool) $this->connection->fetchOne(sprintf('SELECT 1 FROM dbc.TablesV WHERE TableName = %s AND DataBaseName = %s', TeradataQuote::quote($tableName), TeradataQuote::quote($databaseName))); + $qb = new SqlBuilder(); + $isTableExists = function (string $databaseName, string $tableName) use ($qb) { + return (bool) $this->connection->fetchOne($qb->getTableExistsCommand($databaseName, $tableName)); }; $logContent = null; - if ($isTableExists($logTable, $destination->getSchemaName())) { - $logContent = $this->connection->fetchAllAssociative('SELECT * FROM ' . TeradataQuote::quoteSingleIdentifier($logTable)); - $this->connection->executeStatement('DROP TABLE ' . $logTable); + if ($isTableExists($destination->getSchemaName(), $logTable)) { + $logContent = $this->connection->fetchAllAssociative( + sprintf( + 'SELECT * FROM %s.%s', + TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()), + TeradataQuote::quoteSingleIdentifier($logTable) + ) + ); + $this->connection->executeStatement($qb->getDropTableUnsafe($destination->getSchemaName(), $logTable)); } $errContent = null; - if ($isTableExists($errTable, $destination->getSchemaName())) { - $errContent = $this->connection->fetchAllAssociative('SELECT * FROM ' . TeradataQuote::quoteSingleIdentifier($errTable)); - $this->connection->executeStatement('DROP TABLE ' . $errTable); + if ($isTableExists($destination->getSchemaName(), $errTable)) { + $errContent = $this->connection->fetchAllAssociative(sprintf( + 'SELECT * FROM %s.%s', + TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()), + TeradataQuote::quoteSingleIdentifier($errTable) + )); + $this->connection->executeStatement($qb->getDropTableUnsafe($destination->getSchemaName(), $errTable)); } $err2Content = null; - if ($isTableExists($errTable2, $destination->getSchemaName())) { - $err2Content = $this->connection->fetchAllAssociative('SELECT * FROM ' . TeradataQuote::quoteSingleIdentifier($errTable2)); - $this->connection->executeStatement('DROP TABLE ' . $errTable2); + if ($isTableExists($destination->getSchemaName(), $errTable2)) { + $err2Content = $this->connection->fetchAllAssociative(sprintf( + 'SELECT * FROM %s.%s', + TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()), + TeradataQuote::quoteSingleIdentifier($errTable2) + )); + $this->connection->executeStatement($qb->getDropTableUnsafe($destination->getSchemaName(), $errTable2)); } // TODO find the way how to get this out From 0e944d08459bff8ecacfac014406f0fbffdb8d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Mon, 21 Mar 2022 10:29:54 +0100 Subject: [PATCH 26/62] large manifest works --- tests/functional/Teradata/FullImportTest.php | 2 +- tests/functional/Teradata/TeradataBaseTestCase.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index 5436061c..22770caa 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -134,7 +134,7 @@ public function fullImportData(): Generator [] ), [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], - $this->getSimpleImportOptions(), + $this->getSimpleImportOptions(ImportOptions::SKIP_NO_LINE), $expectedLargeSlicedManifest, 1501, self::TABLE_OUT_CSV_2COLS, diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index 61553b91..e9a30d3b 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -271,8 +271,8 @@ protected function initTable(string $tableName): void $this->connection->executeQuery( sprintf( 'CREATE MULTISET TABLE %s.%s, NO FALLBACK ( - "col1" VARCHAR(20000) , - "col2" VARCHAR(20000) , + "col1" VARCHAR(200) , + "col2" VARCHAR(200) , "_timestamp" TIMESTAMP );', TeradataQuote::quoteSingleIdentifier($this->getDestinationDbName()), From 5b3a630227263a1c69df6699ea915170fa9b0562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Mon, 21 Mar 2022 11:12:02 +0100 Subject: [PATCH 27/62] empty manifest fix --- src/Backend/Teradata/Helper/BackendHelper.php | 4 ++++ src/Backend/Teradata/ToStage/FromS3TPTAdapter.php | 5 +++++ tests/functional/Teradata/FullImportTest.php | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Backend/Teradata/Helper/BackendHelper.php b/src/Backend/Teradata/Helper/BackendHelper.php index a2d69e95..4787456b 100644 --- a/src/Backend/Teradata/Helper/BackendHelper.php +++ b/src/Backend/Teradata/Helper/BackendHelper.php @@ -30,6 +30,10 @@ public static function generateTempDedupTableName(): string public static function getMask(SourceFile $source): string { $entries = $source->getManifestEntries(); + if (count($entries) === 0) { + // no entries -> no data to load + return ''; + } $toRemove = $source->getS3Prefix() . '/' . $source->getPrefix(); $entriesAsArrays = []; $min = 99999; diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index bf543c93..50c5276a 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -49,6 +49,11 @@ public function runCopyCommand( assert($destination instanceof TeradataTableDefinition); assert($importOptions instanceof TeradataImportOptions); + // empty manifest. TPT cannot import empty data + if($source->isSliced() && count($source->getManifestEntries()) === 0){ + return 0; + } + /** * @var Temp $temp */ diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index 22770caa..10b07c40 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -149,7 +149,7 @@ public function fullImportData(): Generator [] ), [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], - $this->getImportOptions(), + $this->getSimpleImportOptions(), [], 0, self::TABLE_OUT_CSV_2COLS, From 0824dd27af606d91782433275ca4e3658569420b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Mon, 21 Mar 2022 11:57:11 +0100 Subject: [PATCH 28/62] disable table to table tests for time being --- tests/functional/Teradata/FullImportTest.php | 57 ++++++++++--------- .../Teradata/TeradataBaseTestCase.php | 2 +- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index 10b07c40..96bab37a 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -376,34 +376,35 @@ public function fullImportData(): Generator self::TABLE_OUT_NO_TIMESTAMP_TABLE, ]; // copy from table - yield 'copy from table' => [ - new Table($this->getSourceDbName(), self::TABLE_OUT_CSV_2COLS, $escapingHeader), - [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], - $this->getImportOptions(), - [['a', 'b'], ['c', 'd']], - 2, - self::TABLE_OUT_CSV_2COLS, - ]; - yield 'copy from table 2' => [ - new Table( - $this->getSourceDbName(), - self::TABLE_TYPES, - [ - 'charCol', - 'numCol', - 'floatCol', - 'boolCol', - ] - ), - [ - $this->getDestinationDbName(), - self::TABLE_TYPES, - ], - $this->getImportOptions(), - [['a', '10.5', '0.3', '1']], - 1, - self::TABLE_TYPES, - ]; + // TODO not implemented because we dont have Table adapter yet +// yield 'copy from table' => [ +// new Table($this->getSourceDbName(), self::TABLE_OUT_CSV_2COLS, $escapingHeader), +// [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], +// $this->getImportOptions(), +// [['a', 'b'], ['c', 'd']], +// 2, +// self::TABLE_OUT_CSV_2COLS, +// ]; +// yield 'copy from table 2' => [ +// new Table( +// $this->getSourceDbName(), +// self::TABLE_TYPES, +// [ +// 'charCol', +// 'numCol', +// 'floatCol', +// 'boolCol', +// ] +// ), +// [ +// $this->getDestinationDbName(), +// self::TABLE_TYPES, +// ], +// $this->getImportOptions(), +// [['a', '10.5', '0.3', '1']], +// 1, +// self::TABLE_TYPES, +// ]; } /** diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index e9a30d3b..0f1a34b5 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -380,7 +380,7 @@ protected function initTable(string $tableName): void "charCol" VARCHAR(50) CHARACTER SET UNICODE, "numCol" decimal(10,1) , "floatCol" float , - "boolCol" tinyint + "boolCol" BYTEINT );', TeradataQuote::quoteSingleIdentifier($this->getSourceDbName()) )); From 9eeb8335713976b8feba863741f03f656cbaf52f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Mon, 21 Mar 2022 12:09:55 +0100 Subject: [PATCH 29/62] drop table after failure and reinit tests --- tests/functional/Teradata/FullImportTest.php | 116 +++++++++---------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index 96bab37a..be75d3e3 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -184,37 +184,37 @@ public function fullImportData(): Generator 7, self::TABLE_OUT_CSV_2COLS, ]; -// -// yield 'gzipped standard with enclosure' => [ -// $this->createS3SourceInstance( -// 'gzipped-standard-with-enclosures.csv.gz', -// $escapingHeader, -// false, -// false, -// [] -// ), -// [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], -// $this->getImportOptions(), -// $expectedEscaping, -// 7, -// self::TABLE_OUT_CSV_2COLS, -// ]; -// -// yield 'standard with enclosures tabs' => [ -// $this->createS3SourceInstanceFromCsv( -// 'standard-with-enclosures.tabs.csv', -// new CsvOptions("\t"), -// $escapingHeader, -// false, -// false, -// [] -// ), -// [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], -// $this->getSimpleImportOptions(), -// $expectedEscaping, -// 7, -// self::TABLE_OUT_CSV_2COLS, -// ]; + + yield 'gzipped standard with enclosure' => [ + $this->createS3SourceInstance( + 'gzipped-standard-with-enclosures.csv.gz', + $escapingHeader, + false, + false, + [] + ), + [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], + $this->getSimpleImportOptions(), + $expectedEscaping, + 7, + self::TABLE_OUT_CSV_2COLS, + ]; + + yield 'standard with enclosures tabs' => [ + $this->createS3SourceInstanceFromCsv( + 'standard-with-enclosures.tabs.csv', + new CsvOptions("\t"), + $escapingHeader, + false, + false, + [] + ), + [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], + $this->getSimpleImportOptions(), + $expectedEscaping, + 7, + self::TABLE_OUT_CSV_2COLS, + ]; yield 'accounts changedColumnsOrder' => [ $this->createS3SourceInstance( @@ -248,22 +248,22 @@ public function fullImportData(): Generator 3, self::TABLE_ACCOUNTS_3, ]; -// -// // line ending detection is not supported yet for S3 -// //yield 'accounts crlf' => [ -// // $this->createS3SourceInstance( -// // 'tw_accounts.crlf.csv', -// // $accountsHeader, -// // false, -// // false, -// // ['id'] -// // ), -// // [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], -// // $this->getImportOptions(), -// // $expectedAccounts, -// // 3, -// // self::TABLE_ACCOUNTS_3, -// //]; + + // line ending detection is not supported yet for S3 + //yield 'accounts crlf' => [ + // $this->createS3SourceInstance( + // 'tw_accounts.crlf.csv', + // $accountsHeader, + // false, + // false, + // ['id'] + // ), + // [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], + // $this->getImportOptions(), + // $expectedAccounts, + // 3, + // self::TABLE_ACCOUNTS_3, + //]; // manifests yield 'accounts sliced' => [ @@ -454,17 +454,17 @@ public function testFullImportWithDataSet( $importState ); } finally { -// if ($this->connection->fetchOne( -// (new SqlBuilder())->getTableExistsCommand( -// $stagingTable->getSchemaName(), -// $stagingTable->getTableName() -// ) -// ) > 0) { -// $this->connection->executeStatement((new SqlBuilder())->getDropTableUnsafe( -// $stagingTable->getSchemaName(), -// $stagingTable->getTableName() -// )); -// } + if ($this->connection->fetchOne( + (new SqlBuilder())->getTableExistsCommand( + $stagingTable->getSchemaName(), + $stagingTable->getTableName() + ) + ) > 0) { + $this->connection->executeStatement((new SqlBuilder())->getDropTableUnsafe( + $stagingTable->getSchemaName(), + $stagingTable->getTableName() + )); + } } self::assertEquals($expectedImportedRowCount, $result->getImportedRowsCount()); From fcbe45a110bb05f42d5e906e3aae978e24411df1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Mon, 21 Mar 2022 12:14:38 +0100 Subject: [PATCH 30/62] tmp update datatypes --- composer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index a58fd70a..736f9056 100644 --- a/composer.json +++ b/composer.json @@ -10,9 +10,9 @@ "keboola/php-csv-db-import": "^5.0", "keboola/php-file-storage-utils": ">=0.2", "keboola/table-backend-utils": "^0.18", + "keboola/php-temp": "^1.0", "microsoft/azure-storage-blob": "^1.4", - "symfony/process": "^4.4|^5.0", - "keboola/php-temp": "^1.0" + "symfony/process": "^4.4|^5.0" }, "require-dev": { "phpstan/phpstan": "^0.12.54", From d33ca8eab6688cd025f8329b8692a47ccf703a4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Mon, 21 Mar 2022 15:05:33 +0100 Subject: [PATCH 31/62] tabs support --- src/Backend/Teradata/ToStage/FromS3TPTAdapter.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index 50c5276a..beeda025 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -214,6 +214,9 @@ private function generateTPTScript( $pass = $importOptions->getTeradataPassword(); $csvOptions = $source->getCsvOptions(); $delimiter = $csvOptions->getDelimiter(); + if ($delimiter === "\t") { + $delimiter = 'TAB'; + } $enclosure = $csvOptions->getEnclosure(); if ($enclosure === '\'') { $enclosure = '\\\''; From b161420c9ddddc4c2acf02b95f0bbfc2f061bbe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Mon, 21 Mar 2022 15:56:45 +0100 Subject: [PATCH 32/62] close connection --- src/Backend/Teradata/ToStage/FromS3TPTAdapter.php | 5 ++--- tests/functional/Teradata/FullImportTest.php | 6 ++++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index beeda025..638c3cbd 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -23,8 +23,6 @@ /** * @todo: get logs out on success - * @todo: fix log tables exists statement - * @todo: test sliced files */ class FromS3TPTAdapter implements CopyAdapterInterface { @@ -133,7 +131,7 @@ public function runCopyCommand( $process->getErrorOutput(), $process->getOutput(), $process->getExitCode(), - file_get_contents($temp->getTmpFolder() . '/import-1.out'), + file_exists($temp->getTmpFolder() . '/import-1.out') ? file_get_contents($temp->getTmpFolder() . '/import-1.out') : 'unable to get error', $logContent, $errContent, $err2Content @@ -268,6 +266,7 @@ private function generateTPTScript( EOD; file_put_contents($folder . '/import_vars.txt', $jobVariableFile); + file_put_contents('./import_vars.txt', $jobVariableFile); return [ $temp, diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index be75d3e3..daee548d 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -34,6 +34,12 @@ protected function setUp(): void $this->createDatabase($this->getSourceDbName()); } + protected function tearDown() + { + $this->connection->close(); + parent::tearDown(); + } + public function testLoadToFinalTableWithoutDedup(): void { // table translations checks numeric and string-ish data From 2751ab19ec2db5944c1963f57bb08280cf4c254d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Tue, 22 Mar 2022 11:28:17 +0100 Subject: [PATCH 33/62] init table insertIntoAdapter and test it --- .../ToStage/FromTableInsertIntoAdapter.php | 64 +++++++++++++++++++ .../Teradata/ToStage/ToStageImporter.php | 7 +- tests/functional/Teradata/FullImportTest.php | 56 ++++++++-------- 3 files changed, 95 insertions(+), 32 deletions(-) create mode 100644 src/Backend/Teradata/ToStage/FromTableInsertIntoAdapter.php diff --git a/src/Backend/Teradata/ToStage/FromTableInsertIntoAdapter.php b/src/Backend/Teradata/ToStage/FromTableInsertIntoAdapter.php new file mode 100644 index 00000000..13c79241 --- /dev/null +++ b/src/Backend/Teradata/ToStage/FromTableInsertIntoAdapter.php @@ -0,0 +1,64 @@ +connection = $connection; + } + + public function runCopyCommand( + Storage\SourceInterface $source, + TableDefinitionInterface $destination, + ImportOptionsInterface $importOptions + ): int { + assert($source instanceof SelectSource || $source instanceof Table); + assert($destination instanceof TeradataTableDefinition); + assert($importOptions instanceof TeradataImportOptions); + + $quotedColumns = array_map(static function ($column) { + return TeradataQuote::quoteSingleIdentifier($column); + }, $source->getColumnsNames()); + + $sql = sprintf( + 'INSERT INTO %s.%s (%s) %s', + TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()), + TeradataQuote::quoteSingleIdentifier($destination->getTableName()), + implode(', ', $quotedColumns), + $source->getFromStatement() + ); + + if ($source instanceof SelectSource) { + $this->connection->executeQuery($sql, $source->getQueryBindings(), $source->getDataTypes()); + } else { + $this->connection->executeStatement($sql); + } + + $ref = new TeradataTableReflection( + $this->connection, + $destination->getSchemaName(), + $destination->getTableName() + ); + + return $ref->getRowsCount(); + } +} diff --git a/src/Backend/Teradata/ToStage/ToStageImporter.php b/src/Backend/Teradata/ToStage/ToStageImporter.php index 5c6cc4de..c5962602 100644 --- a/src/Backend/Teradata/ToStage/ToStageImporter.php +++ b/src/Backend/Teradata/ToStage/ToStageImporter.php @@ -59,11 +59,10 @@ private function getAdapter(Storage\SourceInterface $source): CopyAdapterInterfa switch (true) { case $source instanceof Storage\S3\SourceFile: $adapter = new FromS3TPTAdapter($this->connection); - //$adapter = new FromLocalTPTAdapter($this->connection); break; - //case $source instanceof Storage\SqlSourceInterface: - // $adapter = new FromTableInsertIntoAdapter($this->connection); - // break; + case $source instanceof Storage\SqlSourceInterface: + $adapter = new FromTableInsertIntoAdapter($this->connection); + break; default: throw new LogicException( sprintf( diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index daee548d..2e4be2c6 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -383,34 +383,34 @@ public function fullImportData(): Generator ]; // copy from table // TODO not implemented because we dont have Table adapter yet -// yield 'copy from table' => [ -// new Table($this->getSourceDbName(), self::TABLE_OUT_CSV_2COLS, $escapingHeader), -// [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], -// $this->getImportOptions(), -// [['a', 'b'], ['c', 'd']], -// 2, -// self::TABLE_OUT_CSV_2COLS, -// ]; -// yield 'copy from table 2' => [ -// new Table( -// $this->getSourceDbName(), -// self::TABLE_TYPES, -// [ -// 'charCol', -// 'numCol', -// 'floatCol', -// 'boolCol', -// ] -// ), -// [ -// $this->getDestinationDbName(), -// self::TABLE_TYPES, -// ], -// $this->getImportOptions(), -// [['a', '10.5', '0.3', '1']], -// 1, -// self::TABLE_TYPES, -// ]; + yield 'copy from table' => [ + new Table($this->getSourceDbName(), self::TABLE_OUT_CSV_2COLS, $escapingHeader), + [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], + $this->getSimpleImportOptions(), + [['a', 'b'], ['c', 'd']], + 2, + self::TABLE_OUT_CSV_2COLS, + ]; + yield 'copy from table 2' => [ + new Table( + $this->getSourceDbName(), + self::TABLE_TYPES, + [ + 'charCol', + 'numCol', + 'floatCol', + 'boolCol', + ] + ), + [ + $this->getDestinationDbName(), + self::TABLE_TYPES, + ], + $this->getSimpleImportOptions(), + [['a', '10.5', '0.3', '1']], + 1, + self::TABLE_TYPES, + ]; } /** From 7248d9aa17f30bd266659ecc87277369d60d64cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Tue, 22 Mar 2022 11:29:08 +0100 Subject: [PATCH 34/62] load data from directory --- src/Backend/Teradata/Helper/BackendHelper.php | 3 ++- src/Backend/Teradata/ToStage/FromS3TPTAdapter.php | 3 +-- src/Storage/S3/SourceDirectory.php | 1 + tests/functional/Teradata/FullImportTest.php | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Backend/Teradata/Helper/BackendHelper.php b/src/Backend/Teradata/Helper/BackendHelper.php index 4787456b..ae708405 100644 --- a/src/Backend/Teradata/Helper/BackendHelper.php +++ b/src/Backend/Teradata/Helper/BackendHelper.php @@ -34,7 +34,8 @@ public static function getMask(SourceFile $source): string // no entries -> no data to load return ''; } - $toRemove = $source->getS3Prefix() . '/' . $source->getPrefix(); + // SourceDirectory returns fileName as directory/file.csv but SourceFile returns s3://bucket/directory/file.csv + $toRemove = $source->getS3Prefix() . '/'; $entriesAsArrays = []; $min = 99999; $minIndex = 0; diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index 638c3cbd..321e2634 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -167,10 +167,9 @@ private function generateTPTScript( if ($source->isSliced()) { $moduleStr = sprintf( - 'AccessModuleInitStr = \'S3Region=%s S3Bucket=%s S3Prefix="%s" S3Object=%s S3SinglePartFile=True\'', + 'AccessModuleInitStr = \'S3Region=%s S3Bucket=%s S3Prefix="" S3Object=%s S3SinglePartFile=True\'', $source->getRegion(), $source->getBucket(), - $source->getPrefix(), BackendHelper::getMask($source) ); } else { diff --git a/src/Storage/S3/SourceDirectory.php b/src/Storage/S3/SourceDirectory.php index ab93988c..d77ec56e 100644 --- a/src/Storage/S3/SourceDirectory.php +++ b/src/Storage/S3/SourceDirectory.php @@ -7,6 +7,7 @@ class SourceDirectory extends SourceFile { /** + * returns all files in directory * @return string[] */ public function getManifestEntries(): array diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index 2e4be2c6..93282f50 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -312,7 +312,7 @@ public function fullImportData(): Generator ['id'] ), [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], - $this->getSimpleImportOptions(), + $this->getSimpleImportOptions(ImportOptions::SKIP_NO_LINE), $expectedAccounts, 3, self::TABLE_ACCOUNTS_3, From 78a5352ddd892ee347658840042a8d941f9dae5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Tue, 22 Mar 2022 13:24:35 +0100 Subject: [PATCH 35/62] trim implicitly casted values --- src/Storage/Teradata/Table.php | 10 ++++++---- tests/functional/Teradata/FullImportTest.php | 1 - 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Storage/Teradata/Table.php b/src/Storage/Teradata/Table.php index 6182d59e..1ac98431 100644 --- a/src/Storage/Teradata/Table.php +++ b/src/Storage/Teradata/Table.php @@ -7,7 +7,7 @@ use Keboola\Db\ImportExport\Storage\DestinationInterface; use Keboola\Db\ImportExport\Storage\SourceInterface; use Keboola\Db\ImportExport\Storage\SqlSourceInterface; -use Keboola\TableBackendUtils\Escaping\Exasol\ExasolQuote; +use Keboola\TableBackendUtils\Escaping\Teradata\TeradataQuote; class Table implements SourceInterface, DestinationInterface, SqlSourceInterface { @@ -45,7 +45,9 @@ public function getFromStatement(): string $colums = $this->getColumnsNames(); if ($colums !== []) { $quotedColumns = array_map(static function ($column) { - return ExasolQuote::quoteSingleIdentifier($column); + // trim because implicit casting adds right padding spaces + // value 10.5 as DECIMAL(8,1) implicitly casted to varchar would be then " 10.5" + return sprintf("TRIM(%s)", TeradataQuote::quoteSingleIdentifier($column)); }, $colums); $select = implode(', ', $quotedColumns); } @@ -65,8 +67,8 @@ public function getQuotedTableWithScheme(): string { return sprintf( '%s.%s', - ExasolQuote::quoteSingleIdentifier($this->getSchema()), - ExasolQuote::quoteSingleIdentifier($this->getTableName()) + TeradataQuote::quoteSingleIdentifier($this->getSchema()), + TeradataQuote::quoteSingleIdentifier($this->getTableName()) ); } diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index 93282f50..ef0badee 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -382,7 +382,6 @@ public function fullImportData(): Generator self::TABLE_OUT_NO_TIMESTAMP_TABLE, ]; // copy from table - // TODO not implemented because we dont have Table adapter yet yield 'copy from table' => [ new Table($this->getSourceDbName(), self::TABLE_OUT_CSV_2COLS, $escapingHeader), [$this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS], From 29efa45782af6de43ed9c23d8f4dc0c33b2bdaf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Tue, 22 Mar 2022 15:23:46 +0100 Subject: [PATCH 36/62] coalesce for strings only --- src/Backend/Teradata/ToFinalTable/SqlBuilder.php | 12 +++++++++++- tests/functional/Teradata/FullImportTest.php | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php index 0b5995bf..e49ee0a5 100644 --- a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php +++ b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php @@ -5,6 +5,7 @@ namespace Keboola\Db\ImportExport\Backend\Teradata\ToFinalTable; use Keboola\Datatype\Definition\BaseType; +use Keboola\Datatype\Definition\Teradata; use Keboola\Db\ImportExport\Backend\Teradata\TeradataImportOptions; use Keboola\Db\ImportExport\Backend\ToStageImporterInterface; use Keboola\TableBackendUtils\Column\Teradata\TeradataColumn; @@ -83,9 +84,18 @@ public function getInsertAllIntoTargetTableCommand( } else { $columnsSetSql[] = TeradataQuote::quoteSingleIdentifier($columnDefinition->getColumnName()); } + } elseif ($columnDefinition->getColumnDefinition()->getBasetype() === BaseType::STRING) { + $columnsSetSql[] = sprintf( + 'CAST(COALESCE(%s, \'\') as %s) AS %s', + TeradataQuote::quoteSingleIdentifier($columnDefinition->getColumnName()), + $columnDefinition->getColumnDefinition()->getSQLDefinition(), + TeradataQuote::quoteSingleIdentifier($columnDefinition->getColumnName()) + ); } else { + // on columns other than string dont use COALESCE, use direct cast + // this will fail if the column is not null, but this is expected $columnsSetSql[] = sprintf( - 'CAST(COALESCE(%s, \'\') AS %s) AS %s', + 'CAST(%s as %s) AS %s', TeradataQuote::quoteSingleIdentifier($columnDefinition->getColumnName()), $columnDefinition->getColumnDefinition()->getSQLDefinition(), TeradataQuote::quoteSingleIdentifier($columnDefinition->getColumnName()) diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index ef0badee..0fc838e7 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -350,7 +350,7 @@ public function fullImportData(): Generator $this->getDestinationDbName(), self::TABLE_OUT_CSV_2COLS, ], - $this->getImportOptions(), + $this->getSimpleImportOptions(ImportOptions::SKIP_NO_LINE), [ ['a', 'b', '2014-11-10 13:12:06.000000'], ['c', 'd', '2014-11-10 14:12:06.000000'], From 6e4d3a4cc944be7d81874a821fec69c518c45411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Tue, 22 Mar 2022 15:26:37 +0100 Subject: [PATCH 37/62] add aws keys --- azure-pipelines.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9a470d5b..047e92b6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -71,6 +71,9 @@ stages: - script: | docker-compose build --pull production displayName: 'Build project images' + env: + AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - script: | docker-compose run production php -v parallel -j12 --linebuffer docker-compose run production composer ::: \ From dabe77c13f5ffd9ceb51d8b8f160be9e44416750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Tue, 22 Mar 2022 15:51:50 +0100 Subject: [PATCH 38/62] set driver aws credentials --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 047e92b6..eeb48a13 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -72,8 +72,8 @@ stages: docker-compose build --pull production displayName: 'Build project images' env: - AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + AWS_ACCESS_KEY_ID: $(DRIVERS_AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(DRIVERS_AWS_SECRET_ACCESS_KEY) - script: | docker-compose run production php -v parallel -j12 --linebuffer docker-compose run production composer ::: \ From 91e07bbca86b0dc82340f3ff79dcae83e99b346e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Tue, 22 Mar 2022 16:46:48 +0100 Subject: [PATCH 39/62] cs --- .../Teradata/ToFinalTable/FullImporter.php | 5 ++- .../Teradata/ToFinalTable/SqlBuilder.php | 4 +-- .../Teradata/ToStage/FromS3TPTAdapter.php | 24 +++++++------ src/Storage/S3/SourceFile.php | 3 +- src/Storage/Teradata/Table.php | 2 +- tests/functional/Teradata/FullImportTest.php | 35 ++++--------------- tests/functional/Teradata/SqlBuilderTest.php | 6 ---- tests/functional/Teradata/StageImportTest.php | 20 +++++------ .../Teradata/TeradataBaseTestCase.php | 14 ++++---- tests/unit/Backend/Teradata/HelperTest.php | 2 +- tests/unit/Storage/S3/SourceFileTest.php | 2 +- 11 files changed, 46 insertions(+), 71 deletions(-) diff --git a/src/Backend/Teradata/ToFinalTable/FullImporter.php b/src/Backend/Teradata/ToFinalTable/FullImporter.php index fdcad35b..0072ceba 100644 --- a/src/Backend/Teradata/ToFinalTable/FullImporter.php +++ b/src/Backend/Teradata/ToFinalTable/FullImporter.php @@ -19,7 +19,7 @@ final class FullImporter implements ToFinalTableImporterInterface { private const TIMER_COPY_TO_TARGET = 'copyFromStagingToTarget'; - private const TIMER_DEDUP = 'fromStagingToTargetWithDedup'; +// private const TIMER_DEDUP = 'fromStagingToTargetWithDedup'; private const OPTIMIZED_LOAD_TMP_TABLE_SUFFIX = '_tmp'; private const OPTIMIZED_LOAD_RENAME_TABLE_SUFFIX = '_tmp_rename'; @@ -74,7 +74,6 @@ public function importToTable( assert($destinationTableDefinition instanceof TeradataTableDefinition); assert($options instanceof TeradataImportOptions); - /** @var TeradataTableDefinition $destinationTableDefinition */ try { //import files to staging table @@ -121,7 +120,7 @@ public function importToTable( return $state->getResult(); } - protected function tableExists($dbName, $tableName): bool + protected function tableExists(string $dbName, string $tableName): bool { $data = $this->connection->fetchOne($this->sqlBuilder->getTableExistsCommand($dbName, $tableName)); return ((int) $data) > 0; diff --git a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php index e49ee0a5..fa7ab096 100644 --- a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php +++ b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php @@ -14,7 +14,7 @@ class SqlBuilder { - const SRC_ALIAS = 'src'; + private const SRC_ALIAS = 'src'; public function getCommitTransaction(): string { @@ -145,7 +145,7 @@ public function getColumnsString( }, $columns)); } - public function getDeleteOldItemsCommand() + public function getDeleteOldItemsCommand(): void { throw new \Exception('not implemented yet'); } diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index 321e2634..79f8afbd 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -48,7 +48,7 @@ public function runCopyCommand( assert($importOptions instanceof TeradataImportOptions); // empty manifest. TPT cannot import empty data - if($source->isSliced() && count($source->getManifestEntries()) === 0){ + if ($source->isSliced() && count($source->getManifestEntries()) === 0) { return 0; } @@ -122,16 +122,18 @@ public function runCopyCommand( if ($process->getExitCode() !== 0 || $errContent || $err2Content) { $qb = new TeradataTableQueryBuilder(); // drop destination table it's not usable - $this->connection->executeStatement($qb->getDropTableCommand( - $destination->getSchemaName(), - $destination->getTableName() - )); +// $this->connection->executeStatement($qb->getDropTableCommand( +// $destination->getSchemaName(), +// $destination->getTableName() +// )); throw new FailedTPTLoadException( $process->getErrorOutput(), $process->getOutput(), $process->getExitCode(), - file_exists($temp->getTmpFolder() . '/import-1.out') ? file_get_contents($temp->getTmpFolder() . '/import-1.out') : 'unable to get error', + file_exists($temp->getTmpFolder() . '/import-1.out') + ? file_get_contents($temp->getTmpFolder() . '/import-1.out') + : 'unable to get error', $logContent, $errContent, $err2Content @@ -159,13 +161,12 @@ private function generateTPTScript( $temp->initRunFolder(); $folder = $temp->getTmpFolder(); $target = sprintf( - "%s.%s", + '%s.%s', TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()), TeradataQuote::quoteSingleIdentifier($destination->getTableName()), ); if ($source->isSliced()) { - $moduleStr = sprintf( 'AccessModuleInitStr = \'S3Region=%s S3Bucket=%s S3Prefix="" S3Object=%s S3SinglePartFile=True\'', $source->getRegion(), @@ -224,13 +225,14 @@ private function generateTPTScript( } $ignoredLines = $importOptions->getNumberOfIgnoredLines(); + $quotedDestination = TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()); $tablesPrefix = BackendHelper::generateTempTableName(); $logTable = $tablesPrefix . '_log'; - $logTableQuoted = TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()) . '.' . TeradataQuote::quoteSingleIdentifier($logTable); + $logTableQuoted = $quotedDestination . '.' . TeradataQuote::quoteSingleIdentifier($logTable); $errTable1 = $tablesPrefix . '_e1'; - $errTableQuoted = TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()) . '.' . TeradataQuote::quoteSingleIdentifier($errTable1); + $errTableQuoted = $quotedDestination . '.' . TeradataQuote::quoteSingleIdentifier($errTable1); $errTable2 = $tablesPrefix . '_e2'; - $errTable2Quoted = TeradataQuote::quoteSingleIdentifier($destination->getSchemaName()) . '.' . TeradataQuote::quoteSingleIdentifier($errTable2); + $errTable2Quoted = $quotedDestination . '.' . TeradataQuote::quoteSingleIdentifier($errTable2); $jobVariableFile = <<filePath; - if (($prefixLength = strrpos($prefix, '/')) !== false) { + $prefixLength = strrpos($prefix, '/'); + if ($prefixLength !== false) { // include / at the end return substr($prefix, 0, $prefixLength + 1); } diff --git a/src/Storage/Teradata/Table.php b/src/Storage/Teradata/Table.php index 1ac98431..fb46bfd6 100644 --- a/src/Storage/Teradata/Table.php +++ b/src/Storage/Teradata/Table.php @@ -47,7 +47,7 @@ public function getFromStatement(): string $quotedColumns = array_map(static function ($column) { // trim because implicit casting adds right padding spaces // value 10.5 as DECIMAL(8,1) implicitly casted to varchar would be then " 10.5" - return sprintf("TRIM(%s)", TeradataQuote::quoteSingleIdentifier($column)); + return sprintf('TRIM(%s)', TeradataQuote::quoteSingleIdentifier($column)); }, $colums); $select = implode(', ', $quotedColumns); } diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index 0fc838e7..ea35e522 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -34,7 +34,7 @@ protected function setUp(): void $this->createDatabase($this->getSourceDbName()); } - protected function tearDown() + protected function tearDown(): void { $this->connection->close(); parent::tearDown(); @@ -460,11 +460,11 @@ public function testFullImportWithDataSet( ); } finally { if ($this->connection->fetchOne( - (new SqlBuilder())->getTableExistsCommand( - $stagingTable->getSchemaName(), - $stagingTable->getTableName() - ) - ) > 0) { + (new SqlBuilder())->getTableExistsCommand( + $stagingTable->getSchemaName(), + $stagingTable->getTableName() + ) + ) > 0) { $this->connection->executeStatement((new SqlBuilder())->getDropTableUnsafe( $stagingTable->getSchemaName(), $stagingTable->getTableName() @@ -482,27 +482,4 @@ public function testFullImportWithDataSet( 0 ); } -// -// public function testFetch() -// { -// $db = self::TERADATA_SOURCE_DATABASE_NAME; -// $name = "jirka" . time(); -// $this->initSingleTable($name, "transactions"); -// $this->connection->fetchAllAssociative(sprintf( -// 'SELECT * FROM "'.$name.'"."transactions";' -// )); -//// $this->connection->fetchAllAssociative(sprintf( -//// 'SELECT * FROM "jirkadatabazeaa"."jirkatabulka";' -//// )); -// -// -//// $this->cleanDatabase($db); -//// $this->createDatabase($db); -//// $table = self::TABLE_TABLE; -//// $this->initSingleTable(); -//// $count = $this->connection->fetchOne(sprintf("select count(*) from %s.%s", TeradataQuote::quoteSingleIdentifier($db), -//// TeradataQuote::quoteSingleIdentifier($table))); -//// $data = $this->connection->fetchAllAssociative(sprintf('select "Other" from %s.%s', TeradataQuote::quoteSingleIdentifier($db), -//// TeradataQuote::quoteSingleIdentifier($table))); -// } } diff --git a/tests/functional/Teradata/SqlBuilderTest.php b/tests/functional/Teradata/SqlBuilderTest.php index 1771055f..6e1fb056 100644 --- a/tests/functional/Teradata/SqlBuilderTest.php +++ b/tests/functional/Teradata/SqlBuilderTest.php @@ -132,11 +132,6 @@ private function createStagingTableWithData(bool $includeEmptyValues = false): T return $def; } - private function getDummyImportOptions(): TeradataImportOptions - { - return new TeradataImportOptions(); - } - public function testGetDeleteOldItemsCommand(): void { $this->createTestDb(); @@ -322,7 +317,6 @@ public function testGetInsertAllIntoTargetTableCommand(): void $out = $this->connection->executeStatement($sql); self::assertEquals(4, $out); - $result = $this->connection->fetchAllAssociative(sprintf( 'SELECT * FROM %s.%s', TeradataQuote::quoteSingleIdentifier(self::TEST_DB), diff --git a/tests/functional/Teradata/StageImportTest.php b/tests/functional/Teradata/StageImportTest.php index 6471cd9c..37566d00 100644 --- a/tests/functional/Teradata/StageImportTest.php +++ b/tests/functional/Teradata/StageImportTest.php @@ -41,16 +41,16 @@ public function testSimpleStageImport(): void self::TABLE_GENERIC ); - $state = $importer->importToStagingTable( - $this->createS3SourceInstanceFromCsv('csv/simple/a_b_c-1row.csv', new CsvOptions()), - $ref->getTableDefinition(), - $this->getImportOptions( - [], - false, - false, - 1 - ) - ); + $state = $importer->importToStagingTable( + $this->createS3SourceInstanceFromCsv('csv/simple/a_b_c-1row.csv', new CsvOptions()), + $ref->getTableDefinition(), + $this->getImportOptions( + [], + false, + false, + 1 + ) + ); $this->assertEquals(1, $state->getResult()->getImportedRowsCount()); } diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index 0f1a34b5..36beb558 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -101,7 +101,7 @@ public function getGenericTableDefinition( array $columns, array $pks = [] ): TeradataTableDefinition { - return new TeradataTableDefinition ( + return new TeradataTableDefinition( $schemaName, $tableName, false, @@ -117,9 +117,13 @@ protected function cleanDatabase(string $dbname): void } // delete all objects in the DB - $this->connection->executeQuery(sprintf('DELETE DATABASE %s ALL', TeradataQuote::quoteSingleIdentifier($dbname))); + $this->connection->executeQuery( + sprintf('DELETE DATABASE %s ALL', TeradataQuote::quoteSingleIdentifier($dbname)) + ); // drop the empty db - $this->connection->executeQuery(sprintf('DROP DATABASE %s', TeradataQuote::quoteSingleIdentifier($dbname))); + $this->connection->executeQuery( + sprintf('DROP DATABASE %s', TeradataQuote::quoteSingleIdentifier($dbname)) + ); } public function createDatabase(string $dbName): void @@ -139,7 +143,7 @@ protected function dbExists(string $dbname): bool return true; } catch (\Doctrine\DBAL\Exception $e) { // https://docs.teradata.com/r/GVKfXcemJFkTJh_89R34UQ/j2TdlzqRJ9LpndY3efMdlw - if (strpos($e->getMessage(), "3802")) { + if (strpos($e->getMessage(), '3802')) { return false; } throw $e; @@ -401,7 +405,6 @@ protected function getImportOptions( bool $isIncremental = false, bool $useTimestamp = false, int $numberOfIgnoredLines = 0 - ): TeradataImportOptions { return new TeradataImportOptions( @@ -490,5 +493,4 @@ protected function assertTeradataTableEqualsExpected( $message ); } - } diff --git a/tests/unit/Backend/Teradata/HelperTest.php b/tests/unit/Backend/Teradata/HelperTest.php index add7241c..802290d4 100644 --- a/tests/unit/Backend/Teradata/HelperTest.php +++ b/tests/unit/Backend/Teradata/HelperTest.php @@ -12,7 +12,7 @@ class HelperTest extends TestCase /** * @dataProvider dataProvider */ - public function testGetResult($expected, $entries): void + public function testGetResult(string $expected, array $entries): void { $this->assertEquals($expected, BackendHelper::getMask($entries)); } diff --git a/tests/unit/Storage/S3/SourceFileTest.php b/tests/unit/Storage/S3/SourceFileTest.php index e429111f..1156c0cc 100644 --- a/tests/unit/Storage/S3/SourceFileTest.php +++ b/tests/unit/Storage/S3/SourceFileTest.php @@ -25,7 +25,7 @@ public function testDefaultValues(): void self::assertNull($source->getPrimaryKeysNames()); } - public function testGetFilepathParts() + public function testGetFilepathParts(): void { $source = $this->createDummyS3SourceInstance('data/shared/file.csv'); self::assertEquals('data/shared/', $source->getPrefix()); From 0654d2f8cc4394ca9ec87f32b151ef682105ee14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Tue, 22 Mar 2022 17:15:08 +0100 Subject: [PATCH 40/62] stan part1 --- .../Exception/FailedTPTLoadException.php | 17 +++++++++++++++ .../Teradata/ToStage/FromS3TPTAdapter.php | 21 ++++++++++++------- tests/unit/Backend/Teradata/HelperTest.php | 10 +++++++-- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php b/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php index cff2e7ab..b4fc5ae2 100644 --- a/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php +++ b/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php @@ -16,12 +16,20 @@ class FailedTPTLoadException extends Exception private ?int $exitCode; + /** @var mixed[]|null */ private ?array $logTableContent; + /** @var mixed[]|null */ private ?array $errTableContent; + /** @var mixed[]|null */ private ?array $errTable2Content; + /** + * @param mixed[]|null $logTableContent + * @param mixed[]|null $errTableContent + * @param mixed[]|null $errTable2Content + */ public function __construct( string $stdErr, string $stdOut, @@ -61,16 +69,25 @@ public function getExitCode(): ?int return $this->exitCode; } + /** + * @return array|string[]|null + */ public function getLogTableContent(): ?array { return $this->logTableContent; } + /** + * @return array|string[]|null + */ public function getErrTableContent(): ?array { return $this->errTableContent; } + /** + * @return array|string[]|null + */ public function getErrTable2Content(): ?array { return $this->errTable2Content; diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index 79f8afbd..fb05ee85 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -122,18 +122,16 @@ public function runCopyCommand( if ($process->getExitCode() !== 0 || $errContent || $err2Content) { $qb = new TeradataTableQueryBuilder(); // drop destination table it's not usable -// $this->connection->executeStatement($qb->getDropTableCommand( -// $destination->getSchemaName(), -// $destination->getTableName() -// )); + $this->connection->executeStatement($qb->getDropTableCommand( + $destination->getSchemaName(), + $destination->getTableName() + )); throw new FailedTPTLoadException( $process->getErrorOutput(), $process->getOutput(), $process->getExitCode(), - file_exists($temp->getTmpFolder() . '/import-1.out') - ? file_get_contents($temp->getTmpFolder() . '/import-1.out') - : 'unable to get error', + $this->getLogData($temp), $logContent, $errContent, $err2Content @@ -149,6 +147,15 @@ public function runCopyCommand( return $ref->getRowsCount(); } + private function getLogData(Temp $temp): string + { + if (file_exists($temp->getTmpFolder() . '/import-1.out')) { + return file_get_contents($temp->getTmpFolder() . '/import-1.out') ?: 'unable to get error'; + } + + return 'unable to get error'; + } + /** * @return array{0: Temp, 1:string, 2:string, 3:string, 4: string[]} */ diff --git a/tests/unit/Backend/Teradata/HelperTest.php b/tests/unit/Backend/Teradata/HelperTest.php index 802290d4..d89271e4 100644 --- a/tests/unit/Backend/Teradata/HelperTest.php +++ b/tests/unit/Backend/Teradata/HelperTest.php @@ -5,16 +5,22 @@ namespace Tests\Keboola\Db\ImportExportUnit\Backend\Teradata; use Keboola\Db\ImportExport\Backend\Teradata\Helper\BackendHelper; +use Keboola\Db\ImportExport\Storage\S3\SourceFile; use PHPUnit\Framework\TestCase; class HelperTest extends TestCase { /** + * @param string[] $entries * @dataProvider dataProvider */ - public function testGetResult(string $expected, array $entries): void + public function testGetResult(string $expected, array $entriesData): void { - $this->assertEquals($expected, BackendHelper::getMask($entries)); + $mock = $this->createMock(SourceFile::class); + + $mock->method('getManifestEntries') + ->willReturn($entriesData); + $this->assertEquals($expected, BackendHelper::getMask($mock)); } public function dataProvider(): array From b5beef48a8ce06ea6f23ad2f98441c87608f0884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Wed, 23 Mar 2022 11:04:08 +0100 Subject: [PATCH 41/62] tmp fix for import test --- tests/functional/Teradata/FullImportTest.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index ea35e522..e43a0a04 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -406,7 +406,8 @@ public function fullImportData(): Generator self::TABLE_TYPES, ], $this->getSimpleImportOptions(), - [['a', '10.5', '0.3', '1']], + // TODO https://keboola.atlassian.net/browse/KBC-2526 it should cast 0.3 to "0.3" + [['a', '10.5', '3.00000000000000E-001', '1']], 1, self::TABLE_TYPES, ]; From d680a5b4f1f3bbcbb0b4ec0954a496ec267c5532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Wed, 23 Mar 2022 11:12:05 +0100 Subject: [PATCH 42/62] allow crlf test --- tests/functional/Teradata/FullImportTest.php | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index e43a0a04..4f0a4db1 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -256,20 +256,20 @@ public function fullImportData(): Generator ]; // line ending detection is not supported yet for S3 - //yield 'accounts crlf' => [ - // $this->createS3SourceInstance( - // 'tw_accounts.crlf.csv', - // $accountsHeader, - // false, - // false, - // ['id'] - // ), - // [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], - // $this->getImportOptions(), - // $expectedAccounts, - // 3, - // self::TABLE_ACCOUNTS_3, - //]; + yield 'accounts crlf' => [ + $this->createS3SourceInstance( + 'tw_accounts.crlf.csv', + $accountsHeader, + false, + false, + ['id'] + ), + [$this->getDestinationDbName(), self::TABLE_ACCOUNTS_3], + $this->getSimpleImportOptions(), + $expectedAccounts, + 3, + self::TABLE_ACCOUNTS_3, + ]; // manifests yield 'accounts sliced' => [ From e00a0f5659baf5dc4913bed8ed34ec775e053061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Wed, 23 Mar 2022 13:08:54 +0100 Subject: [PATCH 43/62] stan and cs --- .../Teradata/ToFinalTable/SqlBuilder.php | 6 +- tests/functional/Teradata/FullImportTest.php | 1 + tests/functional/Teradata/SqlBuilderTest.php | 131 +----------------- .../Teradata/TeradataBaseTestCase.php | 59 +------- tests/unit/Backend/Teradata/HelperTest.php | 5 +- 5 files changed, 14 insertions(+), 188 deletions(-) diff --git a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php index fa7ab096..8fe79a0d 100644 --- a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php +++ b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php @@ -145,8 +145,10 @@ public function getColumnsString( }, $columns)); } - public function getDeleteOldItemsCommand(): void - { + public function getDeleteOldItemsCommand( + TeradataTableDefinition $stagingTableDefinition, + TeradataTableDefinition $destinationTableDefinition + ): void { throw new \Exception('not implemented yet'); } } diff --git a/tests/functional/Teradata/FullImportTest.php b/tests/functional/Teradata/FullImportTest.php index 4f0a4db1..e342f97b 100644 --- a/tests/functional/Teradata/FullImportTest.php +++ b/tests/functional/Teradata/FullImportTest.php @@ -66,6 +66,7 @@ public function testLoadToFinalTableWithoutDedup(): void $this->getDestinationDbName(), self::TABLE_TRANSLATIONS ); + /** @var TeradataTableDefinition $destination */ $destination = $destinationRef->getTableDefinition(); $stagingTable = StageTableDefinitionFactory::createStagingTableDefinition($destination, [ 'id', diff --git a/tests/functional/Teradata/SqlBuilderTest.php b/tests/functional/Teradata/SqlBuilderTest.php index 6e1fb056..ba6242fd 100644 --- a/tests/functional/Teradata/SqlBuilderTest.php +++ b/tests/functional/Teradata/SqlBuilderTest.php @@ -51,44 +51,7 @@ protected function createTestDb(): void public function testGetDedupCommand(): void { - $this->markTestSkipped('not impl'); - $this->createTestDb(); - $stageDef = $this->createStagingTableWithData(); - - $deduplicationDef = new TeradataTableDefinition( - self::TEST_DB, - 'tempTable', - true, - new ColumnCollection([ - TeradataColumn::createGenericColumn('col1'), - TeradataColumn::createGenericColumn('col2'), - ]), - [ - 'pk1', - 'pk2', - ] - ); - $qb = new TeradataTableQueryBuilder(); - $this->connection->executeStatement($qb->getCreateTableCommandFromDefinition($deduplicationDef)); - - $sql = $this->getBuilder()->getDedupCommand( - $stageDef, - $deduplicationDef, - $deduplicationDef->getPrimaryKeysNames() - ); - self::assertEquals( - // phpcs:ignore - 'INSERT INTO "import-export-test_schema"."tempTable" ("col1", "col2") SELECT a."col1",a."col2" FROM (SELECT "col1", "col2", ROW_NUMBER() OVER (PARTITION BY "pk1","pk2" ORDER BY "pk1","pk2") AS "_row_number_" FROM "import-export-test_schema"."stagingTable") AS a WHERE a."_row_number_" = 1', - $sql - ); - $this->connection->executeStatement($sql); - $result = $this->connection->fetchAllAssociative(sprintf( - 'SELECT * FROM %s.%s', - TeradataQuote::quoteSingleIdentifier(self::TEST_DB), - TeradataQuote::quoteSingleIdentifier($deduplicationDef->getTableName()) - )); - - self::assertCount(2, $result); + $this->markTestSkipped('not implemented'); } private function createStagingTableWithData(bool $includeEmptyValues = false): TeradataTableDefinition @@ -134,97 +97,7 @@ private function createStagingTableWithData(bool $includeEmptyValues = false): T public function testGetDeleteOldItemsCommand(): void { - $this->createTestDb(); - - $tableDefinition = new TeradataTableDefinition( - self::TEST_DB, - self::TEST_TABLE, - false, - new ColumnCollection([ - new TeradataColumn( - 'id', - new Teradata( - Teradata::TYPE_INT - ) - ), - TeradataColumn::createGenericColumn('pk1'), - TeradataColumn::createGenericColumn('pk2'), - TeradataColumn::createGenericColumn('col1'), - TeradataColumn::createGenericColumn('col2'), - ]), - ['pk1', 'pk2'] - ); - $tableSql = sprintf( - '%s.%s', - TeradataQuote::quoteSingleIdentifier(self::TEST_DB), - TeradataQuote::quoteSingleIdentifier($tableDefinition->getTableName()) - ); - $qb = new TeradataTableQueryBuilder(); - $this->connection->executeStatement($qb->getCreateTableCommandFromDefinition($tableDefinition)); - $this->connection->executeStatement( - sprintf( - 'INSERT INTO %s("id","pk1","pk2","col1","col2") VALUES (1,1,1,\'1\',\'1\')', - $tableSql - ) - ); - $stagingTableDefinition = new TeradataTableDefinition( - self::TEST_DB, - self::TEST_STAGING_TABLE, - false, - new ColumnCollection([ - TeradataColumn::createGenericColumn('pk1'), - TeradataColumn::createGenericColumn('pk2'), - TeradataColumn::createGenericColumn('col1'), - TeradataColumn::createGenericColumn('col2'), - ]), - ['pk1', 'pk2'] - ); - $this->connection->executeStatement($qb->getCreateTableCommandFromDefinition($stagingTableDefinition)); - $stagingTableSql = sprintf( - '%s.%s', - // TODO where the getDbName came from - TeradataQuote::quoteSingleIdentifier(self::TEST_DB), - TeradataQuote::quoteSingleIdentifier($stagingTableDefinition->getTableName()) - ); - $this->connection->executeStatement( - sprintf( - 'INSERT INTO %s("pk1","pk2","col1","col2") VALUES (1,1,\'1\',\'1\')', - $stagingTableSql - ) - ); - $this->connection->executeStatement( - sprintf( - 'INSERT INTO %s("pk1","pk2","col1","col2") VALUES (2,1,\'1\',\'1\')', - $stagingTableSql - ) - ); - - $sql = $this->getBuilder()->getDeleteOldItemsCommand( - $stagingTableDefinition, - $tableDefinition - ); - - self::assertEquals( - // phpcs:ignore - 'DELETE FROM "import-export-test_schema"."stagingTable" WHERE EXISTS (SELECT * FROM "import-export-test_schema"."import-export-test_test" WHERE COALESCE("import-export-test_schema"."import-export-test_test"."pk1", \'KBC_$#\') = COALESCE("import-export-test_schema"."stagingTable"."pk1", \'KBC_$#\') AND COALESCE("import-export-test_schema"."import-export-test_test"."pk2", \'KBC_$#\') = COALESCE("import-export-test_schema"."stagingTable"."pk2", \'KBC_$#\'))', - $sql - ); - $this->connection->executeStatement($sql); - - $result = $this->connection->fetchAllAssociative(sprintf( - 'SELECT * FROM %s', - $stagingTableSql - )); - - self::assertCount(1, $result); - self::assertSame([ - [ - 'pk1' => '2', - 'pk2' => '1', - 'col1' => '1', - 'col2' => '1', - ], - ], $result); + $this->markTestSkipped('not implemented'); } private function assertTableNotExists(string $schemaName, string $tableName): void diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index 36beb558..e9a9e3b0 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -150,62 +150,6 @@ protected function dbExists(string $dbname): bool } } - /** - * @param int|string $sortKey - * @param array $expected - * @param string|int $sortKey - */ - protected function assertSynapseTableEqualsExpected( - SourceInterface $source, - TeradataTableDefinition $destination, - ImportOptions $options, - array $expected, - $sortKey, - string $message = 'Imported tables are not the same as expected' - ): void { - $tableColumns = (new TeradataTableReflection( - $this->connection, - $destination->getDbName(), - $destination->getTableName() - ))->getColumnsNames(); - - if ($options->useTimestamp()) { - self::assertContains('_timestamp', $tableColumns); - } else { - self::assertNotContains('_timestamp', $tableColumns); - } - - if (!in_array('_timestamp', $source->getColumnsNames())) { - $tableColumns = array_filter($tableColumns, function ($column) { - return $column !== '_timestamp'; - }); - } - - $tableColumns = array_map(function ($column) { - return sprintf('[%s]', $column); - }, $tableColumns); - - $sql = sprintf( - 'SELECT %s FROM [%s].[%s]', - implode(', ', $tableColumns), - $destination->getDbName(), - $destination->getTableName() - ); - - $queryResult = array_map(function ($row) { - return array_map(function ($column) { - return $column; - }, array_values($row)); - }, $this->connection->fetchAll($sql)); - - $this->assertArrayEqualsSorted( - $expected, - $queryResult, - $sortKey, - $message - ); - } - protected function initSingleTable( string $db = self::TERADATA_SOURCE_DATABASE_NAME, string $table = self::TABLE_TABLE @@ -400,6 +344,9 @@ protected function initTable(string $tableName): void } } + /** + * @param string[] $convertEmptyValuesToNull + */ protected function getImportOptions( array $convertEmptyValuesToNull = [], bool $isIncremental = false, diff --git a/tests/unit/Backend/Teradata/HelperTest.php b/tests/unit/Backend/Teradata/HelperTest.php index d89271e4..b103346a 100644 --- a/tests/unit/Backend/Teradata/HelperTest.php +++ b/tests/unit/Backend/Teradata/HelperTest.php @@ -11,7 +11,7 @@ class HelperTest extends TestCase { /** - * @param string[] $entries + * @param string[] $entriesData * @dataProvider dataProvider */ public function testGetResult(string $expected, array $entriesData): void @@ -23,6 +23,9 @@ public function testGetResult(string $expected, array $entriesData): void $this->assertEquals($expected, BackendHelper::getMask($mock)); } + /** + * @return array[] + */ public function dataProvider(): array { return [ From 870285ad89790f804e76a18024794ed32e012771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Wed, 23 Mar 2022 14:56:29 +0100 Subject: [PATCH 44/62] build length and type --- src/Backend/Teradata/ToFinalTable/SqlBuilder.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php index 8fe79a0d..42fd7e79 100644 --- a/src/Backend/Teradata/ToFinalTable/SqlBuilder.php +++ b/src/Backend/Teradata/ToFinalTable/SqlBuilder.php @@ -88,7 +88,7 @@ public function getInsertAllIntoTargetTableCommand( $columnsSetSql[] = sprintf( 'CAST(COALESCE(%s, \'\') as %s) AS %s', TeradataQuote::quoteSingleIdentifier($columnDefinition->getColumnName()), - $columnDefinition->getColumnDefinition()->getSQLDefinition(), + $columnDefinition->getColumnDefinition()->buildTypeWithLength(), TeradataQuote::quoteSingleIdentifier($columnDefinition->getColumnName()) ); } else { @@ -97,7 +97,7 @@ public function getInsertAllIntoTargetTableCommand( $columnsSetSql[] = sprintf( 'CAST(%s as %s) AS %s', TeradataQuote::quoteSingleIdentifier($columnDefinition->getColumnName()), - $columnDefinition->getColumnDefinition()->getSQLDefinition(), + $columnDefinition->getColumnDefinition()->buildTypeWithLength(), TeradataQuote::quoteSingleIdentifier($columnDefinition->getColumnName()) ); } From ed61df62eb7f8c01fa8c27ba1dccf134d1f00d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Wed, 23 Mar 2022 15:01:40 +0100 Subject: [PATCH 45/62] run TD tests in pipeline --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index eeb48a13..5856b349 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -141,6 +141,7 @@ stages: tests-synapse-heap4000temp-optimized \ tests-synapse-heap4000temp-optimized-hash \ tests-synapse-next + tests-teradata PARALLEL_EXIT_CODE=$? cat /tmp/parallel-joblog sleep 1 From f0abc5f8eb2ca4e4a636e6e7f108c3b96218390a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Thu, 24 Mar 2022 11:09:19 +0100 Subject: [PATCH 46/62] fix suite --- phpunit.xml.dist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 4a0eb67b..a20d82d0 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -61,7 +61,7 @@ tests/functional/Synapse/SqlCommandBuilderTest.php - + tests/functional/Teradata From 23a3af928989b303a11cd94bb175d080e7691a57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Thu, 24 Mar 2022 11:25:09 +0100 Subject: [PATCH 47/62] run tests --- azure-pipelines.yml | 2 +- composer.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5856b349..5b7521f6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -140,7 +140,7 @@ stages: tests-synapse-heap4000temp-hash \ tests-synapse-heap4000temp-optimized \ tests-synapse-heap4000temp-optimized-hash \ - tests-synapse-next + tests-synapse-next \ tests-teradata PARALLEL_EXIT_CODE=$? cat /tmp/parallel-joblog diff --git a/composer.json b/composer.json index 736f9056..f35d07aa 100644 --- a/composer.json +++ b/composer.json @@ -49,7 +49,7 @@ "tests-synapse-clusterdindextemp": "SUITE=tests-synapse-clusterdindextemp CREDENTIALS_IMPORT_TYPE=SAS CREDENTIALS_EXPORT_TYPE=MASTER_KEY TEMP_TABLE_TYPE=CLUSTERED_INDEX DEDUP_TYPE=TMP_TABLE phpunit --colors=always --testsuite tests-synapse-clusterdindextemp", "tests-synapse-mi": "SUITE=tests-synapse-mi CREDENTIALS_IMPORT_TYPE=MANAGED_IDENTITY CREDENTIALS_EXPORT_TYPE=MANAGED_IDENTITY TEMP_TABLE_TYPE=HEAP DEDUP_TYPE=TMP_TABLE phpunit --colors=always --testsuite synapse-mi", "tests-exasol": "SUITE=tests-exasol STORAGE_TYPE=S3 phpunit --colors=always --testsuite exasol", - "tests-teradata": "SUITE=tests-teradata STORAGE_TYPE=S3 phpunit --colors=always --testsuite teradata", + "tests-teradata": "SUITE=tests-teradata STORAGE_TYPE=S3 phpunit --colors=always --testsuite tests-teradata", "tests-functional": [ "@tests-snowflake-abs", "@tests-snowflake-s3", From 1ab81fcee465f979f95c97204f8897a558c8fb80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Thu, 24 Mar 2022 13:07:06 +0100 Subject: [PATCH 48/62] add TD credentials --- azure-pipelines.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5b7521f6..fb804937 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -170,6 +170,10 @@ stages: EXASOL_HOST: $(EXASOL_CLUSTER_DNS) EXASOL_USERNAME: devel EXASOL_PASSWORD: $(EXA_SAAS_TOKEN) + TERADATA_HOST: $(TERADATA_HOST) + TERADATA_PASSWORD: $(TERADATA_PASSWORD) + TERADATA_PORT: $(TERADATA_PORT) + TERADATA_USERNAME: $(TERADATA_USERNAME) - script: | docker-compose stop php ./provisioning/cli.php app:delete:synapse \ From 88328e9b72b887e1381185a3b6fa50d2703009d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Thu, 24 Mar 2022 15:07:57 +0100 Subject: [PATCH 49/62] try to rerun --- azure-pipelines.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fb804937..3d4d566e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -129,18 +129,6 @@ stages: EXASOL_USERNAME=devel docker-compose run production php -v parallel -j12 --joblog /tmp/parallel-joblog --linebuffer docker-compose run production composer ::: \ - tests-storage \ - tests-snowflake-abs \ - tests-snowflake-s3 \ - tests-synapse \ - tests-synapse-mi \ - tests-synapse-columnstoretemp \ - tests-synapse-clusterdindextemp \ - tests-synapse-heap4000temp \ - tests-synapse-heap4000temp-hash \ - tests-synapse-heap4000temp-optimized \ - tests-synapse-heap4000temp-optimized-hash \ - tests-synapse-next \ tests-teradata PARALLEL_EXIT_CODE=$? cat /tmp/parallel-joblog From e1b8e76b32213efb3c8fd7ee9c339cf362219ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Thu, 24 Mar 2022 16:55:58 +0100 Subject: [PATCH 50/62] add tear down --- tests/functional/Teradata/TeradataBaseTestCase.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index e9a9e3b0..1380393c 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -440,4 +440,11 @@ protected function assertTeradataTableEqualsExpected( $message ); } + + protected function tearDown() + { + $this->cleanDatabase($this->getDestinationDbName()); + $this->cleanDatabase($this->getSourceDbName()); + parent::tearDown(); + } } From d0c8efe351065d5838956277e44085994e8e4203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Thu, 24 Mar 2022 17:20:53 +0100 Subject: [PATCH 51/62] void --- tests/functional/Teradata/TeradataBaseTestCase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index 1380393c..df969a8e 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -441,7 +441,7 @@ protected function assertTeradataTableEqualsExpected( ); } - protected function tearDown() + protected function tearDown(): void { $this->cleanDatabase($this->getDestinationDbName()); $this->cleanDatabase($this->getSourceDbName()); From aa2a965f560fa5b0a386f2f98c15819a9e03dbda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Thu, 24 Mar 2022 20:20:44 +0100 Subject: [PATCH 52/62] fix test --- tests/functional/Teradata/SqlBuilderTest.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/Teradata/SqlBuilderTest.php b/tests/functional/Teradata/SqlBuilderTest.php index ba6242fd..2b465563 100644 --- a/tests/functional/Teradata/SqlBuilderTest.php +++ b/tests/functional/Teradata/SqlBuilderTest.php @@ -183,7 +183,7 @@ public function testGetInsertAllIntoTargetTableCommand(): void self::assertEquals( // phpcs:ignore - 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2") SELECT CAST(COALESCE("col1", \'\') AS VARCHAR (4000)) AS "col1",CAST(COALESCE("col2", \'\') AS VARCHAR (4000)) AS "col2" FROM "import-export-test_schema"."stagingTable" AS "src"', + 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2") SELECT CAST(COALESCE("col1", \'\') as VARCHAR (4000)) AS "col1",CAST(COALESCE("col2", \'\') as VARCHAR (4000)) AS "col2" FROM "import-export-test_schema"."stagingTable" AS "src"', $sql ); @@ -303,7 +303,7 @@ public function testGetInsertAllIntoTargetTableCommandConvertToNull(): void ); self::assertEquals( // phpcs:ignore - 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2") SELECT NULLIF("col1", \'\'),CAST(COALESCE("col2", \'\') AS VARCHAR (4000)) AS "col2" FROM "import-export-test_schema"."stagingTable" AS "src"', + 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2") SELECT NULLIF("col1", \'\'),CAST(COALESCE("col2", \'\') as VARCHAR (4000)) AS "col2" FROM "import-export-test_schema"."stagingTable" AS "src"', $sql ); $out = $this->connection->executeStatement($sql); @@ -365,7 +365,7 @@ public function testGetInsertAllIntoTargetTableCommandConvertToNullWithTimestamp ); self::assertEquals( // phpcs:ignore - 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2", "_timestamp") SELECT NULLIF("col1", \'\'),CAST(COALESCE("col2", \'\') AS VARCHAR (4000)) AS "col2",\'2020-01-01 00:00:00\' FROM "import-export-test_schema"."stagingTable" AS "src"', + 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2", "_timestamp") SELECT NULLIF("col1", \'\'),CAST(COALESCE("col2", \'\') as VARCHAR (4000)) AS "col2",\'2020-01-01 00:00:00\' FROM "import-export-test_schema"."stagingTable" AS "src"', $sql ); $out = $this->connection->executeStatement($sql); From 887108001b1397e53aca68b24e87a5664b84a38e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Thu, 24 Mar 2022 23:24:43 +0100 Subject: [PATCH 53/62] fix test --- tests/functional/Teradata/SqlBuilderTest.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional/Teradata/SqlBuilderTest.php b/tests/functional/Teradata/SqlBuilderTest.php index 2b465563..c84a3c6b 100644 --- a/tests/functional/Teradata/SqlBuilderTest.php +++ b/tests/functional/Teradata/SqlBuilderTest.php @@ -183,7 +183,7 @@ public function testGetInsertAllIntoTargetTableCommand(): void self::assertEquals( // phpcs:ignore - 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2") SELECT CAST(COALESCE("col1", \'\') as VARCHAR (4000)) AS "col1",CAST(COALESCE("col2", \'\') as VARCHAR (4000)) AS "col2" FROM "import-export-test_schema"."stagingTable" AS "src"', + 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2") SELECT CAST(COALESCE("col1", \'\') as VARCHAR (50)) AS "col1",CAST(COALESCE("col2", \'\') as VARCHAR (50)) AS "col2" FROM "import-export-test_schema"."stagingTable" AS "src"', $sql ); @@ -264,7 +264,7 @@ private function createNullableGenericColumn(string $columnName): TeradataColumn $definition = new Teradata( Teradata::TYPE_VARCHAR, [ - 'length' => '4000', // should be changed to max in future + 'length' => '50', // should be changed to max in future 'nullable' => true, ] ); @@ -303,7 +303,7 @@ public function testGetInsertAllIntoTargetTableCommandConvertToNull(): void ); self::assertEquals( // phpcs:ignore - 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2") SELECT NULLIF("col1", \'\'),CAST(COALESCE("col2", \'\') as VARCHAR (4000)) AS "col2" FROM "import-export-test_schema"."stagingTable" AS "src"', + 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2") SELECT NULLIF("col1", \'\'),CAST(COALESCE("col2", \'\') as VARCHAR (50)) AS "col2" FROM "import-export-test_schema"."stagingTable" AS "src"', $sql ); $out = $this->connection->executeStatement($sql); @@ -365,7 +365,7 @@ public function testGetInsertAllIntoTargetTableCommandConvertToNullWithTimestamp ); self::assertEquals( // phpcs:ignore - 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2", "_timestamp") SELECT NULLIF("col1", \'\'),CAST(COALESCE("col2", \'\') as VARCHAR (4000)) AS "col2",\'2020-01-01 00:00:00\' FROM "import-export-test_schema"."stagingTable" AS "src"', + 'INSERT INTO "import-export-test_schema"."import-export-test_test" ("col1", "col2", "_timestamp") SELECT NULLIF("col1", \'\'),CAST(COALESCE("col2", \'\') as VARCHAR (50)) AS "col2",\'2020-01-01 00:00:00\' FROM "import-export-test_schema"."stagingTable" AS "src"', $sql ); $out = $this->connection->executeStatement($sql); From 56a319520abbda8fd1fea8f10ba1abd9b98c7c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Thu, 24 Mar 2022 23:25:33 +0100 Subject: [PATCH 54/62] run all suites --- azure-pipelines.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3d4d566e..fb804937 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -129,6 +129,18 @@ stages: EXASOL_USERNAME=devel docker-compose run production php -v parallel -j12 --joblog /tmp/parallel-joblog --linebuffer docker-compose run production composer ::: \ + tests-storage \ + tests-snowflake-abs \ + tests-snowflake-s3 \ + tests-synapse \ + tests-synapse-mi \ + tests-synapse-columnstoretemp \ + tests-synapse-clusterdindextemp \ + tests-synapse-heap4000temp \ + tests-synapse-heap4000temp-hash \ + tests-synapse-heap4000temp-optimized \ + tests-synapse-heap4000temp-optimized-hash \ + tests-synapse-next \ tests-teradata PARALLEL_EXIT_CODE=$? cat /tmp/parallel-joblog From 7e122de7d22d76ec6daa7a5cfeddabc5eecaac68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 25 Mar 2022 08:39:03 +0100 Subject: [PATCH 55/62] fix test --- tests/functional/Teradata/StageImportTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/Teradata/StageImportTest.php b/tests/functional/Teradata/StageImportTest.php index 37566d00..1e0bc1ab 100644 --- a/tests/functional/Teradata/StageImportTest.php +++ b/tests/functional/Teradata/StageImportTest.php @@ -26,8 +26,8 @@ public function testSimpleStageImport(): void 'CREATE MULTISET TABLE %s.%s ,NO FALLBACK ( "id" INTEGER NOT NULL, - "first_name" CHAR(32000), - "last_name" CHAR(32000) + "first_name" CHAR(50), + "last_name" CHAR(50) );', TeradataQuote::quoteSingleIdentifier(self::TEST_DATABASE), TeradataQuote::quoteSingleIdentifier(self::TABLE_GENERIC) From a06787bba5448de0c3c2e308935e731c7fea831b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 25 Mar 2022 08:39:42 +0100 Subject: [PATCH 56/62] increase timeout --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fb804937..8340246b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,7 +23,7 @@ stages: displayName: Build and test jobs: - job: Build - timeoutInMinutes: 100 + timeoutInMinutes: 150 displayName: Build pool: vmImage: 'ubuntu-latest' From 32e547250b40b20ff08d057a32db1b47bde9b42f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 25 Mar 2022 09:53:18 +0100 Subject: [PATCH 57/62] print out the error --- .../ToStage/Exception/FailedTPTLoadException.php | 10 +++++++++- src/Backend/Teradata/ToStage/FromS3TPTAdapter.php | 14 +++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php b/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php index b4fc5ae2..2e164f25 100644 --- a/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php +++ b/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php @@ -39,7 +39,15 @@ public function __construct( ?array $errTableContent, ?array $errTable2Content ) { - parent::__construct('Teradata TPT load ended with Error.', $exitCode ?? 0); + parent::__construct("Teradata TPT load ended with Error. \n\n + StdErr :$stdErr \n\n + stdOut :$stdOut \n\n + logContent :$logContent \n\n + logTableContent : ".implode($logTableContent) ." \n\n + errTableContent : ".implode($errTableContent) ." \n\n + errTable2Content : ".implode($errTable2Content) ." \n\n + " + , $exitCode ?? 0); $this->stdErr = $stdErr; $this->stdOut = $stdOut; $this->logContent = $logContent; diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index fb05ee85..ab70227b 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -76,13 +76,13 @@ public function runCopyCommand( $process->wait(); // debug stuff - foreach ($process as $type => $data) { - if ($process::OUT === $type) { - echo "\nRead from stdout: " . $data; - } else { // $process::ERR === $type - echo "\nRead from stderr: " . $data; - } - } +// foreach ($process as $type => $data) { +// if ($process::OUT === $type) { +// echo "\nRead from stdout: " . $data; +// } else { // $process::ERR === $type +// echo "\nRead from stderr: " . $data; +// } +// } $qb = new SqlBuilder(); $isTableExists = function (string $databaseName, string $tableName) use ($qb) { return (bool) $this->connection->fetchOne($qb->getTableExistsCommand($databaseName, $tableName)); From 031f51b6528df0ddb1a314bf7c8bc0884ca75587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 25 Mar 2022 09:53:27 +0100 Subject: [PATCH 58/62] run td only --- azure-pipelines.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8340246b..2b93679d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -129,18 +129,6 @@ stages: EXASOL_USERNAME=devel docker-compose run production php -v parallel -j12 --joblog /tmp/parallel-joblog --linebuffer docker-compose run production composer ::: \ - tests-storage \ - tests-snowflake-abs \ - tests-snowflake-s3 \ - tests-synapse \ - tests-synapse-mi \ - tests-synapse-columnstoretemp \ - tests-synapse-clusterdindextemp \ - tests-synapse-heap4000temp \ - tests-synapse-heap4000temp-hash \ - tests-synapse-heap4000temp-optimized \ - tests-synapse-heap4000temp-optimized-hash \ - tests-synapse-next \ tests-teradata PARALLEL_EXIT_CODE=$? cat /tmp/parallel-joblog From fd51d5b21f27197f47fe3dc194e5dd9b8f74ed49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 25 Mar 2022 10:25:46 +0100 Subject: [PATCH 59/62] fix cs --- .../ToStage/Exception/FailedTPTLoadException.php | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php b/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php index 2e164f25..54f6c287 100644 --- a/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php +++ b/src/Backend/Teradata/ToStage/Exception/FailedTPTLoadException.php @@ -39,15 +39,17 @@ public function __construct( ?array $errTableContent, ?array $errTable2Content ) { - parent::__construct("Teradata TPT load ended with Error. \n\n + parent::__construct( + "Teradata TPT load ended with Error. \n\n StdErr :$stdErr \n\n stdOut :$stdOut \n\n logContent :$logContent \n\n - logTableContent : ".implode($logTableContent) ." \n\n - errTableContent : ".implode($errTableContent) ." \n\n - errTable2Content : ".implode($errTable2Content) ." \n\n - " - , $exitCode ?? 0); + logTableContent : " . ($logTableContent ? implode($logTableContent) : '') . " \n\n + errTableContent : " . ($errTableContent ? implode($errTableContent) : '') . "\n\n + errTable2Content : " . ($errTable2Content ? implode($errTable2Content) : '') . " \n\n + ", + $exitCode ?? 0 + ); $this->stdErr = $stdErr; $this->stdOut = $stdOut; $this->logContent = $logContent; From 309e18cd82a0ec45ed188fdb75364b75f4d5ddf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Fri, 25 Mar 2022 14:14:19 +0100 Subject: [PATCH 60/62] source to destination --- azure-pipelines.yml | 14 +++++++++++++- .../ToStage/FromTableInsertIntoAdapter.php | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2b93679d..fdd75d28 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,7 +23,7 @@ stages: displayName: Build and test jobs: - job: Build - timeoutInMinutes: 150 + timeoutInMinutes: 200 displayName: Build pool: vmImage: 'ubuntu-latest' @@ -129,6 +129,18 @@ stages: EXASOL_USERNAME=devel docker-compose run production php -v parallel -j12 --joblog /tmp/parallel-joblog --linebuffer docker-compose run production composer ::: \ + tests-storage \ + tests-snowflake-abs \ + tests-snowflake-s3 \ + tests-synapse \ + tests-synapse-mi \ + tests-synapse-columnstoretemp \ + tests-synapse-clusterdindextemp \ + tests-synapse-heap4000temp \ + tests-synapse-heap4000temp-hash \ + tests-synapse-heap4000temp-optimized \ + tests-synapse-heap4000temp-optimized-hash \ + tests-synapse-next \ tests-teradata PARALLEL_EXIT_CODE=$? cat /tmp/parallel-joblog diff --git a/src/Backend/Teradata/ToStage/FromTableInsertIntoAdapter.php b/src/Backend/Teradata/ToStage/FromTableInsertIntoAdapter.php index 13c79241..d30e17d5 100644 --- a/src/Backend/Teradata/ToStage/FromTableInsertIntoAdapter.php +++ b/src/Backend/Teradata/ToStage/FromTableInsertIntoAdapter.php @@ -37,7 +37,7 @@ public function runCopyCommand( $quotedColumns = array_map(static function ($column) { return TeradataQuote::quoteSingleIdentifier($column); - }, $source->getColumnsNames()); + }, $destination->getColumnsNames()); $sql = sprintf( 'INSERT INTO %s.%s (%s) %s', From 810b7aa4230a581c81cdc3927900a3a2bb8ef9a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= Date: Mon, 28 Mar 2022 00:31:17 +0200 Subject: [PATCH 61/62] remove teardown --- tests/functional/Teradata/TeradataBaseTestCase.php | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/functional/Teradata/TeradataBaseTestCase.php b/tests/functional/Teradata/TeradataBaseTestCase.php index df969a8e..e9a9e3b0 100644 --- a/tests/functional/Teradata/TeradataBaseTestCase.php +++ b/tests/functional/Teradata/TeradataBaseTestCase.php @@ -440,11 +440,4 @@ protected function assertTeradataTableEqualsExpected( $message ); } - - protected function tearDown(): void - { - $this->cleanDatabase($this->getDestinationDbName()); - $this->cleanDatabase($this->getSourceDbName()); - parent::tearDown(); - } } From 87c705b70753dd4973f84e3a7f67d881ec3749bc Mon Sep 17 00:00:00 2001 From: zajca Date: Mon, 28 Mar 2022 15:30:54 +0200 Subject: [PATCH 62/62] split objet and prefix for TPT --- src/Backend/Teradata/ToStage/FromS3TPTAdapter.php | 15 +++++++++++---- tests/unit/Backend/Teradata/HelperTest.php | 11 +++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php index ab70227b..832a026f 100644 --- a/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php +++ b/src/Backend/Teradata/ToStage/FromS3TPTAdapter.php @@ -13,6 +13,8 @@ use Keboola\Db\ImportExport\ImportOptions; use Keboola\Db\ImportExport\ImportOptionsInterface; use Keboola\Db\ImportExport\Storage; +use Keboola\FileStorage\Path\RelativePath; +use Keboola\FileStorage\S3\S3Provider; use Keboola\TableBackendUtils\Escaping\Teradata\TeradataQuote; use Keboola\TableBackendUtils\Table\TableDefinitionInterface; use Keboola\TableBackendUtils\Table\Teradata\TeradataTableDefinition; @@ -174,15 +176,20 @@ private function generateTPTScript( ); if ($source->isSliced()) { + $mask = BackendHelper::getMask($source); + $path = RelativePath::createFromRootAndPath(new S3Provider(), $source->getBucket(), $mask); $moduleStr = sprintf( - 'AccessModuleInitStr = \'S3Region=%s S3Bucket=%s S3Prefix="" S3Object=%s S3SinglePartFile=True\'', + // phpcs:ignore + 'AccessModuleInitStr = \'S3Region="%s" S3Bucket="%s" S3Prefix="%s" S3Object="%s" S3SinglePartFile=True\'', $source->getRegion(), - $source->getBucket(), - BackendHelper::getMask($source) + $path->getRoot(), + $path->getPathWithoutRoot() . '/', + $path->getFileName() ); } else { $moduleStr = sprintf( - 'AccessModuleInitStr = \'S3Region=%s S3Bucket=%s S3Prefix="%s" S3Object=%s S3SinglePartFile=True\'', + // phpcs:ignore + 'AccessModuleInitStr = \'S3Region="%s" S3Bucket="%s" S3Prefix="%s" S3Object="%s" S3SinglePartFile=True\'', $source->getRegion(), $source->getBucket(), $source->getPrefix(), diff --git a/tests/unit/Backend/Teradata/HelperTest.php b/tests/unit/Backend/Teradata/HelperTest.php index b103346a..f67000bc 100644 --- a/tests/unit/Backend/Teradata/HelperTest.php +++ b/tests/unit/Backend/Teradata/HelperTest.php @@ -20,6 +20,8 @@ public function testGetResult(string $expected, array $entriesData): void $mock->method('getManifestEntries') ->willReturn($entriesData); + $mock->method('getS3Prefix') + ->willReturn('s3://zajca-php-db-import-test-s3filesbucket-bwdj3sk0c9xy'); $this->assertEquals($expected, BackendHelper::getMask($mock)); } @@ -29,6 +31,15 @@ public function testGetResult(string $expected, array $entriesData): void public function dataProvider(): array { return [ + [ + 'sliced/accounts-gzip/tw_accounts.csv.gz000*_part_00.gz', + [ + // phpcs:ignore + 's3://zajca-php-db-import-test-s3filesbucket-bwdj3sk0c9xy/sliced/accounts-gzip/tw_accounts.csv.gz0001_part_00.gz', + // phpcs:ignore + 's3://zajca-php-db-import-test-s3filesbucket-bwdj3sk0c9xy/sliced/accounts-gzip/tw_accounts.csv.gz0002_part_00.gz', + ], + ], [ 'sliced.csv_*', [