diff --git a/airbyte-cdk/bulk/toolkits/load-iceberg-parquet/build.gradle b/airbyte-cdk/bulk/toolkits/load-iceberg-parquet/build.gradle index 652e7a1e577b..d395ecd3a6f0 100644 --- a/airbyte-cdk/bulk/toolkits/load-iceberg-parquet/build.gradle +++ b/airbyte-cdk/bulk/toolkits/load-iceberg-parquet/build.gradle @@ -1,12 +1,16 @@ +ext { + apacheIcebergVersion = '1.7.0' +} + dependencies { api project(':airbyte-cdk:bulk:toolkits:bulk-cdk-toolkit-load-s3') implementation project(':airbyte-cdk:bulk:core:bulk-cdk-core-base') implementation project(':airbyte-cdk:bulk:core:bulk-cdk-core-load') - api 'org.apache.iceberg:iceberg-core:1.6.1' - api 'org.apache.iceberg:iceberg-api:1.6.1' - api 'org.apache.iceberg:iceberg-parquet:1.6.1' - api 'org.apache.iceberg:iceberg-nessie:1.6.1' + api "org.apache.iceberg:iceberg-core:${project.ext.apacheIcebergVersion}" + api "org.apache.iceberg:iceberg-api:${project.ext.apacheIcebergVersion}" + api "org.apache.iceberg:iceberg-parquet:${project.ext.apacheIcebergVersion}" + api "org.apache.iceberg:iceberg-nessie:${project.ext.apacheIcebergVersion}" testFixturesImplementation testFixtures(project(":airbyte-cdk:bulk:core:bulk-cdk-core-load")) } diff --git a/airbyte-integrations/connectors/destination-iceberg-v2/build.gradle b/airbyte-integrations/connectors/destination-iceberg-v2/build.gradle index cf763c04823c..b9d28301b1e1 100644 --- a/airbyte-integrations/connectors/destination-iceberg-v2/build.gradle +++ b/airbyte-integrations/connectors/destination-iceberg-v2/build.gradle @@ -19,9 +19,9 @@ application { } ext { - apacheIcebergVersion = "1.6.1" - awsSdkVersion = "2.29.9" - junitVersion = "5.11.3" + apacheIcebergVersion = '1.7.0' + awsSdkVersion = '2.29.9' + junitVersion = '5.11.3' } // Uncomment to run locally diff --git a/airbyte-integrations/connectors/destination-iceberg-v2/metadata.yaml b/airbyte-integrations/connectors/destination-iceberg-v2/metadata.yaml index f7b77615ad2e..09062b56f860 100644 --- a/airbyte-integrations/connectors/destination-iceberg-v2/metadata.yaml +++ b/airbyte-integrations/connectors/destination-iceberg-v2/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: 37a928c1-2d5c-431a-a97d-ae236bd1ea0c - dockerImageTag: 0.1.9 + dockerImageTag: 0.1.10 dockerRepository: airbyte/destination-iceberg-v2 githubIssueLabel: destination-iceberg-v2 icon: s3.svg diff --git a/airbyte-integrations/connectors/destination-iceberg-v2/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleaner.kt b/airbyte-integrations/connectors/destination-iceberg-v2/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleaner.kt index fb664641de53..e14ffcfaffeb 100644 --- a/airbyte-integrations/connectors/destination-iceberg-v2/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleaner.kt +++ b/airbyte-integrations/connectors/destination-iceberg-v2/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleaner.kt @@ -56,14 +56,12 @@ class IcebergTableCleaner(private val icebergUtil: IcebergUtil) { table.newScan().planFiles().use { tasks -> tasks - .filter { task -> - genIdsToDelete.any { id -> task.file().path().toString().contains(id) } - } + .filter { task -> genIdsToDelete.any { id -> task.file().location().contains(id) } } .forEach { task -> table .newDelete() .toBranch(stagingBranchName) - .deleteFile(task.file().path()) + .deleteFile(task.file().location()) .commit() } } diff --git a/airbyte-integrations/connectors/destination-iceberg-v2/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleanerTest.kt b/airbyte-integrations/connectors/destination-iceberg-v2/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleanerTest.kt index e2df9ebb3b2a..b7e8b1e6b864 100644 --- a/airbyte-integrations/connectors/destination-iceberg-v2/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleanerTest.kt +++ b/airbyte-integrations/connectors/destination-iceberg-v2/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleanerTest.kt @@ -102,11 +102,11 @@ internal class IcebergTableCleanerTest { every { tasks.close() } just Runs every { table.newScan().planFiles() } returns tasks - every { fileScanTask.file().path().toString() } returns filePathToDelete + every { fileScanTask.file().location() } returns filePathToDelete val delete = mockk() every { table.newDelete().toBranch("staging") } returns delete - every { delete.deleteFile(fileScanTask.file().path()) } returns delete + every { delete.deleteFile(filePathToDelete) } returns delete every { delete.commit() } just Runs assertDoesNotThrow { @@ -116,7 +116,7 @@ internal class IcebergTableCleanerTest { verify { icebergUtil.assertGenerationIdSuffixIsOfValidFormat(generationIdSuffix) table.newDelete().toBranch(eq("staging")) - delete.deleteFile(fileScanTask.file().path()) + delete.deleteFile(filePathToDelete) delete.commit() } } @@ -138,11 +138,11 @@ internal class IcebergTableCleanerTest { every { tasks.close() } just Runs every { table.newScan().planFiles() } returns tasks - every { fileScanTask.file().path().toString() } returns filePathToDelete + every { fileScanTask.file().location().toString() } returns filePathToDelete val delete = mockk() every { table.newDelete().toBranch("staging") } returns delete - every { delete.deleteFile(fileScanTask.file().path()) } returns delete + every { delete.deleteFile(fileScanTask.file().location()) } returns delete every { delete.commit() } just Runs assertDoesNotThrow {