From 4f07ad09ac1c8d03eed543e64089b05647960839 Mon Sep 17 00:00:00 2001 From: ZhouJinsong Date: Thu, 20 Jun 2024 10:48:12 +0800 Subject: [PATCH] [Improvement] Improve some maven profile configurations (#2944) * Improve maven profile configurations * Add github workflows configurations change --- .github/workflows/docker-images.yml | 4 +- README.md | 2 +- .../amoro-optimizer-flink/pom.xml | 2 +- .../pom.xml | 28 --- .../pom.xml | 28 --- .../pom.xml | 28 --- .../pom.xml | 28 --- .../pom.xml | 28 --- .../amoro-mixed-format-trino/pom.xml | 49 +---- .../parquet/reader/TimestampColumnReader.java | 10 +- .../DefaultMixedFormatCatalogFactory.java | 4 +- .../MixedFormatCatalogSupportTableSuffix.java | 4 +- .../trino/MixedFormatConnectorMetadata.java | 17 +- .../MixedFormatConnectorSplitManager.java | 6 +- .../trino/MixedFormatHdfsAuthentication.java | 4 +- .../trino/MixedFormatMetadataFactory.java | 4 +- .../apache/amoro/trino/MixedFormatModule.java | 23 ++- .../trino/MixedFormatPageSourceProvider.java | 10 +- .../trino/MixedFormatSessionProperties.java | 13 +- .../trino/MixedFormatTransactionManager.java | 3 +- .../amoro/trino/MixedFormatTrinoCatalog.java | 12 +- .../amoro/trino/delete/DummyFileScanTask.java | 2 +- .../amoro/trino/delete/TrinoDeleteFile.java | 4 +- .../apache/amoro/trino/delete/TrinoRow.java | 2 +- .../trino/keyed/KeyedConnectorMetadata.java | 23 ++- .../trino/keyed/KeyedConnectorPageSource.java | 20 +- .../trino/keyed/KeyedConnectorSplit.java | 4 +- .../keyed/KeyedConnectorSplitManager.java | 18 +- .../amoro/trino/keyed/KeyedDeleteFilter.java | 4 +- .../trino/keyed/KeyedPageSourceProvider.java | 12 +- .../amoro/trino/keyed/KeyedTableHandle.java | 4 +- .../amoro/trino/unkeyed/IcebergMetadata.java | 38 ++-- .../trino/unkeyed/IcebergPageSource.java | 8 +- .../unkeyed/IcebergPageSourceProvider.java | 20 +- .../amoro/trino/unkeyed/IcebergSplit.java | 7 +- .../trino/unkeyed/IcebergSplitManager.java | 10 +- .../trino/unkeyed/IcebergSplitSource.java | 26 +-- .../trino/unkeyed/TrinoDeleteFilter.java | 6 +- .../hive/metastore/HiveMetaStoreClient.java | 187 ++++++++---------- .../org/apache/hadoop/util/VersionInfo.java | 1 - .../org/apache/parquet/io/ColumnIOUtil.java | 10 +- .../amoro/trino/mixed/HiveTestRecords.java | 4 +- .../trino/mixed/MixedFormatQueryRunner.java | 2 +- .../trino/mixed/TableTestBaseForTrino.java | 12 +- .../TableTestBaseWithInitDataForTrino.java | 4 +- .../amoro/trino/mixed/TestHiveTable.java | 50 +++-- .../mixed/TestHiveTableBaseForTrino.java | 16 +- .../amoro/trino/mixed/TestKeyedTable.java | 18 +- pom.xml | 80 ++------ 49 files changed, 351 insertions(+), 548 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 3494dc4b03..5ed6e541ce 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -147,8 +147,8 @@ jobs: OPTIMIZER_FLINK=${{ matrix.flink }} && \ echo "OPTIMIZER_FLINK=-Dflink-optimizer.flink-version${OPTIMIZER_FLINK}" >> $GITHUB_ENV if [[ "$OPTIMIZER_FLINK" < "1.15" ]]; then - echo "Adding -Pflink-pre-1.15 for Flink version less than 1.15" - echo "OPTIMIZER_FLINK=-Pflink-pre-1.15 -Dflink-optimizer.flink-version=${OPTIMIZER_FLINK}" >> $GITHUB_ENV + echo "Adding -Pflink-optimizer-pre-1.15 for Flink version less than 1.15" + echo "OPTIMIZER_FLINK=-Pflink-optimizer-pre-1.15 -Dflink-optimizer.flink-version=${OPTIMIZER_FLINK}" >> $GITHUB_ENV fi - name: Set ENV Amoro version diff --git a/README.md b/README.md index 4c7663a3ce..a4cf21e4db 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ Amoro is built using Maven with JDK 8 and JDK 17(only for `amoro-mixed-format/am * Build and skip dashboard: `mvn clean package -Pskip-dashboard-build` * Build with hadoop 2.x(the default is 3.x) dependencies: `mvn clean package -DskipTests -Dhadoop=v2` * Specify Flink version for Flink optimizer(the default is 1.18.1): `mvn clean package -DskipTests -Dflink-optimizer.flink-version=1.15.4` - * If the version of Flink is below 1.15.0, you also need to add the `-Pflink-pre-1.15` parameter: `mvn clean package -DskipTests -Pflink-pre-1.15 -Dflink-optimizer.flink-version=1.14.6` + * If the version of Flink is below 1.15.0, you also need to add the `-Pflink-optimizer-pre-1.15` parameter: `mvn clean package -DskipTests -Pflink-optimizer-pre-1.15 -Dflink-optimizer.flink-version=1.14.6` * Specify Spark version for Spark optimizer(the default is 3.3.3): `mvn clean package -DskipTests -Dspark-optimizer.spark-version=3.2.2` * Build `amoro-mixed-format-trino` module under JDK 17: `mvn clean package -DskipTests -Ptrino-spotless,build-mixed-format-trino -pl 'amoro-mixed-format/amoro-mixed-format-trino' -am`. * Build all modules: `mvn clean package -DskipTests -Ptoolchain,build-mixed-format-trino`, besides you need config `toolchains.xml` in `${user.home}/.m2/` dir with content below. diff --git a/amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/pom.xml b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/pom.xml index 98cadb2fc3..b8e6a3f1e7 100644 --- a/amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/pom.xml +++ b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/pom.xml @@ -218,7 +218,7 @@ - flink-pre-1.15 + flink-optimizer-pre-1.15 flink-optimizer.flink-version diff --git a/amoro-mixed-format/amoro-mixed-format-flink/v1.15/amoro-mixed-format-flink-runtime-1.15/pom.xml b/amoro-mixed-format/amoro-mixed-format-flink/v1.15/amoro-mixed-format-flink-runtime-1.15/pom.xml index fa325091da..f808bf2039 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/v1.15/amoro-mixed-format-flink-runtime-1.15/pom.xml +++ b/amoro-mixed-format/amoro-mixed-format-flink/v1.15/amoro-mixed-format-flink-runtime-1.15/pom.xml @@ -249,32 +249,4 @@ - - - - deploy-maven-central - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin-version} - - - empty-javadoc-jar - - jar - - package - - javadoc - ${basedir}/javadoc - - - - - - - - diff --git a/amoro-mixed-format/amoro-mixed-format-flink/v1.16/amoro-mixed-format-flink-runtime-1.16/pom.xml b/amoro-mixed-format/amoro-mixed-format-flink/v1.16/amoro-mixed-format-flink-runtime-1.16/pom.xml index fb4c564471..b6cb7bd492 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/v1.16/amoro-mixed-format-flink-runtime-1.16/pom.xml +++ b/amoro-mixed-format/amoro-mixed-format-flink/v1.16/amoro-mixed-format-flink-runtime-1.16/pom.xml @@ -249,32 +249,4 @@ - - - - deploy-maven-central - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin-version} - - - empty-javadoc-jar - - jar - - package - - javadoc - ${basedir}/javadoc - - - - - - - - diff --git a/amoro-mixed-format/amoro-mixed-format-flink/v1.17/amoro-mixed-format-flink-runtime-1.17/pom.xml b/amoro-mixed-format/amoro-mixed-format-flink/v1.17/amoro-mixed-format-flink-runtime-1.17/pom.xml index 71a9d3de85..24578a6cb8 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/v1.17/amoro-mixed-format-flink-runtime-1.17/pom.xml +++ b/amoro-mixed-format/amoro-mixed-format-flink/v1.17/amoro-mixed-format-flink-runtime-1.17/pom.xml @@ -250,32 +250,4 @@ - - - - deploy-maven-central - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin-version} - - - empty-javadoc-jar - - jar - - package - - javadoc - ${basedir}/javadoc - - - - - - - - diff --git a/amoro-mixed-format/amoro-mixed-format-spark/v3.2/amoro-mixed-format-spark-runtime-3.2/pom.xml b/amoro-mixed-format/amoro-mixed-format-spark/v3.2/amoro-mixed-format-spark-runtime-3.2/pom.xml index 510be42643..484be250e9 100644 --- a/amoro-mixed-format/amoro-mixed-format-spark/v3.2/amoro-mixed-format-spark-runtime-3.2/pom.xml +++ b/amoro-mixed-format/amoro-mixed-format-spark/v3.2/amoro-mixed-format-spark-runtime-3.2/pom.xml @@ -447,32 +447,4 @@ - - - - deploy-maven-central - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin-version} - - - empty-javadoc-jar - package - - jar - - - javadoc - ${basedir}/javadoc - - - - - - - - diff --git a/amoro-mixed-format/amoro-mixed-format-spark/v3.3/amoro-mixed-format-spark-runtime-3.3/pom.xml b/amoro-mixed-format/amoro-mixed-format-spark/v3.3/amoro-mixed-format-spark-runtime-3.3/pom.xml index 6ac948192c..d473df156a 100644 --- a/amoro-mixed-format/amoro-mixed-format-spark/v3.3/amoro-mixed-format-spark-runtime-3.3/pom.xml +++ b/amoro-mixed-format/amoro-mixed-format-spark/v3.3/amoro-mixed-format-spark-runtime-3.3/pom.xml @@ -470,32 +470,4 @@ - - - - deploy-maven-central - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin-version} - - - empty-javadoc-jar - package - - jar - - - javadoc - ${basedir}/javadoc - - - - - - - - diff --git a/amoro-mixed-format/amoro-mixed-format-trino/pom.xml b/amoro-mixed-format/amoro-mixed-format-trino/pom.xml index f26e80fb0c..657a8e59de 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/pom.xml +++ b/amoro-mixed-format/amoro-mixed-format-trino/pom.xml @@ -49,6 +49,7 @@ 2.4 0.16 3.2.5 + true @@ -553,7 +554,7 @@ maven-assembly-plugin 2.6 - true + ${skip-build-mixed-format-trino} @@ -575,8 +576,8 @@ org.apache.maven.plugins maven-compiler-plugin - true - true + ${skip-build-mixed-format-trino} + ${skip-build-mixed-format-trino} true true 17 @@ -599,7 +600,7 @@ org.apache.maven.plugins maven-deploy-plugin - true + ${skip-build-mixed-format-trino} @@ -622,40 +623,9 @@ build-mixed-format-trino - - - - org.apache.maven.plugins - maven-compiler-plugin - - false - false - - - - org.apache.maven.plugins - maven-assembly-plugin - 2.6 - - false - - - - make-assembly - - single - - package - - false - amoro-mixed-trino-${project.version} - assembly.xml - - - - - - + + false + toolchain @@ -695,7 +665,8 @@ false - **/org/apache/**/*.java + **/org/apache/hadoop/**/*.java + **/org/apache/parquet/**/*.java **/io/trino/**/*.java diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/io/trino/parquet/reader/TimestampColumnReader.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/io/trino/parquet/reader/TimestampColumnReader.java index 4df65237ec..fa2842c915 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/io/trino/parquet/reader/TimestampColumnReader.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/io/trino/parquet/reader/TimestampColumnReader.java @@ -1,7 +1,11 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/DefaultMixedFormatCatalogFactory.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/DefaultMixedFormatCatalogFactory.java index e491fcd0e7..4a909fe826 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/DefaultMixedFormatCatalogFactory.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/DefaultMixedFormatCatalogFactory.java @@ -18,12 +18,12 @@ package org.apache.amoro.trino; +import io.trino.spi.classloader.ThreadContextClassLoader; import org.apache.amoro.api.CatalogMeta; -import org.apache.amoro.mixed.MixedFormatCatalog; import org.apache.amoro.mixed.CatalogLoader; +import org.apache.amoro.mixed.MixedFormatCatalog; import org.apache.amoro.table.TableMetaStore; import org.apache.amoro.utils.MixedCatalogUtil; -import io.trino.spi.classloader.ThreadContextClassLoader; import javax.inject.Inject; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatCatalogSupportTableSuffix.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatCatalogSupportTableSuffix.java index c9228c0131..cf63d11301 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatCatalogSupportTableSuffix.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatCatalogSupportTableSuffix.java @@ -19,15 +19,15 @@ package org.apache.amoro.trino; import org.apache.amoro.TableFormat; -import org.apache.amoro.mixed.MixedFormatCatalog; import org.apache.amoro.io.AuthenticatedFileIO; +import org.apache.amoro.mixed.MixedFormatCatalog; import org.apache.amoro.op.UpdatePartitionProperties; import org.apache.amoro.scan.ChangeTableIncrementalScan; -import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.BasicUnkeyedTable; import org.apache.amoro.table.ChangeTable; import org.apache.amoro.table.KeyedTable; import org.apache.amoro.table.MetadataColumns; +import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.TableBuilder; import org.apache.amoro.table.TableIdentifier; import org.apache.amoro.table.TableMetaStore; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatConnectorMetadata.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatConnectorMetadata.java index 0872a6dcf1..6e0e566b95 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatConnectorMetadata.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatConnectorMetadata.java @@ -21,12 +21,6 @@ import static io.trino.plugin.hive.util.HiveUtil.isHiveSystemSchema; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import org.apache.amoro.mixed.MixedFormatCatalog; -import org.apache.amoro.table.MixedTable; -import org.apache.amoro.table.TableIdentifier; -import org.apache.amoro.trino.keyed.KeyedConnectorMetadata; -import org.apache.amoro.trino.keyed.KeyedTableHandle; -import org.apache.amoro.trino.unkeyed.IcebergMetadata; import io.airlift.slice.Slice; import io.trino.spi.TrinoException; import io.trino.spi.connector.BeginTableExecuteResult; @@ -55,9 +49,15 @@ import io.trino.spi.expression.ConnectorExpression; import io.trino.spi.statistics.ComputedStatistics; import io.trino.spi.statistics.TableStatistics; -import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.amoro.mixed.MixedFormatCatalog; import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; import org.apache.amoro.shade.guava32.com.google.common.collect.Iterators; +import org.apache.amoro.table.MixedTable; +import org.apache.amoro.table.TableIdentifier; +import org.apache.amoro.trino.keyed.KeyedConnectorMetadata; +import org.apache.amoro.trino.keyed.KeyedTableHandle; +import org.apache.amoro.trino.unkeyed.IcebergMetadata; +import org.apache.iceberg.exceptions.NoSuchTableException; import java.util.Collection; import java.util.Iterator; @@ -455,7 +455,8 @@ public void rollback() { public MixedTable getMixedTable(SchemaTableName schemaTableName) { return tableCache.computeIfAbsent( - schemaTableName, ignore -> mixedFormatCatalog.loadTable(getTableIdentifier(schemaTableName))); + schemaTableName, + ignore -> mixedFormatCatalog.loadTable(getTableIdentifier(schemaTableName))); } private TableIdentifier getTableIdentifier(SchemaTableName schemaTableName) { diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatConnectorSplitManager.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatConnectorSplitManager.java index 3215fb9d8b..693ae87b7f 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatConnectorSplitManager.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatConnectorSplitManager.java @@ -18,9 +18,6 @@ package org.apache.amoro.trino; -import org.apache.amoro.trino.keyed.KeyedConnectorSplitManager; -import org.apache.amoro.trino.keyed.KeyedTableHandle; -import org.apache.amoro.trino.unkeyed.IcebergSplitManager; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.ConnectorSplitManager; import io.trino.spi.connector.ConnectorSplitSource; @@ -28,6 +25,9 @@ import io.trino.spi.connector.ConnectorTransactionHandle; import io.trino.spi.connector.Constraint; import io.trino.spi.connector.DynamicFilter; +import org.apache.amoro.trino.keyed.KeyedConnectorSplitManager; +import org.apache.amoro.trino.keyed.KeyedTableHandle; +import org.apache.amoro.trino.unkeyed.IcebergSplitManager; import javax.inject.Inject; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatHdfsAuthentication.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatHdfsAuthentication.java index 8090ea66de..e9f3d99d91 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatHdfsAuthentication.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatHdfsAuthentication.java @@ -18,11 +18,11 @@ package org.apache.amoro.trino; -import org.apache.amoro.table.TableMetaStore; import io.trino.hdfs.authentication.GenericExceptionAction; import io.trino.hdfs.authentication.HdfsAuthentication; import io.trino.spi.classloader.ThreadContextClassLoader; import io.trino.spi.security.ConnectorIdentity; +import org.apache.amoro.table.TableMetaStore; import javax.inject.Inject; @@ -35,7 +35,7 @@ public class MixedFormatHdfsAuthentication implements HdfsAuthentication { @Inject public MixedFormatHdfsAuthentication( - MixedFormatCatalogFactory mixedFormatCatalogFactory, MixedFormatConfig mixedFormatConfig) { + MixedFormatCatalogFactory mixedFormatCatalogFactory, MixedFormatConfig mixedFormatConfig) { this.tableMetaStore = mixedFormatCatalogFactory.getTableMetastore(); this.mixedFormatConfig = mixedFormatConfig; } diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatMetadataFactory.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatMetadataFactory.java index 31cb81cff9..8972dd8cff 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatMetadataFactory.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatMetadataFactory.java @@ -20,14 +20,14 @@ import static java.util.Objects.requireNonNull; -import org.apache.amoro.trino.unkeyed.IcebergMetadata; -import org.apache.amoro.trino.keyed.KeyedConnectorMetadata; import io.airlift.json.JsonCodec; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.plugin.iceberg.CommitTaskData; import io.trino.plugin.iceberg.TableStatisticsWriter; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; import io.trino.spi.type.TypeManager; +import org.apache.amoro.trino.keyed.KeyedConnectorMetadata; +import org.apache.amoro.trino.unkeyed.IcebergMetadata; import javax.inject.Inject; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatModule.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatModule.java index e26eecb3c7..470d268197 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatModule.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatModule.java @@ -27,10 +27,6 @@ import com.google.inject.Module; import com.google.inject.Scopes; import com.google.inject.multibindings.Multibinder; -import org.apache.amoro.trino.unkeyed.IcebergPageSourceProvider; -import org.apache.amoro.trino.unkeyed.IcebergSplitManager; -import org.apache.amoro.trino.keyed.KeyedConnectorSplitManager; -import org.apache.amoro.trino.keyed.KeyedPageSourceProvider; import io.airlift.configuration.ConfigBinder; import io.trino.hdfs.HdfsConfig; import io.trino.hdfs.HdfsConfiguration; @@ -64,6 +60,10 @@ import io.trino.spi.connector.TableProcedureMetadata; import io.trino.spi.procedure.Procedure; import io.trino.spi.type.TypeManager; +import org.apache.amoro.trino.keyed.KeyedConnectorSplitManager; +import org.apache.amoro.trino.keyed.KeyedPageSourceProvider; +import org.apache.amoro.trino.unkeyed.IcebergPageSourceProvider; +import org.apache.amoro.trino.unkeyed.IcebergSplitManager; import org.weakref.jmx.guice.ExportBinder; /** Mixed-format table module of Trino */ @@ -87,7 +87,10 @@ public void configure(Binder binder) { .bind(MixedFormatCatalogFactory.class) .to(DefaultMixedFormatCatalogFactory.class) .in(Scopes.SINGLETON); - binder.bind(TrinoCatalogFactory.class).to(MixedFormatTrinoCatalogFactory.class).in(Scopes.SINGLETON); + binder + .bind(TrinoCatalogFactory.class) + .to(MixedFormatTrinoCatalogFactory.class) + .in(Scopes.SINGLETON); binder.bind(MixedFormatTransactionManager.class).in(Scopes.SINGLETON); binder.bind(MixedFormatMetadataFactory.class).in(Scopes.SINGLETON); binder.bind(TableStatisticsWriter.class).in(Scopes.SINGLETON); @@ -154,8 +157,14 @@ public void configure(Binder binder) { // hdfs ConfigBinder.configBinder(binder).bindConfig(HdfsConfig.class); - binder.bind(HdfsConfiguration.class).to(MixedFormatHdfsConfiguration.class).in(Scopes.SINGLETON); - binder.bind(HdfsAuthentication.class).to(MixedFormatHdfsAuthentication.class).in(Scopes.SINGLETON); + binder + .bind(HdfsConfiguration.class) + .to(MixedFormatHdfsConfiguration.class) + .in(Scopes.SINGLETON); + binder + .bind(HdfsAuthentication.class) + .to(MixedFormatHdfsAuthentication.class) + .in(Scopes.SINGLETON); binder.bind(HdfsEnvironment.class).in(Scopes.SINGLETON); binder.bind(NamenodeStats.class).in(Scopes.SINGLETON); ExportBinder.newExporter(binder).export(NamenodeStats.class).withGeneratedName(); diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatPageSourceProvider.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatPageSourceProvider.java index b3c95e72df..51cb1c6f56 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatPageSourceProvider.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatPageSourceProvider.java @@ -18,9 +18,6 @@ package org.apache.amoro.trino; -import org.apache.amoro.trino.unkeyed.IcebergPageSourceProvider; -import org.apache.amoro.trino.keyed.KeyedPageSourceProvider; -import org.apache.amoro.trino.keyed.KeyedTableHandle; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ConnectorPageSource; import io.trino.spi.connector.ConnectorPageSourceProvider; @@ -29,14 +26,17 @@ import io.trino.spi.connector.ConnectorTableHandle; import io.trino.spi.connector.ConnectorTransactionHandle; import io.trino.spi.connector.DynamicFilter; +import org.apache.amoro.trino.keyed.KeyedPageSourceProvider; +import org.apache.amoro.trino.keyed.KeyedTableHandle; +import org.apache.amoro.trino.unkeyed.IcebergPageSourceProvider; import javax.inject.Inject; import java.util.List; /** - * {@link MixedFormatPageSourceProvider} is a Union {@link ConnectorPageSourceProvider} contain {@link - * KeyedPageSourceProvider} and {@link IcebergPageSourceProvider}. This is final {@link + * {@link MixedFormatPageSourceProvider} is a Union {@link ConnectorPageSourceProvider} contain + * {@link KeyedPageSourceProvider} and {@link IcebergPageSourceProvider}. This is final {@link * ConnectorPageSourceProvider} provided to Trino */ public class MixedFormatPageSourceProvider implements ConnectorPageSourceProvider { diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatSessionProperties.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatSessionProperties.java index 4c98bd83cc..926dee8c36 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatSessionProperties.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatSessionProperties.java @@ -31,38 +31,37 @@ import java.util.List; -/** Session properties provider for mixed-format table. */ +/** Session properties provider for mixed-format table. */ public final class MixedFormatSessionProperties implements SessionPropertiesProvider { private static final String MIXED_STATISTICS_ENABLED = "mixed_table_statistics_enabled"; - private static final String MIXED_SPLIT_TASK_BY_DELETE_RATIO = - "mixed_split_task_by_delete_ratio"; + private static final String MIXED_SPLIT_TASK_BY_DELETE_RATIO = "mixed_split_task_by_delete_ratio"; private static final String MIXED_ENABLE_SPLIT_TASK_BY_DELETE_RATIO = "mixed_enable_split_task_by_delete_ratio"; private final List> sessionProperties; @Inject public MixedFormatSessionProperties( - MixedFormatConfig mixedFormatConfig, IcebergSessionProperties icebergSessionProperties) { + MixedFormatConfig mixedFormatConfig, IcebergSessionProperties icebergSessionProperties) { sessionProperties = ImmutableList.>builder() .addAll(icebergSessionProperties.getSessionProperties()) .add( booleanProperty( - MIXED_STATISTICS_ENABLED, + MIXED_STATISTICS_ENABLED, "Expose table statistics for mixed-format table", mixedFormatConfig.isTableStatisticsEnabled(), false)) .add( doubleProperty( - MIXED_SPLIT_TASK_BY_DELETE_RATIO, + MIXED_SPLIT_TASK_BY_DELETE_RATIO, "If task delete ratio less than this value will be split to more task", mixedFormatConfig.getSplitTaskByDeleteRatio(), false)) .add( booleanProperty( - MIXED_ENABLE_SPLIT_TASK_BY_DELETE_RATIO, + MIXED_ENABLE_SPLIT_TASK_BY_DELETE_RATIO, "Enable task split by ratio", mixedFormatConfig.isEnableSplitTaskByDeleteRatio(), false)) diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatTransactionManager.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatTransactionManager.java index 51982a9f65..813d5e750e 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatTransactionManager.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatTransactionManager.java @@ -44,7 +44,8 @@ public MixedFormatTransactionManager(MixedFormatMetadataFactory metadataFactory) this(metadataFactory, Thread.currentThread().getContextClassLoader()); } - public MixedFormatTransactionManager(MixedFormatMetadataFactory metadataFactory, ClassLoader classLoader) { + public MixedFormatTransactionManager( + MixedFormatMetadataFactory metadataFactory, ClassLoader classLoader) { this.metadataFactory = requireNonNull(metadataFactory, "metadataFactory is null"); this.classLoader = requireNonNull(classLoader, "classLoader is null"); } diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatTrinoCatalog.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatTrinoCatalog.java index 053da15ca7..7b6d3d1d9f 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatTrinoCatalog.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatTrinoCatalog.java @@ -22,9 +22,6 @@ import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; import static java.util.Locale.ENGLISH; -import org.apache.amoro.mixed.MixedFormatCatalog; -import org.apache.amoro.table.MixedTable; -import org.apache.amoro.table.TableIdentifier; import io.trino.plugin.hive.util.HiveUtil; import io.trino.plugin.iceberg.ColumnIdentity; import io.trino.plugin.iceberg.catalog.TrinoCatalog; @@ -35,11 +32,14 @@ import io.trino.spi.connector.ConnectorViewDefinition; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.security.TrinoPrincipal; +import org.apache.amoro.mixed.MixedFormatCatalog; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; +import org.apache.amoro.table.MixedTable; +import org.apache.amoro.table.TableIdentifier; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.Transaction; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; import java.util.Collections; import java.util.List; @@ -156,7 +156,9 @@ public void registerTable( public void dropTable(ConnectorSession session, SchemaTableName schemaTableName) { mixedFormatCatalog.dropTable( TableIdentifier.of( - mixedFormatCatalog.name(), schemaTableName.getSchemaName(), schemaTableName.getTableName()), + mixedFormatCatalog.name(), + schemaTableName.getSchemaName(), + schemaTableName.getTableName()), true); } diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/delete/DummyFileScanTask.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/delete/DummyFileScanTask.java index 1c96afad20..6f253dda1c 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/delete/DummyFileScanTask.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/delete/DummyFileScanTask.java @@ -20,6 +20,7 @@ import static java.util.Objects.requireNonNull; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataTask; @@ -29,7 +30,6 @@ import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.StructLike; import org.apache.iceberg.expressions.Expression; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; import java.util.List; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/delete/TrinoDeleteFile.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/delete/TrinoDeleteFile.java index ed63308bc1..4830e85210 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/delete/TrinoDeleteFile.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/delete/TrinoDeleteFile.java @@ -27,12 +27,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileContent; import org.apache.iceberg.FileFormat; import org.apache.iceberg.StructLike; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; import org.openjdk.jol.info.ClassLayout; import javax.annotation.Nullable; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/delete/TrinoRow.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/delete/TrinoRow.java index 5b359ad8b2..e37e7ad8c1 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/delete/TrinoRow.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/delete/TrinoRow.java @@ -24,8 +24,8 @@ import io.trino.spi.Page; import io.trino.spi.type.Type; -import org.apache.iceberg.StructLike; import org.apache.amoro.shade.guava32.com.google.common.collect.AbstractIterator; +import org.apache.iceberg.StructLike; import javax.annotation.Nullable; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorMetadata.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorMetadata.java index 7eb21f0167..8cc1d7b50d 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorMetadata.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorMetadata.java @@ -30,12 +30,6 @@ import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap.toImmutableMap; import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableSet.toImmutableSet; -import org.apache.amoro.mixed.MixedFormatCatalog; -import org.apache.amoro.table.MixedTable; -import org.apache.amoro.table.KeyedTable; -import org.apache.amoro.table.TableIdentifier; -import org.apache.amoro.trino.MixedFormatSessionProperties; -import org.apache.amoro.trino.util.ObjectSerializerUtil; import io.trino.plugin.hive.HiveApplyProjectionUtil; import io.trino.plugin.iceberg.ColumnIdentity; import io.trino.plugin.iceberg.IcebergColumnHandle; @@ -61,15 +55,21 @@ import io.trino.spi.predicate.TupleDomain; import io.trino.spi.statistics.TableStatistics; import io.trino.spi.type.TypeManager; +import org.apache.amoro.mixed.MixedFormatCatalog; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableSet; +import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; +import org.apache.amoro.table.KeyedTable; +import org.apache.amoro.table.MixedTable; +import org.apache.amoro.table.TableIdentifier; +import org.apache.amoro.trino.MixedFormatSessionProperties; +import org.apache.amoro.trino.util.ObjectSerializerUtil; import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.SchemaParser; import org.apache.iceberg.TableProperties; import org.apache.iceberg.exceptions.NotFoundException; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableSet; -import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -448,8 +448,7 @@ public TableStatistics getTableStatistics( typeManager, session, withSnapshotId( - handle, - mixedTable.asKeyedTable().baseTable().currentSnapshot().snapshotId()), + handle, mixedTable.asKeyedTable().baseTable().currentSnapshot().snapshotId()), mixedTable.asKeyedTable().baseTable()); return baseTableStatistics; }); diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorPageSource.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorPageSource.java index 6bf1750de4..792e4a0616 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorPageSource.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorPageSource.java @@ -23,14 +23,6 @@ import static org.apache.amoro.shade.guava32.com.google.common.base.Preconditions.checkArgument; import static org.apache.amoro.shade.guava32.com.google.common.base.Throwables.throwIfInstanceOf; -import org.apache.amoro.data.DataFileType; -import org.apache.amoro.data.PrimaryKeyedFile; -import org.apache.amoro.scan.MixedFileScanTask; -import org.apache.amoro.table.MetadataColumns; -import org.apache.amoro.trino.delete.TrinoDeleteFile; -import org.apache.amoro.trino.delete.TrinoRow; -import org.apache.amoro.trino.unkeyed.IcebergPageSourceProvider; -import org.apache.amoro.trino.unkeyed.IcebergSplit; import io.trino.plugin.iceberg.IcebergColumnHandle; import io.trino.plugin.iceberg.IcebergFileFormat; import io.trino.spi.Page; @@ -42,11 +34,19 @@ import io.trino.spi.connector.DynamicFilter; import io.trino.spi.type.Type; import io.trino.spi.type.TypeManager; +import org.apache.amoro.data.DataFileType; +import org.apache.amoro.data.PrimaryKeyedFile; import org.apache.amoro.hive.io.reader.AdaptHiveMixedDeleteFilter; -import org.apache.iceberg.io.CloseableIterable; +import org.apache.amoro.scan.MixedFileScanTask; import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; -import org.joda.time.DateTimeZone; +import org.apache.amoro.table.MetadataColumns; import org.apache.amoro.trino.AmoroErrorCode; +import org.apache.amoro.trino.delete.TrinoDeleteFile; +import org.apache.amoro.trino.delete.TrinoRow; +import org.apache.amoro.trino.unkeyed.IcebergPageSourceProvider; +import org.apache.amoro.trino.unkeyed.IcebergSplit; +import org.apache.iceberg.io.CloseableIterable; +import org.joda.time.DateTimeZone; import java.io.IOException; import java.util.HashMap; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorSplit.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorSplit.java index 853a4c555f..bfac6f35c6 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorSplit.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorSplit.java @@ -20,11 +20,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.trino.spi.HostAddress; +import io.trino.spi.connector.ConnectorSplit; import org.apache.amoro.scan.KeyedTableScanTask; import org.apache.amoro.scan.NodeFileScanTask; import org.apache.amoro.trino.util.ObjectSerializerUtil; -import io.trino.spi.HostAddress; -import io.trino.spi.connector.ConnectorSplit; import java.util.Collections; import java.util.List; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorSplitManager.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorSplitManager.java index c7cd14abf9..c10f3fcb3d 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorSplitManager.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedConnectorSplitManager.java @@ -20,15 +20,6 @@ import static io.trino.plugin.iceberg.ExpressionConverter.toIcebergExpression; -import org.apache.amoro.scan.MixedFileScanTask; -import org.apache.amoro.scan.CombinedScanTask; -import org.apache.amoro.scan.KeyedTableScan; -import org.apache.amoro.scan.KeyedTableScanTask; -import org.apache.amoro.table.KeyedTable; -import org.apache.amoro.trino.MixedFormatSessionProperties; -import org.apache.amoro.trino.MixedFormatTransactionManager; -import org.apache.amoro.trino.util.MetricUtil; -import org.apache.amoro.trino.util.ObjectSerializerUtil; import io.trino.plugin.iceberg.IcebergTableHandle; import io.trino.plugin.iceberg.PartitionData; import io.trino.spi.classloader.ThreadContextClassLoader; @@ -42,6 +33,15 @@ import io.trino.spi.connector.FixedSplitSource; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.connector.TableNotFoundException; +import org.apache.amoro.scan.CombinedScanTask; +import org.apache.amoro.scan.KeyedTableScan; +import org.apache.amoro.scan.KeyedTableScanTask; +import org.apache.amoro.scan.MixedFileScanTask; +import org.apache.amoro.table.KeyedTable; +import org.apache.amoro.trino.MixedFormatSessionProperties; +import org.apache.amoro.trino.MixedFormatTransactionManager; +import org.apache.amoro.trino.util.MetricUtil; +import org.apache.amoro.trino.util.ObjectSerializerUtil; import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.io.CloseableIterable; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedDeleteFilter.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedDeleteFilter.java index 4072412a03..c2505818db 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedDeleteFilter.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedDeleteFilter.java @@ -21,11 +21,11 @@ import static io.trino.plugin.iceberg.TypeConverter.toIcebergType; import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList.toImmutableList; +import io.trino.plugin.iceberg.IcebergColumnHandle; +import org.apache.amoro.hive.io.reader.AdaptHiveMixedDeleteFilter; import org.apache.amoro.scan.KeyedTableScanTask; import org.apache.amoro.table.PrimaryKeySpec; import org.apache.amoro.trino.delete.TrinoRow; -import io.trino.plugin.iceberg.IcebergColumnHandle; -import org.apache.amoro.hive.io.reader.AdaptHiveMixedDeleteFilter; import org.apache.iceberg.Schema; import org.apache.iceberg.StructLike; import org.apache.iceberg.io.FileIO; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedPageSourceProvider.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedPageSourceProvider.java index a3f2290cc2..7128341534 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedPageSourceProvider.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedPageSourceProvider.java @@ -19,11 +19,6 @@ package org.apache.amoro.trino.keyed; import com.google.inject.Inject; -import org.apache.amoro.data.PrimaryKeyedFile; -import org.apache.amoro.scan.MixedFileScanTask; -import org.apache.amoro.scan.KeyedTableScanTask; -import org.apache.amoro.trino.delete.TrinoRow; -import org.apache.amoro.trino.unkeyed.IcebergPageSourceProvider; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.plugin.iceberg.IcebergColumnHandle; import io.trino.plugin.iceberg.IcebergUtil; @@ -36,10 +31,15 @@ import io.trino.spi.connector.ConnectorTransactionHandle; import io.trino.spi.connector.DynamicFilter; import io.trino.spi.type.TypeManager; +import org.apache.amoro.data.PrimaryKeyedFile; import org.apache.amoro.hive.io.reader.AdaptHiveMixedDeleteFilter; +import org.apache.amoro.scan.KeyedTableScanTask; +import org.apache.amoro.scan.MixedFileScanTask; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; +import org.apache.amoro.trino.delete.TrinoRow; +import org.apache.amoro.trino.unkeyed.IcebergPageSourceProvider; import org.apache.iceberg.Schema; import org.apache.iceberg.SchemaParser; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; import java.util.List; import java.util.stream.Collectors; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedTableHandle.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedTableHandle.java index fffa619ed4..fdf7bd31cc 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedTableHandle.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/keyed/KeyedTableHandle.java @@ -20,11 +20,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.amoro.table.PrimaryKeySpec; -import org.apache.amoro.trino.util.ObjectSerializerUtil; import io.trino.plugin.iceberg.IcebergColumnHandle; import io.trino.plugin.iceberg.IcebergTableHandle; import io.trino.spi.connector.ConnectorTableHandle; +import org.apache.amoro.table.PrimaryKeySpec; +import org.apache.amoro.trino.util.ObjectSerializerUtil; import java.util.Set; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergMetadata.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergMetadata.java index 34e449ee4b..ac6ef59d8c 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergMetadata.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergMetadata.java @@ -82,6 +82,14 @@ import static java.util.function.Function.identity; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.joining; +import static org.apache.amoro.shade.guava32.com.google.common.base.Preconditions.checkArgument; +import static org.apache.amoro.shade.guava32.com.google.common.base.Preconditions.checkState; +import static org.apache.amoro.shade.guava32.com.google.common.base.Verify.verify; +import static org.apache.amoro.shade.guava32.com.google.common.base.Verify.verifyNotNull; +import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap.toImmutableMap; +import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableSet.toImmutableSet; +import static org.apache.amoro.shade.guava32.com.google.common.collect.Maps.transformValues; import static org.apache.iceberg.FileContent.POSITION_DELETES; import static org.apache.iceberg.ReachableFileUtil.metadataFileLocations; import static org.apache.iceberg.ReachableFileUtil.versionHintLocation; @@ -92,17 +100,8 @@ import static org.apache.iceberg.TableProperties.DELETE_ISOLATION_LEVEL_DEFAULT; import static org.apache.iceberg.TableProperties.FORMAT_VERSION; import static org.apache.iceberg.TableProperties.WRITE_LOCATION_PROVIDER_IMPL; -import static org.apache.amoro.shade.guava32.com.google.common.base.Preconditions.checkArgument; -import static org.apache.amoro.shade.guava32.com.google.common.base.Preconditions.checkState; -import static org.apache.amoro.shade.guava32.com.google.common.base.Verify.verify; -import static org.apache.amoro.shade.guava32.com.google.common.base.Verify.verifyNotNull; -import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList.toImmutableList; -import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap.toImmutableMap; -import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableSet.toImmutableSet; -import static org.apache.amoro.shade.guava32.com.google.common.collect.Maps.transformValues; import static org.apache.iceberg.types.TypeUtil.indexParents; -import org.apache.amoro.table.MixedTable; import io.airlift.json.JsonCodec; import io.airlift.log.Logger; import io.airlift.slice.Slice; @@ -199,6 +198,15 @@ import io.trino.spi.statistics.TableStatisticsMetadata; import io.trino.spi.type.TypeManager; import org.apache.amoro.hive.utils.TableTypeUtil; +import org.apache.amoro.shade.guava32.com.google.common.base.Splitter; +import org.apache.amoro.shade.guava32.com.google.common.base.Suppliers; +import org.apache.amoro.shade.guava32.com.google.common.base.VerifyException; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableSet; +import org.apache.amoro.shade.guava32.com.google.common.collect.Iterables; +import org.apache.amoro.shade.guava32.com.google.common.collect.Sets; +import org.apache.amoro.table.MixedTable; import org.apache.datasketches.theta.CompactSketch; import org.apache.iceberg.AppendFiles; import org.apache.iceberg.ContentFile; @@ -233,14 +241,6 @@ import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.io.CloseableIterable; -import org.apache.amoro.shade.guava32.com.google.common.base.Splitter; -import org.apache.amoro.shade.guava32.com.google.common.base.Suppliers; -import org.apache.amoro.shade.guava32.com.google.common.base.VerifyException; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableSet; -import org.apache.amoro.shade.guava32.com.google.common.collect.Iterables; -import org.apache.amoro.shade.guava32.com.google.common.collect.Sets; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.IntegerType; @@ -274,8 +274,8 @@ import java.util.stream.Stream; /** - * Iceberg original metadata has some problems for mixed-format table, such as iceberg version, table type. So - * copy from IcebergMetadata and made some change + * Iceberg original metadata has some problems for mixed-format table, such as iceberg version, + * table type. So copy from IcebergMetadata and made some change */ public class IcebergMetadata implements ConnectorMetadata { private static final Logger log = Logger.get(io.trino.plugin.iceberg.IcebergMetadata.class); diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergPageSource.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergPageSource.java index 1738923c37..17e7155f22 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergPageSource.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergPageSource.java @@ -24,8 +24,6 @@ import static org.apache.amoro.shade.guava32.com.google.common.base.Preconditions.checkArgument; import static org.apache.amoro.shade.guava32.com.google.common.base.Throwables.throwIfInstanceOf; -import org.apache.amoro.io.reader.DeleteFilter; -import org.apache.amoro.trino.delete.TrinoRow; import io.trino.plugin.hive.ReaderProjectionsAdapter; import io.trino.plugin.iceberg.IcebergColumnHandle; import io.trino.plugin.iceberg.delete.IcebergPositionDeletePageSink; @@ -33,6 +31,8 @@ import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorPageSource; import io.trino.spi.type.Type; +import org.apache.amoro.io.reader.DeleteFilter; +import org.apache.amoro.trino.delete.TrinoRow; import org.apache.iceberg.io.CloseableIterable; import javax.annotation.Nullable; @@ -45,8 +45,8 @@ import java.util.function.Supplier; /** - * Iceberg original IcebergPageSource has some problems for mixed-format table, such as iceberg version, table - * type. + * Iceberg original IcebergPageSource has some problems for mixed-format table, such as iceberg + * version, table type. */ public class IcebergPageSource implements ConnectorPageSource { private final Type[] columnTypes; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergPageSourceProvider.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergPageSourceProvider.java index e07303bf31..ba3f69e01d 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergPageSourceProvider.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergPageSourceProvider.java @@ -70,21 +70,15 @@ import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toUnmodifiableList; -import static org.apache.iceberg.MetadataColumns.ROW_POSITION; import static org.apache.amoro.shade.guava32.com.google.common.base.Preconditions.checkState; import static org.apache.amoro.shade.guava32.com.google.common.base.Verify.verify; import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList.toImmutableList; import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap.toImmutableMap; import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableSet.toImmutableSet; import static org.apache.amoro.shade.guava32.com.google.common.collect.Maps.uniqueIndex; +import static org.apache.iceberg.MetadataColumns.ROW_POSITION; import static org.joda.time.DateTimeZone.UTC; -import org.apache.amoro.data.ChangeAction; -import org.apache.amoro.data.DataFileType; -import org.apache.amoro.io.reader.DeleteFilter; -import org.apache.amoro.table.MetadataColumns; -import org.apache.amoro.trino.delete.DummyFileScanTask; -import org.apache.amoro.trino.delete.TrinoRow; import io.airlift.json.JsonCodec; import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; @@ -149,6 +143,15 @@ import io.trino.spi.type.StandardTypes; import io.trino.spi.type.Type; import io.trino.spi.type.TypeManager; +import org.apache.amoro.data.ChangeAction; +import org.apache.amoro.data.DataFileType; +import org.apache.amoro.io.reader.DeleteFilter; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; +import org.apache.amoro.shade.guava32.com.google.common.graph.Traverser; +import org.apache.amoro.table.MetadataColumns; +import org.apache.amoro.trino.delete.DummyFileScanTask; +import org.apache.amoro.trino.delete.TrinoRow; import org.apache.avro.file.DataFileStream; import org.apache.avro.generic.GenericDatumReader; import org.apache.hadoop.hdfs.BlockMissingException; @@ -166,9 +169,6 @@ import org.apache.iceberg.mapping.NameMapping; import org.apache.iceberg.mapping.NameMappingParser; import org.apache.iceberg.parquet.ParquetSchemaUtil; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; -import org.apache.amoro.shade.guava32.com.google.common.graph.Traverser; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.FileMetaData; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergSplit.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergSplit.java index 6e7c9fab97..a8772dc516 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergSplit.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergSplit.java @@ -25,19 +25,20 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.amoro.data.DataFileType; -import org.apache.amoro.trino.delete.TrinoDeleteFile; import io.trino.plugin.iceberg.IcebergFileFormat; import io.trino.spi.HostAddress; import io.trino.spi.connector.ConnectorSplit; +import org.apache.amoro.data.DataFileType; import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; +import org.apache.amoro.trino.delete.TrinoDeleteFile; import org.openjdk.jol.info.ClassLayout; import java.util.List; /** - * Iceberg original IcebergSplit has some problems for mixed-format table, such as iceberg version, table type. + * Iceberg original IcebergSplit has some problems for mixed-format table, such as iceberg version, + * table type. */ public class IcebergSplit implements ConnectorSplit { private static final int INSTANCE_SIZE = diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergSplitManager.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergSplitManager.java index bfec818c41..39d823ecef 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergSplitManager.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergSplitManager.java @@ -22,8 +22,6 @@ import static io.trino.plugin.iceberg.IcebergSessionProperties.getMinimumAssignedSplitWeight; import static java.util.Objects.requireNonNull; -import org.apache.amoro.trino.MixedFormatTransactionManager; -import org.apache.amoro.trino.TableNameResolve; import io.airlift.units.Duration; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorSplitSource; @@ -37,15 +35,17 @@ import io.trino.spi.connector.DynamicFilter; import io.trino.spi.connector.FixedSplitSource; import io.trino.spi.type.TypeManager; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; +import org.apache.amoro.trino.MixedFormatTransactionManager; +import org.apache.amoro.trino.TableNameResolve; import org.apache.iceberg.Table; import org.apache.iceberg.TableScan; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; import javax.inject.Inject; /** - * Iceberg original IcebergSplitManager has some problems for mixed-format table, such as iceberg version, table - * type. + * Iceberg original IcebergSplitManager has some problems for mixed-format table, such as iceberg + * version, table type. */ public class IcebergSplitManager implements ConnectorSplitManager { diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergSplitSource.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergSplitSource.java index 17363cc644..3822dd55f7 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergSplitSource.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/IcebergSplitSource.java @@ -44,11 +44,6 @@ import static org.apache.amoro.shade.guava32.com.google.common.collect.Sets.intersection; import static org.apache.iceberg.types.Conversions.fromByteBuffer; -import org.apache.amoro.data.DataFileType; -import org.apache.amoro.data.PrimaryKeyedFile; -import org.apache.amoro.scan.MixedFileScanTask; -import org.apache.amoro.scan.ChangeTableIncrementalScan; -import org.apache.amoro.trino.delete.TrinoDeleteFile; import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.trino.filesystem.TrinoFileSystemFactory; @@ -71,6 +66,17 @@ import io.trino.spi.predicate.TupleDomain; import io.trino.spi.predicate.ValueSet; import io.trino.spi.type.TypeManager; +import org.apache.amoro.data.DataFileType; +import org.apache.amoro.data.PrimaryKeyedFile; +import org.apache.amoro.scan.ChangeTableIncrementalScan; +import org.apache.amoro.scan.MixedFileScanTask; +import org.apache.amoro.shade.guava32.com.google.common.annotations.VisibleForTesting; +import org.apache.amoro.shade.guava32.com.google.common.base.Stopwatch; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableSet; +import org.apache.amoro.shade.guava32.com.google.common.collect.Iterators; +import org.apache.amoro.shade.guava32.com.google.common.io.Closer; +import org.apache.amoro.trino.delete.TrinoDeleteFile; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; @@ -78,12 +84,6 @@ import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; -import org.apache.amoro.shade.guava32.com.google.common.annotations.VisibleForTesting; -import org.apache.amoro.shade.guava32.com.google.common.base.Stopwatch; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableSet; -import org.apache.amoro.shade.guava32.com.google.common.collect.Iterators; -import org.apache.amoro.shade.guava32.com.google.common.io.Closer; import org.apache.iceberg.types.Type; import org.apache.iceberg.util.TableScanUtil; @@ -102,8 +102,8 @@ import java.util.function.Supplier; /** - * Iceberg original IcebergSplitSource has some problems for mixed-format table, such as iceberg version, table - * type. + * Iceberg original IcebergSplitSource has some problems for mixed-format table, such as iceberg + * version, table type. */ public class IcebergSplitSource implements ConnectorSplitSource { private static final ConnectorSplitBatch EMPTY_BATCH = diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/TrinoDeleteFilter.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/TrinoDeleteFilter.java index 6665045598..93504fe88d 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/TrinoDeleteFilter.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/amoro/trino/unkeyed/TrinoDeleteFilter.java @@ -22,9 +22,9 @@ import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList.toImmutableList; import static org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableSet.toImmutableSet; +import io.trino.plugin.iceberg.IcebergColumnHandle; import org.apache.amoro.io.reader.DeleteFilter; import org.apache.amoro.trino.delete.TrinoRow; -import io.trino.plugin.iceberg.IcebergColumnHandle; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.Schema; import org.apache.iceberg.StructLike; @@ -38,8 +38,8 @@ import java.util.Set; /** - * Iceberg original TrinoDeleteFilter has some problems for mixed-format table, such as iceberg version, table - * type. + * Iceberg original TrinoDeleteFilter has some problems for mixed-format table, such as iceberg + * version, table type. */ public class TrinoDeleteFilter extends DeleteFilter { private final FileIO fileIO; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 104a092067..7275faa7ce 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,6 +18,10 @@ package org.apache.hadoop.hive.metastore; +import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME; +import static org.apache.hadoop.hive.metastore.MetaStoreUtils.isIndexTable; + +import org.apache.amoro.shade.guava32.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.classification.InterfaceAudience; @@ -123,7 +127,6 @@ import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; -import org.apache.amoro.shade.guava32.com.google.common.annotations.VisibleForTesting; import org.apache.thrift.TApplicationException; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; @@ -137,6 +140,7 @@ import org.slf4j.LoggerFactory; import javax.security.auth.login.LoginException; + import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationHandler; @@ -162,12 +166,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME; -import static org.apache.hadoop.hive.metastore.MetaStoreUtils.isIndexTable; - -/** - * Copy form hive 2.1.1 to change some code to adapt jdk 11. - */ +/** Copy form hive 2.1.1 to change some code to adapt jdk 11. */ @Public @Unstable public class HiveMetaStoreClient implements IMetaStoreClient { @@ -340,8 +339,8 @@ public boolean isCompatibleWith(HiveConf conf) { String newVar = conf.get(oneVar.varname, ""); if (oldVar == null || (oneVar.isCaseSensitive() - ? !oldVar.equals(newVar) - : !oldVar.equalsIgnoreCase(newVar))) { + ? !oldVar.equals(newVar) + : !oldVar.equalsIgnoreCase(newVar))) { LOG.info( "Mestastore configuration " + oneVar.varname @@ -385,7 +384,7 @@ public void reconnect() throws MetaException { * @throws MetaException * @throws TException * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#alter_table( - *java.lang.String, java.lang.String, org.apache.hadoop.hive.metastore.api.Table) + * java.lang.String, java.lang.String, org.apache.hadoop.hive.metastore.api.Table) */ @Override public void alter_table(String dbname, String tblName, Table newTbl) @@ -408,8 +407,8 @@ public void alter_table_with_environmentContext( * @throws MetaException * @throws TException * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#rename_partition( - *java.lang.String, java.lang.String, java.util.List, - * org.apache.hadoop.hive.metastore.api.Partition) + * java.lang.String, java.lang.String, java.util.List, + * org.apache.hadoop.hive.metastore.api.Partition) */ @Override public void renamePartition( @@ -599,7 +598,7 @@ public String getMetaConf(String key) throws TException { * @throws MetaException * @throws TException * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#add_partition( - *org.apache.hadoop.hive.metastore.api.Partition) + * org.apache.hadoop.hive.metastore.api.Partition) */ @Override public Partition add_partition(Partition newPart) @@ -655,8 +654,9 @@ public int add_partitions_pspec(PartitionSpecProxy partitionSpec) throws TExcept * @throws AlreadyExistsException * @throws MetaException * @throws TException - * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#append_partition(java.lang.String, - * java.lang.String, java.util.List) + * @see + * org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#append_partition(java.lang.String, + * java.lang.String, java.util.List) */ @Override public Partition appendPartition(String dbName, String tableName, List partVals) @@ -690,8 +690,8 @@ public Partition appendPartition( /** * Exchange the partition between two tables * - * @param partitionSpecs partitions specs of the parent partition to be exchanged - * @param destDb the db of the destination table + * @param partitionSpecs partitions specs of the parent partition to be exchanged + * @param destDb the db of the destination table * @param destinationTableName the destination table name @ @return new partition after exchanging */ @Override @@ -709,10 +709,10 @@ public Partition exchange_partition( /** * Exchange the partitions between two tables * - * @param partitionSpecs partitions specs of the parent partition to be exchanged - * @param destDb the db of the destination table + * @param partitionSpecs partitions specs of the parent partition to be exchanged + * @param destDb the db of the destination table * @param destinationTableName the destination table name @ @return new partitions after - * exchanging + * exchanging */ @Override public List exchange_partitions( @@ -753,7 +753,7 @@ public void createDatabase(Database db) * @throws NoSuchObjectException * @throws TException * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#create_table( - *org.apache.hadoop.hive.metastore.api.Table) + * org.apache.hadoop.hive.metastore.api.Table) */ @Override public void createTable(Table tbl) throws MetaException, NoSuchObjectException, TException { @@ -827,7 +827,7 @@ public void addForeignKey(List foreignKeyCols) throws TException * @throws MetaException * @throws TException * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#create_type( - *org.apache.hadoop.hive.metastore.api.Type) + * org.apache.hadoop.hive.metastore.api.Type) */ public boolean createType(Type type) throws AlreadyExistsException, InvalidObjectException, MetaException, TException { @@ -841,7 +841,7 @@ public boolean createType(Type type) * @throws MetaException * @throws TException * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_database( - *java.lang.String, boolean, boolean) + * java.lang.String, boolean, boolean) */ @Override public void dropDatabase(String name) @@ -905,8 +905,9 @@ public boolean dropPartition(String dbName, String tableName, String partName, b * @throws NoSuchObjectException * @throws MetaException * @throws TException - * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_partition(java.lang.String, - * java.lang.String, java.util.List, boolean) + * @see + * org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_partition(java.lang.String, + * java.lang.String, java.util.List, boolean) */ @Override public boolean dropPartition( @@ -946,8 +947,9 @@ public boolean dropPartition( * @throws NoSuchObjectException * @throws MetaException * @throws TException - * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_partition(java.lang.String, - * java.lang.String, java.util.List, boolean) + * @see + * org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_partition(java.lang.String, + * java.lang.String, java.util.List, boolean) */ public boolean dropPartition(String dbName, String tblName, List partVals) throws NoSuchObjectException, MetaException, TException { @@ -1072,9 +1074,7 @@ public void dropTable( dropTable(dbname, name, deleteData, ignoreUnknownTab, envContext); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override @Deprecated public void dropTable(String tableName, boolean deleteData) throws TException { @@ -1095,15 +1095,16 @@ public void dropTable(String dbname, String name) throws TException { * * @param dbname * @param name - * @param deleteData delete the underlying data or just delete the table in metadata + * @param deleteData delete the underlying data or just delete the table in metadata * @param ignoreUnknownTab don't throw if the requested table doesn't exist - * @param envContext for communicating with thrift - * @throws MetaException could not drop table properly - * @throws NoSuchObjectException the table wasn't found - * @throws TException a thrift communication error occurred + * @param envContext for communicating with thrift + * @throws MetaException could not drop table properly + * @throws NoSuchObjectException the table wasn't found + * @throws TException a thrift communication error occurred * @throws UnsupportedOperationException dropping an index table is not allowed - * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_table(java.lang.String, - * java.lang.String, boolean) + * @see + * org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_table(java.lang.String, + * java.lang.String, boolean) */ public void dropTable( String dbname, @@ -1162,7 +1163,8 @@ public boolean dropType(String type) throws MetaException, TException { * @return map of types * @throws MetaException * @throws TException - * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#get_type_all(java.lang.String) + * @see + * org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#get_type_all(java.lang.String) */ public Map getTypeAll(String name) throws MetaException, TException { Map result = null; @@ -1176,9 +1178,7 @@ public Map getTypeAll(String name) throws MetaException, TExceptio return result; } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public List getDatabases(String databasePattern) throws MetaException { try { @@ -1189,9 +1189,7 @@ public List getDatabases(String databasePattern) throws MetaException { return null; } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public List getAllDatabases() throws MetaException { try { @@ -1259,12 +1257,12 @@ public List listPartitionsWithAuthInfo( /** * Get list of partitions matching specified filter * - * @param dbName the database name - * @param tblName the table name - * @param filter the filter string, for example "part1 = \"p1_abc\" and part2 <= "\p2_test\"". - * Filtering can be done only on string partition keys. + * @param dbName the database name + * @param tblName the table name + * @param filter the filter string, for example "part1 = \"p1_abc\" and part2 <= "\p2_test\"". + * Filtering can be done only on string partition keys. * @param maxParts the maximum number of partitions to return, all partitions are returned if -1 - * is passed + * is passed * @return list of partitions * @throws MetaException * @throws NoSuchObjectException @@ -1332,7 +1330,8 @@ public boolean listPartitionsByExpr( * @throws NoSuchObjectException * @throws MetaException * @throws TException - * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#get_database(java.lang.String) + * @see + * org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#get_database(java.lang.String) */ @Override public Database getDatabase(String name) throws NoSuchObjectException, MetaException, TException { @@ -1347,8 +1346,9 @@ public Database getDatabase(String name) throws NoSuchObjectException, MetaExcep * @return the partition * @throws MetaException * @throws TException - * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#get_partition(java.lang.String, - * java.lang.String, java.util.List) + * @see + * org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#get_partition(java.lang.String, + * java.lang.String, java.util.List) */ @Override public Partition getPartition(String dbName, String tblName, List partVals) @@ -1391,7 +1391,7 @@ public Partition getPartitionWithAuthInfo( * @throws TException * @throws NoSuchObjectException * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#get_table(java.lang.String, - * java.lang.String) + * java.lang.String) */ @Override public Table getTable(String dbname, String name) @@ -1400,9 +1400,7 @@ public Table getTable(String dbname, String name) return fastpath ? t : deepCopy(filterHook.filterTable(t)); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override @Deprecated public Table getTable(String tableName) throws TException { @@ -1410,9 +1408,7 @@ public Table getTable(String tableName) throws TException { return fastpath ? t : filterHook.filterTable(t); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public List getTableObjectsByName(String dbName, List tableNames) throws TException { @@ -1420,9 +1416,7 @@ public List
getTableObjectsByName(String dbName, List tableNames) return fastpath ? tabs : deepCopyTables(filterHook.filterTables(tabs)); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public List listTableNamesByFilter(String dbName, String filter, short maxTables) throws TException { @@ -1442,9 +1436,7 @@ public Type getType(String name) throws NoSuchObjectException, MetaException, TE return deepCopy(client.get_type(name)); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public List getTables(String dbname, String tablePattern) throws MetaException { try { @@ -1486,9 +1478,7 @@ private List filterNames(List metas) throws MetaException return filtered; } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public List getAllTables(String dbname) throws MetaException { try { @@ -1508,9 +1498,7 @@ public boolean tableExists(String databaseName, String tableName) throws TExcept } } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override @Deprecated public boolean tableExists(String tableName) throws TException { @@ -1534,10 +1522,10 @@ public List listPartitionNames( /** * Get number of partitions matching specified filter * - * @param dbName the database name + * @param dbName the database name * @param tblName the table name - * @param filter the filter string, for example "part1 = \"p1_abc\" and part2 <= "\p2_test\"". - * Filtering can be done only on string partition keys. + * @param filter the filter string, for example "part1 = \"p1_abc\" and part2 <= "\p2_test\"". + * Filtering can be done only on string partition keys. * @return number of partitions * @throws MetaException * @throws NoSuchObjectException @@ -1577,8 +1565,9 @@ public void alterDatabase(String dbName, Database db) throws TException { * @throws UnknownDBException * @throws MetaException * @throws TException - * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#get_fields(java.lang.String, - * java.lang.String) + * @see + * org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#get_fields(java.lang.String, + * java.lang.String) */ @Override public List getFields(String db, String tableName) @@ -1590,7 +1579,7 @@ public List getFields(String db, String tableName) /** * create an index * - * @param index the index object + * @param index the index object * @param indexTable which stores the index data * @throws InvalidObjectException * @throws MetaException @@ -1601,7 +1590,7 @@ public List getFields(String db, String tableName) @Override public void createIndex(Index index, Table indexTable) throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, - TException { + TException { client.add_index(index, indexTable); } @@ -1613,8 +1602,9 @@ public void createIndex(Index index, Table indexTable) * @throws InvalidOperationException * @throws MetaException * @throws TException - * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#alter_index(java.lang.String, - * java.lang.String, java.lang.String, org.apache.hadoop.hive.metastore.api.Index) + * @see + * org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#alter_index(java.lang.String, + * java.lang.String, java.lang.String, org.apache.hadoop.hive.metastore.api.Index) */ @Override public void alter_index(String dbname, String baseTblName, String idxName, Index newIdx) @@ -1682,9 +1672,7 @@ public List getForeignKeys(ForeignKeysRequest req) throws TExcept return client.get_foreign_keys(req).getForeignKeys(); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override @Deprecated // use setPartitionColumnStatistics instead @@ -1692,9 +1680,7 @@ public boolean updateTableColumnStatistics(ColumnStatistics statsObj) throws TEx return client.update_table_column_statistics(statsObj); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override @Deprecated // use setPartitionColumnStatistics instead @@ -1702,9 +1688,7 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics statsObj) throws return client.update_partition_column_statistics(statsObj); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public boolean setPartitionColumnStatistics(SetPartitionsStatsRequest request) throws TException { return client.set_aggr_stats_for(request); @@ -1720,9 +1704,7 @@ public void flushCache() { } } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public List getTableColumnStatistics( String dbName, String tableName, List colNames) throws TException { @@ -1731,9 +1713,7 @@ public List getTableColumnStatistics( .getTableStats(); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public Map> getPartitionColumnStatistics( String dbName, String tableName, List partNames, List colNames) @@ -1744,18 +1724,14 @@ public Map> getPartitionColumnStatistics( .getPartStats(); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public boolean deletePartitionColumnStatistics( String dbName, String tableName, String partName, String colName) throws TException { return client.delete_partition_column_statistics(dbName, tableName, partName, colName); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public boolean deleteTableColumnStatistics(String dbName, String tableName, String colName) throws TException { @@ -1769,8 +1745,9 @@ public boolean deleteTableColumnStatistics(String dbName, String tableName, Stri * @throws UnknownDBException * @throws MetaException * @throws TException - * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#get_schema(java.lang.String, - * java.lang.String) + * @see + * org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#get_schema(java.lang.String, + * java.lang.String) */ @Override public List getSchema(String db, String tableName) @@ -2339,7 +2316,7 @@ public static IMetaStoreClient newSynchronizedClient(IMetaStoreClient client) { return (IMetaStoreClient) Proxy.newProxyInstance( HiveMetaStoreClient.class.getClassLoader(), - new Class[]{IMetaStoreClient.class}, + new Class[] {IMetaStoreClient.class}, new SynchronizedHandler(client)); } diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/hadoop/util/VersionInfo.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/hadoop/util/VersionInfo.java index bb8183232e..dc4e31e8dc 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/hadoop/util/VersionInfo.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/hadoop/util/VersionInfo.java @@ -178,5 +178,4 @@ public static String getBuildVersion() { public static String getProtocVersion() { return COMMON_VERSION_INFO._getProtocVersion(); } - } diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/parquet/io/ColumnIOUtil.java b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/parquet/io/ColumnIOUtil.java index eb962602c3..61c0885d44 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/parquet/io/ColumnIOUtil.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/main/java/org/apache/parquet/io/ColumnIOUtil.java @@ -1,7 +1,11 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/HiveTestRecords.java b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/HiveTestRecords.java index 7ba1a00094..2eb9d5efd6 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/HiveTestRecords.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/HiveTestRecords.java @@ -31,10 +31,10 @@ import static org.apache.amoro.trino.mixed.TestHiveTableBaseForTrino.HIVE_TABLE_SCHEMA; import static org.apache.amoro.trino.mixed.TestHiveTableBaseForTrino.STRUCT_SUB_SCHEMA; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.Record; import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; import java.math.BigDecimal; import java.time.LocalDateTime; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/MixedFormatQueryRunner.java b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/MixedFormatQueryRunner.java index d94fa611ed..8deb7f12e9 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/MixedFormatQueryRunner.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/MixedFormatQueryRunner.java @@ -25,8 +25,8 @@ import io.airlift.log.Logger; import io.trino.plugin.tpch.TpchPlugin; import io.trino.testing.DistributedQueryRunner; -import org.apache.amoro.trino.MixedFormatPlugin; import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; +import org.apache.amoro.trino.MixedFormatPlugin; import java.io.File; import java.util.HashMap; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TableTestBaseForTrino.java b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TableTestBaseForTrino.java index f057107f11..1a4e57c468 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TableTestBaseForTrino.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TableTestBaseForTrino.java @@ -21,24 +21,26 @@ import static org.apache.amoro.MockAmoroManagementServer.TEST_CATALOG_NAME; import static org.apache.amoro.MockAmoroManagementServer.TEST_DB_NAME; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testng.services.ManageTestResources; import org.apache.amoro.MockAmoroManagementServer; import org.apache.amoro.api.CatalogMeta; -import org.apache.amoro.mixed.MixedFormatCatalog; -import org.apache.amoro.mixed.CatalogLoader; import org.apache.amoro.catalog.CatalogTestHelper; import org.apache.amoro.data.ChangeAction; import org.apache.amoro.io.reader.GenericKeyedDataReader; import org.apache.amoro.io.writer.GenericBaseTaskWriter; import org.apache.amoro.io.writer.GenericChangeTaskWriter; import org.apache.amoro.io.writer.GenericTaskWriters; +import org.apache.amoro.mixed.CatalogLoader; +import org.apache.amoro.mixed.MixedFormatCatalog; import org.apache.amoro.scan.CombinedScanTask; +import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; +import org.apache.amoro.shade.guava32.com.google.common.collect.Sets; import org.apache.amoro.table.KeyedTable; import org.apache.amoro.table.PrimaryKeySpec; import org.apache.amoro.table.TableIdentifier; import org.apache.amoro.table.TableProperties; import org.apache.amoro.table.UnkeyedTable; -import io.trino.testing.AbstractTestQueryFramework; -import io.trino.testng.services.ManageTestResources; import org.apache.iceberg.AppendFiles; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; @@ -54,8 +56,6 @@ import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.WriteResult; -import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; -import org.apache.amoro.shade.guava32.com.google.common.collect.Sets; import org.apache.iceberg.types.Types; import org.junit.rules.TemporaryFolder; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TableTestBaseWithInitDataForTrino.java b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TableTestBaseWithInitDataForTrino.java index 6bc6cf506a..90aa52ede5 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TableTestBaseWithInitDataForTrino.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TableTestBaseWithInitDataForTrino.java @@ -23,14 +23,14 @@ import org.apache.amoro.io.writer.GenericChangeTaskWriter; import org.apache.amoro.io.writer.GenericTaskWriters; import org.apache.amoro.io.writer.SortedPosDeleteWriter; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; import org.apache.iceberg.AppendFiles; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; import org.apache.iceberg.io.WriteResult; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; import java.io.IOException; import java.time.LocalDateTime; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TestHiveTable.java b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TestHiveTable.java index 058420ad5d..2e42b17c17 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TestHiveTable.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TestHiveTable.java @@ -24,19 +24,22 @@ import static org.apache.amoro.table.TableProperties.DEFAULT_FILE_FORMAT; import static org.assertj.core.api.Assertions.assertThat; +import io.trino.sql.query.QueryAssertions; +import io.trino.testing.QueryRunner; import org.apache.amoro.MockAmoroManagementServer; import org.apache.amoro.data.ChangeAction; -import org.apache.amoro.table.MixedTable; +import org.apache.amoro.hive.io.writer.AdaptHiveGenericTaskWriterBuilder; +import org.apache.amoro.hive.table.HiveLocationKind; +import org.apache.amoro.hive.table.KeyedHiveTable; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; +import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; +import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.table.BaseLocationKind; import org.apache.amoro.table.ChangeLocationKind; import org.apache.amoro.table.LocationKind; +import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.TableIdentifier; import org.apache.amoro.table.TableProperties; -import io.trino.sql.query.QueryAssertions; -import io.trino.testing.QueryRunner; -import org.apache.amoro.hive.io.writer.AdaptHiveGenericTaskWriterBuilder; -import org.apache.amoro.hive.table.HiveLocationKind; -import org.apache.amoro.hive.table.KeyedHiveTable; import org.apache.iceberg.AppendFiles; import org.apache.iceberg.FileFormat; import org.apache.iceberg.Files; @@ -49,9 +52,6 @@ import org.apache.iceberg.io.TaskWriter; import org.apache.iceberg.io.WriteResult; import org.apache.iceberg.parquet.AdaptHiveParquet; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; -import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -66,13 +66,19 @@ public class TestHiveTable extends TestHiveTableBaseForTrino { private final String TEST_HIVE_TABLE_FULL_NAME = - MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + HIVE_TABLE_ID.getDatabase() + "." + HIVE_TABLE_ID.getTableName(); + MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + + HIVE_TABLE_ID.getDatabase() + + "." + + HIVE_TABLE_ID.getTableName(); private final String TEST_HIVE_PK_TABLE_FULL_NAME = - MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + HIVE_PK_TABLE_ID.getDatabase() + "." + HIVE_PK_TABLE_ID.getTableName(); + MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + + HIVE_PK_TABLE_ID.getDatabase() + + "." + + HIVE_PK_TABLE_ID.getTableName(); private final String TEST_HIVE_PK_TABLE_FULL_NAME_BASE = - MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + HIVE_PK_TABLE_ID.getDatabase() + "." + "\"" @@ -80,19 +86,19 @@ public class TestHiveTable extends TestHiveTableBaseForTrino { + "#base\""; private final String TEST_UN_PARTITION_HIVE_TABLE_FULL_NAME = - MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + UN_PARTITION_HIVE_TABLE_ID.getDatabase() + "." + UN_PARTITION_HIVE_TABLE_ID.getTableName(); private final String TEST_UN_PARTITION_HIVE_PK_TABLE_FULL_NAME = - MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + UN_PARTITION_HIVE_PK_TABLE_ID.getDatabase() + "." + UN_PARTITION_HIVE_PK_TABLE_ID.getTableName(); private final String TEST_UN_PARTITION_HIVE_PK_TABLE_FULL_NAME_BASE = - MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + UN_PARTITION_HIVE_PK_TABLE_ID.getDatabase() + "." + "\"" @@ -106,10 +112,16 @@ public class TestHiveTable extends TestHiveTableBaseForTrino { TableIdentifier.of(TEST_CATALOG_NAME, HIVE_DB_NAME, "test_pk_hive_table_parquet"); private final String TEST_HIVE_PK_TABLE_ORC_FULL_NAME = - MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + HIVE_PK_TABLE_ID.getDatabase() + "." + HIVE_PK_TABLE_ORC_ID.getTableName(); + MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + + HIVE_PK_TABLE_ID.getDatabase() + + "." + + HIVE_PK_TABLE_ORC_ID.getTableName(); private final String TEST_HIVE_PK_TABLE_PARQUET_FULL_NAME = - MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + HIVE_PK_TABLE_ID.getDatabase() + "." + HIVE_PK_TABLE_PARQUET_ID.getTableName(); + MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + + HIVE_PK_TABLE_ID.getDatabase() + + "." + + HIVE_PK_TABLE_PARQUET_ID.getTableName(); private KeyedHiveTable testKeyedHiveTableOrc; private KeyedHiveTable testKeyedHiveTableParquet; @@ -334,13 +346,13 @@ private void write(MixedTable table, LocationKind locationKind, List rec } private void write( - MixedTable table, LocationKind locationKind, List records, ChangeAction changeAction) + MixedTable table, LocationKind locationKind, List records, ChangeAction changeAction) throws IOException { write(table, locationKind, records, changeAction, null); } private void write( - MixedTable table, LocationKind locationKind, List records, FileFormat fileFormat) + MixedTable table, LocationKind locationKind, List records, FileFormat fileFormat) throws IOException { write(table, locationKind, records, ChangeAction.INSERT, fileFormat); } diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TestHiveTableBaseForTrino.java b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TestHiveTableBaseForTrino.java index de0c3d2d47..27a9a30afb 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TestHiveTableBaseForTrino.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TestHiveTableBaseForTrino.java @@ -20,17 +20,19 @@ import static org.apache.amoro.MockAmoroManagementServer.TEST_CATALOG_NAME; -import org.apache.amoro.TableFormat; -import org.apache.amoro.mixed.CatalogLoader; -import org.apache.amoro.table.MixedTable; -import org.apache.amoro.table.TableIdentifier; -import org.apache.amoro.table.TableProperties; import io.trino.testng.services.ManageTestResources; +import org.apache.amoro.TableFormat; import org.apache.amoro.hive.HMSMockServer; -import org.apache.amoro.hive.catalog.MixedHiveCatalog; import org.apache.amoro.hive.catalog.HiveCatalogTestHelper; +import org.apache.amoro.hive.catalog.MixedHiveCatalog; import org.apache.amoro.hive.table.KeyedHiveTable; import org.apache.amoro.hive.table.UnkeyedHiveTable; +import org.apache.amoro.mixed.CatalogLoader; +import org.apache.amoro.shade.guava32.com.google.common.base.Joiner; +import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; +import org.apache.amoro.table.MixedTable; +import org.apache.amoro.table.TableIdentifier; +import org.apache.amoro.table.TableProperties; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Partition; @@ -39,8 +41,6 @@ import org.apache.iceberg.DataFiles; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; -import org.apache.amoro.shade.guava32.com.google.common.base.Joiner; -import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; import org.apache.thrift.TException; import org.junit.Assert; diff --git a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TestKeyedTable.java b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TestKeyedTable.java index 9256f62e8e..a6e474f3c4 100644 --- a/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TestKeyedTable.java +++ b/amoro-mixed-format/amoro-mixed-format-trino/src/test/java/org/apache/amoro/trino/mixed/TestKeyedTable.java @@ -21,20 +21,22 @@ import static org.apache.amoro.MockAmoroManagementServer.TEST_CATALOG_NAME; import static org.assertj.core.api.Assertions.assertThat; +import io.trino.testing.QueryRunner; import org.apache.amoro.TableFormat; import org.apache.amoro.TestedCatalogs; import org.apache.amoro.catalog.CatalogTestHelper; -import io.trino.testing.QueryRunner; import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; public class TestKeyedTable extends TableTestBaseWithInitDataForTrino { - private static final String CATALOG_DATABASE_NAME = MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + "test_db."; + private static final String CATALOG_DATABASE_NAME = + MixedFormatQueryRunner.MIXED_FORMAT_CATALOG_PREFIX + "test_db."; private static final String PK_TABLE_FULL_NAME = CATALOG_DATABASE_NAME + "test_pk_table"; private static final String PK_TABLE_BASE_NAME = CATALOG_DATABASE_NAME + "\"test_pk_table#base\""; - private static final String PK_TABLE_CHANGE_NAME = CATALOG_DATABASE_NAME + "\"test_pk_table#change\""; + private static final String PK_TABLE_CHANGE_NAME = + CATALOG_DATABASE_NAME + "\"test_pk_table#change\""; @Override protected QueryRunner createQueryRunner() throws Exception { @@ -88,7 +90,9 @@ public void baseQuery() { @Test public void baseQueryWhenTableNameContainCatalogAndDataBase() { assertQuery( - "select id from " + CATALOG_DATABASE_NAME + "\"test_mixed_format_catalog.test_db.test_pk_table#base\"", + "select id from " + + CATALOG_DATABASE_NAME + + "\"test_mixed_format_catalog.test_db.test_pk_table#base\"", "VALUES 1, 2, 3"); } @@ -96,7 +100,7 @@ public void baseQueryWhenTableNameContainCatalogAndDataBase() { public void baseQueryWhenTableNameContainDataBase() { assertQuery( "select id from " + CATALOG_DATABASE_NAME + "\"test_db.test_pk_table#base\"", - "VALUES 1, 2, 3"); + "VALUES 1, 2, 3"); } @Test @@ -111,7 +115,9 @@ public void changeQuery() { @Test public void changeQueryWhenTableNameContainCatalogAndDataBase() { assertQuery( - "select * from " + CATALOG_DATABASE_NAME + "\"test_mixed_format_catalog.test_db.test_pk_table#change\"", + "select * from " + + CATALOG_DATABASE_NAME + + "\"test_mixed_format_catalog.test_db.test_pk_table#change\"", "VALUES (6,'mack',TIMESTAMP '2022-01-01 12:00:00.000000' ,3,1,'INSERT')," + "(5,'mary',TIMESTAMP '2022-01-01 12:00:00.000000',2,1,'INSERT')," + "(5,'mary',TIMESTAMP '2022-01-01 12:00:00.000000',4,1,'DELETE')"); diff --git a/pom.xml b/pom.xml index fc7d2e9434..160fb50391 100644 --- a/pom.xml +++ b/pom.xml @@ -1163,6 +1163,25 @@ apache-release + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.apache.resources + apache-source-release-assembly-descriptor + 1.0.6 + + + + + source-release-assembly + + none + + + + org.apache.maven.plugins maven-javadoc-plugin @@ -1213,66 +1232,5 @@ 3.3 - - deploy-maven-central - - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - - - - org.apache.maven.plugins - maven-source-plugin - ${maven-source-plugin-version} - - - attach-sources - - jar - - - - - - org.apache.maven.plugins - maven-gpg-plugin - ${maven-gpg-plugin.version} - - - sign-release-artifacts - - sign - - verify - - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven-javadoc-plugin.version} - - true - none - - - - attach-javadocs - - jar - - - - - - -