Merge branch 'apache:master' into master

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -47,13 +47,23 @@ body:
     validations:
       required: true
 
+  - type: dropdown
+    id: table-format
+    attributes:
+      label: What table format are you seeing the problem on?
+      multiple: true
+      options:
+        - Iceberg
+        - Paimon
+        - Mixed-Iceberg
+        - Mixed-Hive
+
   - type: dropdown
     id: engines
     attributes:
       label: What engines are you seeing the problem on?
       multiple: true
       options:
-        - Core
         - AMS
         - Optimizer
         - Flink

.github/workflows/core-hadoop2-ci.yml

@@ -18,6 +18,7 @@ on:
       - "master"
       - "0.**"
     paths:
+      - ".github/workflows/**"
       - "amoro-ams/**"
       - "amoro-core/**"
       - "amoro-mixed-format/amoro-mixed-format-flink/**"
@@ -43,7 +44,7 @@ jobs:
         run: mvn validate
 
       - name: Build all module with Maven
-        run: mvn clean install -pl '!amoro-mixed-format/amoro-mixed-format-trino' -Djacoco.flink.skip=true -B -ntp -Dhadoop=v2
+        run: mvn clean install -Djacoco.flink.skip=true -B -ntp -Dhadoop=v2
 
       - name: Code coverage
         uses: codecov/codecov-action@v3

.github/workflows/core-hadoop3-ci.yml

@@ -18,6 +18,7 @@ on:
       - "master"
       - "0.**"
     paths:
+      - ".github/workflows/**"
       - "amoro-ams/**"
       - "amoro-core/**"
       - "amoro-mixed-format/amoro-mixed-format-flink/**"
@@ -43,7 +44,7 @@ jobs:
         run: mvn validate
 
       - name: Build all module with Maven
-        run: mvn clean install -pl '!amoro-mixed-format/amoro-mixed-format-trino' -Djacoco.flink.skip=true -B -ntp
+        run: mvn clean install -Djacoco.flink.skip=true -B -ntp
 
       - name: Code coverage
         uses: codecov/codecov-action@v3

.github/workflows/docker-images.yml

@@ -147,8 +147,8 @@ jobs:
           OPTIMIZER_FLINK=${{ matrix.flink }} && \
           echo "OPTIMIZER_FLINK=-Dflink-optimizer.flink-version${OPTIMIZER_FLINK}" >> $GITHUB_ENV
           if [[ "$OPTIMIZER_FLINK" < "1.15" ]]; then
-            echo "Adding -Pflink-pre-1.15 for Flink version less than 1.15"
-            echo "OPTIMIZER_FLINK=-Pflink-pre-1.15 -Dflink-optimizer.flink-version=${OPTIMIZER_FLINK}" >> $GITHUB_ENV
+            echo "Adding -Pflink-optimizer-pre-1.15 for Flink version less than 1.15"
+            echo "OPTIMIZER_FLINK=-Pflink-optimizer-pre-1.15 -Dflink-optimizer.flink-version=${OPTIMIZER_FLINK}" >> $GITHUB_ENV
           fi
 
       - name: Set ENV Amoro version

.github/workflows/trino-ci.yml

@@ -18,6 +18,7 @@ on:
       - "master"
       - "0.**"
     paths:
+      - ".github/workflows/**"
       - "amoro-ams/**"
       - "amoro-core/**"
       - "amoro-mixed-format/amoro-mixed-format-hive/**"
@@ -40,11 +41,8 @@ jobs:
       - name: Validate checkstyle first
         run: mvn validate -P trino-spotless
 
-      - name: Install dependency with Maven
-        run: mvn clean install -DskipTests -pl 'amoro-ams/amoro-ams-api,amoro-core,amoro-mixed-format/amoro-mixed-format-hive' -Dhadoop=v2 -am -B -P trino-spotless
-
-      - name: Build with Maven
-        run: mvn clean test -pl 'amoro-mixed-format/amoro-mixed-format-trino' -B -ntp -Dhadoop=v2 -P trino-spotless
+      - name: Build trino module with Maven
+        run: mvn clean install -pl 'amoro-mixed-format/amoro-mixed-format-trino' -Dhadoop=v2 -am -B -P trino-spotless,build-mixed-format-trino
 
       - name: Code coverage
         uses: codecov/codecov-action@v3

.gitignore

@@ -71,5 +71,5 @@ conf/
 amoro-ams/amoro-ams-dashboard/src/main/resources/static/
 amoro-ams/amoro-ams-dashboard/node/
 
-!amoro-ams/dist/src/main/arctic-bin/bin/
-!amoro-ams/dist/src/main/arctic-bin/conf/
+!amoro-ams/dist/src/main/amoro-bin/bin/
+!amoro-ams/dist/src/main/amoro-bin/conf/

CONTRIBUTING.md

@@ -80,37 +80,7 @@ for reference.
 
 ## Building the Project Locally
 
-Amoro is built using Maven with Java 1.8 and Java 17(only for `amoro-mixed-format/amoro-mixed-format-trino` module).
-
-* To build Trino module need config `toolchains.xml` in `${user.home}/.m2/` dir, the content is
-
-```
-<?xml version="1.0" encoding="UTF-8"?>
-<toolchains>
-    <toolchain>
-        <type>jdk</type>
-        <provides>
-            <version>17</version>
-            <vendor>sun</vendor>
-        </provides>
-        <configuration>
-            <jdkHome>${YourJDK17Home}</jdkHome>
-        </configuration>
-    </toolchain>
-</toolchains>
-```
-
-* To invoke a build and run tests: `mvn package -P toolchain`
-* To skip tests: `mvn -DskipTests package -P toolchain`
-* To package without trino module and JAVA 17 dependency: `mvn clean package -DskipTests -pl '!amoro-mixed-format/amoro-mixed-format-trino'`
-* To build with hadoop 2.x(the default is 3.x) `mvn clean package -DskipTests -Dhadoop=v2`
-* To indicate Flink version for optimizer (the default is 1.18.1): `mvn clean package -Dflink-optimizer.flink-version=1.15.4`. If the version of Flink is below 1.15.0, you also need to add the `-Pflink-pre-1.15` parameter: `mvn clean package -Pflink-pre-1.15 -Dflink-optimizer.flink-version=1.14.6`.
-  `mvn clean package -Pflink-pre-1.15 -Dflink-optimizer.flink-version=1.14.6 -DskipTests`
-
->Spotless is skipped by default in `trino` module. So if you want to perform checkstyle when building `trino` module, you must be in a Java 17 environment.
-
-* To invoke a build include `amoro-mixed-format/amoro-mixed-format-trino` module in Java 17 environment: `mvn clean package -DskipTests -P trino-spotless`
-* To only build `amoro-mixed-format/amoro-mixed-format-trino` and its dependent modules in Java 17 environment: `mvn clean package -DskipTests -P trino-spotless -pl 'amoro-mixed-format/amoro-mixed-format-trino' -am`
+[Build Guide](https://github.com/apache/amoro?tab=readme-ov-file#building) can be found in GitHub readme.
 
 ## Code suggestions
 

DISCLAIMER

@@ -0,0 +1,10 @@
+Apache Amoro (incubating) is an effort undergoing incubation at the Apache
+Software Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further review
+indicates that the infrastructure, communications, and decision making process
+have stabilized in a manner consistent with other successful ASF projects.
+
+While incubation status is not necessarily a reflection of the completeness
+or stability of the code, it does indicate that the project has yet to be
+fully endorsed by the ASF.

LICENSE

@@ -201,46 +201,4 @@
    See the License for the specific language governing permissions and
    limitations under the License.
 
---------------------------------------------------------------------------------
-
-This product includes code from Apache Iceberg.
-
-* org.apache.iceberg.data.DeleteFilter copied and modified to org.apache.amoro.io.reader.DeleteFilter
-* org.apache.iceberg.deletes.Deletes copied and modified to org.apache.amoro.io.reader.Deletes
-* org.apache.iceberg.data.parquet.BaseParquetReaders copied and modified to org.apache.iceberg.data.parquet.AdaptHiveBaseParquetReaders
-* org.apache.iceberg.data.parquet.BaseParquetWriter copied and modified to org.apache.iceberg.data.parquet.AdaptHiveBaseParquetWriter
-* org.apache.iceberg.data.parquet.GenericParquetReaders copied and modified to org.apache.iceberg.data.parquet.AdaptHiveGenericParquetReaders
-* org.apache.iceberg.data.parquet.GenericParquetWriter copied and modified to org.apache.iceberg.data.parquet.AdaptHiveGenericParquetWriter
-* org.apache.iceberg.data.GenericAppenderFactory copied and modified to org.apache.iceberg.data.AdaptHiveGenericAppenderFactory
-* org.apache.iceberg.parquet.ColumnWriter copied and modified to org.apache.iceberg.parquet.AdaptHiveColumnWriter
-* org.apache.iceberg.parquet.Parquet copied and modified to org.apache.iceberg.parquet.AdaptHiveParquet
-* org.apache.iceberg.parquet.ParquetSchemaUtil copied and modified to org.apache.iceberg.parquet.AdaptHiveParquetSchemaUtil
-* org.apache.iceberg.parquet.ParquetWriter copied and modified to org.apache.iceberg.parquet.AdaptHiveParquetWriter
-* org.apache.iceberg.parquet.PrimitiveWriter copied and modified to org.apache.iceberg.parquet.AdaptHivePrimitiveWriter
-* org.apache.iceberg.parquet.TypeToMessageType copied and modified to org.apache.iceberg.parquet.AdaptHiveTypeToMessageType
-* org.apache.iceberg.parquet.ParquetConversions copied and modified to org.apache.iceberg.parquet.AdaptHiveParquetConversions
-* org.apache.iceberg.parquet.ParquetDictionaryRowGroupFilter copied and modified to org.apache.iceberg.parquet.AdaptHiveParquetDictionaryRowGroupFilter
-* org.apache.iceberg.parquet.ParquetMetricsRowGroupFilter copied and modified to org.apache.iceberg.parquet.AdaptHiveParquetMetricsRowGroupFilter
-* org.apache.iceberg.parquet.ParquetReader copied and modified to org.apache.iceberg.parquet.AdaptHiveParquetReader
-* org.apache.iceberg.parquet.ReadConf copied and modified to org.apache.iceberg.parquet.AdaptHiveReadConf
-* org.apache.iceberg.spark.data.ParquetWithSparkSchemaVisitor copied and modified to org.apache.iceberg.spark.data.AdaptHiveParquetWithSparkSchemaVisitor
-* org.apache.iceberg.flink.data.ParquetWithFlinkSchemaVisitor copied and modified to org.apache.iceberg.flink.data.AdaptHiveParquetWithFlinkSchemaVisitor
-* org.apache.iceberg.parquet.ParquetUtil copied and modified to org.apache.iceberg.parquet.AdaptHiveParquetUtil
-* org.apache.paimon.spark.SparkTypeUtils copied and modified to org.apache.amoro.server.dashboard.component.reverser.PaimonTypeToSparkType
-* org.apache.iceberg.spark.TypeToSparkType copied and modified to org.apache.amoro.server.dashboard.component.reverser.IcebergTypeToSparkType
-
-
-
-Copyright: 2011-2019 The Apache Software Foundation
-Home page: http://iceberg.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This product includes code from Apache Paimon (incubating).
-
-* org.apache.paimon.spark.SparkTypeUtils copied and modified to org.apache.amoro.server.dashboard.component.reverser.PaimonTypeToSparkType
-
-Copyright: 2011-2019 The Apache Software Foundation
-Home page: http://paimon.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
+--------------------------------------------------------------------------------

NOTICE

@@ -3,3 +3,12 @@ Copyright 2024 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
+
+Apache Iceberg
+Copyright 2017-2024 The Apache Software Foundation
+
+Apache Paimon
+Copyright 2023-2024 The Apache Software Foundation
+
+Apache Flink
+Copyright 2014-2024 The Apache Software Foundation

README.md

@@ -110,9 +110,17 @@ Amoro contains modules as below:
 
 ## Building
 
-Amoro is built using Maven with Java 1.8 and Java 17(only for `amoro-mixed-format/amoro-mixed-format-trino` module).
-
-* To build Trino module need config `toolchains.xml` in `${user.home}/.m2/` dir, the content is
+Amoro is built using Maven with JDK 8 and JDK 17(only for `amoro-mixed-format/amoro-mixed-format-trino` module).
+
+* Build all modules without `amoro-mixed-format-trino`: `mvn clean package`
+* Build and skip tests: `mvn clean package -DskipTests`
+* Build and skip dashboard: `mvn clean package -Pskip-dashboard-build`
+* Build with hadoop 2.x(the default is 3.x) dependencies: `mvn clean package -DskipTests -Dhadoop=v2`
+* Specify Flink version for Flink optimizer(the default is 1.18.1): `mvn clean package -DskipTests -Dflink-optimizer.flink-version=1.15.4`
+  * If the version of Flink is below 1.15.0, you also need to add the `-Pflink-optimizer-pre-1.15` parameter: `mvn clean package -DskipTests -Pflink-optimizer-pre-1.15 -Dflink-optimizer.flink-version=1.14.6`
+* Specify Spark version for Spark optimizer(the default is 3.3.3): `mvn clean package -DskipTests -Dspark-optimizer.spark-version=3.2.2`
+* Build `amoro-mixed-format-trino` module under JDK 17: `mvn clean package -DskipTests -Ptrino-spotless,build-mixed-format-trino -pl 'amoro-mixed-format/amoro-mixed-format-trino' -am`.
+* Build all modules: `mvn clean package -DskipTests -Ptoolchain,build-mixed-format-trino`, besides you need config `toolchains.xml` in `${user.home}/.m2/` dir with content below.
 
 ```
 <?xml version="1.0" encoding="UTF-8"?>
@@ -130,18 +138,6 @@ Amoro is built using Maven with Java 1.8 and Java 17(only for `amoro-mixed-forma
 </toolchains>
 ```
 
-* To invoke a build and run tests: `mvn package -P toolchain`
-* To skip tests: `mvn -DskipTests package -P toolchain`
-* To package without trino module and JAVA 17 dependency: `mvn clean package -DskipTests -pl '!amoro-mixed-format/amoro-mixed-format-trino'`
-* To build with hadoop 2.x(the default is 3.x) `mvn clean package -DskipTests -Dhadoop=v2`
-* To indicate Flink version for optimizer (the default is 1.18.1): `mvn clean package -Dflink-optimizer.flink-version=1.15.4`. If the version of Flink is below 1.15.0, you also need to add the `-Pflink-pre-1.15` parameter: `mvn clean package -Pflink-pre-1.15 -Dflink-optimizer.flink-version=1.14.6`.
-  `mvn clean package -Pflink-pre-1.15 -Dflink-optimizer.flink-version=1.14.6 -DskipTests`
-
->Spotless is skipped by default in `trino` module. So if you want to perform checkstyle when building `trino` module, you must be in a Java 17 environment.
-
-* To invoke a build include `amoro-mixed-format/amoro-mixed-format-trino` module in Java 17 environment: `mvn clean package -DskipTests -P trino-spotless`
-* To only build `amoro-mixed-format/amoro-mixed-format-trino` and its dependent modules in Java 17 environment: `mvn clean package -DskipTests -P trino-spotless -pl 'amoro-mixed-format/amoro-mixed-format-trino' -am`
-
 ## Quickstart
 
 Visit [https://amoro.apache.org/quick-demo/](https://amoro.apache.org/quick-demo/) to quickly

amoro-ams/amoro-ams-api/pom.xml

@@ -53,12 +53,7 @@
             <version>${curator.version}</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-annotations</artifactId>
-            <version>2.10.2</version>
-            <scope>provided</scope>
-        </dependency>
+
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>

amoro-ams/amoro-ams-api/src/main/java/org/apache/amoro/api/Action.java

@@ -20,7 +20,7 @@
 
 import org.apache.amoro.api.process.TableProcess;
 import org.apache.amoro.api.process.TableProcessState;
-import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
 
 import java.util.Collections;
 import java.util.Set;

amoro-ams/amoro-ams-api/src/main/java/org/apache/amoro/api/config/ConfigHelpers.java

@@ -18,8 +18,8 @@
 
 package org.apache.amoro.api.config;
 
-import static org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkArgument;
-import static org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.amoro.shade.guava32.com.google.common.base.Preconditions.checkArgument;
+import static org.apache.amoro.shade.guava32.com.google.common.base.Preconditions.checkNotNull;
 
 import javax.annotation.Nonnull;
 

amoro-ams/amoro-ams-api/src/main/java/org/apache/amoro/api/config/ConfigOption.java

@@ -18,7 +18,7 @@
 
 package org.apache.amoro.api.config;
 
-import static org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.amoro.shade.guava32.com.google.common.base.Preconditions.checkNotNull;
 
 /**
  * A {@code ConfigOption} describes a configuration parameter. It encapsulates the configuration

amoro-ams/amoro-ams-api/src/main/java/org/apache/amoro/api/config/ConfigOptions.java

@@ -18,7 +18,7 @@
 
 package org.apache.amoro.api.config;
 
-import static org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.amoro.shade.guava32.com.google.common.base.Preconditions.checkNotNull;
 
 import java.time.Duration;
 import java.util.Arrays;

amoro-ams/amoro-ams-api/src/main/java/org/apache/amoro/api/config/Configurations.java

@@ -18,7 +18,7 @@
 
 package org.apache.amoro.api.config;
 
-import static org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.amoro.shade.guava32.com.google.common.base.Preconditions.checkNotNull;
 
 import java.util.Arrays;
 import java.util.HashMap;

amoro-ams/amoro-ams-api/src/main/java/org/apache/amoro/api/config/DataExpirationConfig.java

@@ -18,16 +18,16 @@
 
 package org.apache.amoro.api.config;
 
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.amoro.shade.guava32.com.google.common.annotations.VisibleForTesting;
+import org.apache.amoro.shade.guava32.com.google.common.base.Objects;
+import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions;
+import org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
+import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.amoro.table.MixedTable;
 import org.apache.amoro.table.TableProperties;
 import org.apache.amoro.utils.CompatiblePropertyUtil;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
-import org.apache.iceberg.relocated.com.google.common.base.Objects;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Sets;
 import org.apache.iceberg.types.Type;
 import org.apache.iceberg.types.Types;
 import org.slf4j.Logger;

amoro-ams/amoro-ams-api/src/main/java/org/apache/amoro/api/config/OptimizingConfig.java

@@ -18,12 +18,12 @@
 
 package org.apache.amoro.api.config;
 
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import org.apache.amoro.properties.HiveTableProperties;
+import org.apache.amoro.shade.guava32.com.google.common.base.MoreObjects;
+import org.apache.amoro.shade.guava32.com.google.common.base.Objects;
+import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import org.apache.amoro.table.TableProperties;
 import org.apache.amoro.utils.CompatiblePropertyUtil;
-import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
-import org.apache.iceberg.relocated.com.google.common.base.Objects;
 import org.apache.iceberg.util.PropertyUtil;
 
 import java.util.Map;

amoro-ams/amoro-ams-api/src/main/java/org/apache/amoro/api/config/StructuredOptionsSplitter.java

@@ -18,7 +18,7 @@
 
 package org.apache.amoro.api.config;
 
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions;
 
 import java.util.ArrayList;
 import java.util.Arrays;

amoro-ams/amoro-ams-api/src/main/java/org/apache/amoro/api/config/TableConfiguration.java

@@ -18,10 +18,10 @@
 
 package org.apache.amoro.api.config;
 
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.apache.amoro.shade.guava32.com.google.common.base.Objects;
+import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import org.apache.amoro.table.TableProperties;
 import org.apache.amoro.utils.CompatiblePropertyUtil;
-import org.apache.iceberg.relocated.com.google.common.base.Objects;
 
 import java.util.Map;
 import java.util.Optional;

amoro-ams/amoro-ams-api/src/main/java/org/apache/amoro/api/config/TagConfiguration.java

@@ -18,11 +18,11 @@
 
 package org.apache.amoro.api.config;
 
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.apache.amoro.shade.guava32.com.google.common.base.MoreObjects;
+import org.apache.amoro.shade.guava32.com.google.common.base.Objects;
+import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import org.apache.amoro.table.TableProperties;
 import org.apache.amoro.utils.CompatiblePropertyUtil;
-import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
-import org.apache.iceberg.relocated.com.google.common.base.Objects;
 
 import java.time.Duration;
 import java.time.LocalDateTime;