Merge branch 'apache:master' into master

.github/ISSUE_TEMPLATE/config.yml

@@ -17,5 +17,5 @@
 blank_issues_enabled: true
 contact_links:
   - name: GitHub Community Support
-    url: https://github.com/NetEase/amoro/discussions
+    url: https://github.com/apache/amoro/discussions
     about: Please ask and answer questions here.

.github/ISSUE_TEMPLATE/improvement.yml

@@ -29,7 +29,7 @@ body:
       label: Search before asking
       options:
         - label: >
-            I have searched in the [issues](https://github.com/NetEase/amoro/issues?q=is%3Aissue) and found no similar
+            I have searched in the [issues](https://github.com/apache/amoro/issues?q=is%3Aissue) and found no similar
             issues.
           required: true
 

.github/ISSUE_TEMPLATE/subtask.yml

@@ -29,7 +29,7 @@ body:
       label: Search before asking
       options:
         - label: >
-            I have searched in the [issues](https://github.com/NetEase/amoro/issues?q=is%3Aissue) and found no similar
+            I have searched in the [issues](https://github.com/apache/amoro/issues?q=is%3Aissue) and found no similar
             issues.
           required: true
 

.github/PULL_REQUEST_TEMPLATE

@@ -2,8 +2,8 @@
 Thanks for sending a pull request!
 
 Here are some tips for you:
-  1. If this is your first time, please read our contributor guidelines: https://amoro.netease.com/how-to-contribute/
-  2. If the PR is related to an issue in https://github.com/NetEase/amoro/issues, add '[AMORO-XXXX]' in your PR title, e.g., '[AMORO-XXXX] Your PR title ...'.
+  1. If this is your first time, please read our contributor guidelines: https://amoro.apache.org/how-to-contribute/
+  2. If the PR is related to an issue in https://github.com/apache/amoro/issues, add '[AMORO-XXXX]' in your PR title, e.g., '[AMORO-XXXX] Your PR title ...'.
   3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][AMORO-XXXX] Your PR title ...'.
 -->
 

.github/labeler.yml

@@ -35,7 +35,7 @@
   - ".rat-excludes"
   - ".scalafmt"
   - "**/*pom.xml"
-  - "ams/dist/**/*"
+  - "amoro-ams/dist/**/*"
   - "docker/**/*"
   - "charts/**/"
 
@@ -44,26 +44,26 @@
   - "docs/**/*"
 
 "module:core":
-  - "core/**/*"
+  - "amoro-core/**/*"
 
 "module:ams-server":
-  - "ams/api/**/*"
-  - "ams/ams-server/**/*"
+  - "amoro-ams/api/**/*"
+  - "amoro-ams/server/**/*"
 
 "module:ams-dashboard":
-  - "ams/dashboard/**/*"
+  - "amoro-ams/amoro-ams-dashboard/**/*"
 
 "module:ams-optimizer":
-  - "ams/optimizer/**/*"
+  - "amoro-ams/amoro-ams-optimizer/**/*"
 
 "module:mixed-spark":
-  - "mixed/spark/**/*"
+  - "amoro-mixed-format/amoro-mixed-format-spark/**/*"
 
 "module:mixed-flink":
-  - "mixed/flink/**/*"
+  - "amoro-mixed-format/amoro-mixed-format-flink/**/*"
 
 "module:mixed-hive":
-  - "mixed/hive/**/*"
+  - "amoro-mixed-format/amoro-mixed-format-hive/**/*"
 
 "module:mixed-trino":
-  - "mixed/trino/**/*"
+  - "amoro-mixed-format/amoro-mixed-format-trino/**/*"

.github/workflows/core-hadoop2-ci.yml

@@ -18,12 +18,13 @@ on:
       - "master"
       - "0.**"
     paths:
-      - "ams/**"
-      - "core/**"
-      - "mixed/flink/**"
-      - "mixed/hive/**"
-      - "mixed/spark/**"
+      - "amoro-ams/**"
+      - "amoro-core/**"
+      - "amoro-mixed-format/amoro-mixed-format-flink/**"
+      - "amoro-mixed-format/amoro-mixed-format-hive/**"
+      - "amoro-mixed-format/amoro-mixed-format-spark/**"
       - "pom.xml"
+      - "tools/maven/**"
 
 jobs:
   build:
@@ -42,7 +43,7 @@ jobs:
         run: mvn validate
 
       - name: Build all module with Maven
-        run: mvn clean install -pl '!mixed/trino' -Djacoco.flink.skip=true -B -ntp -Dhadoop=v2
+        run: mvn clean install -pl '!amoro-mixed-format/amoro-mixed-format-trino' -Djacoco.flink.skip=true -B -ntp -Dhadoop=v2
 
       - name: Code coverage
         uses: codecov/codecov-action@v3

.github/workflows/core-hadoop3-ci.yml

@@ -18,12 +18,13 @@ on:
       - "master"
       - "0.**"
     paths:
-      - "ams/**"
-      - "core/**"
-      - "mixed/flink/**"
-      - "mixed/hive/**"
-      - "mixed/spark/**"
+      - "amoro-ams/**"
+      - "amoro-core/**"
+      - "amoro-mixed-format/amoro-mixed-format-flink/**"
+      - "amoro-mixed-format/amoro-mixed-format-hive/**"
+      - "amoro-mixed-format/amoro-mixed-format-spark/**"
       - "pom.xml"
+      - "tools/maven/**"
 
 jobs:
   build:
@@ -42,7 +43,7 @@ jobs:
         run: mvn validate
 
       - name: Build all module with Maven
-        run: mvn clean install -pl '!mixed/trino' -Djacoco.flink.skip=true -B -ntp
+        run: mvn clean install -pl '!amoro-mixed-format/amoro-mixed-format-trino' -Djacoco.flink.skip=true -B -ntp
 
       - name: Code coverage
         uses: codecov/codecov-action@v3

.github/workflows/docker-images.yml

@@ -37,7 +37,7 @@ jobs:
   docker-amoro:
     name: Push Amoro Docker Image to Docker Hub
     runs-on: ubuntu-latest
-    if: ${{ startsWith(github.repository, 'NetEase/') }}
+    if: ${{ startsWith(github.repository, 'apache/') }}
     strategy:
       matrix:
         hadoop: [ "v3" ]
@@ -62,7 +62,7 @@ jobs:
           flavor: |
             latest=auto
           images: |
-            name=arctic163/amoro
+            name=apache/amoro
           tags: |
             type=ref,event=branch,enable=${{ matrix.hadoop == 'v3' }},suffix=-snapshot
             type=ref,event=branch,enable=${{ matrix.hadoop == 'v3' }},suffix=-snapshot-hadoop3
@@ -85,7 +85,7 @@ jobs:
           echo "MVN_HADOOP=-Dhadoop=${{ matrix.hadoop }}" >> $GITHUB_ENV
 
       - name: Build dist module with Maven
-        run: mvn clean install -pl 'ams/dist' -am -e ${MVN_HADOOP} -DskipTests -B -ntp
+        run: mvn clean install -pl 'amoro-ams/dist' -am -e ${MVN_HADOOP} -DskipTests -B -ntp
 
       - name: Build and Push Amoro Docker Image
         uses: docker/build-push-action@v4
@@ -99,10 +99,10 @@ jobs:
   docker-optimizer-flink:
     name: Push Amoro Optimizer-Flink Docker Image to Docker Hub
     runs-on: ubuntu-latest
-    if: ${{ startsWith(github.repository, 'NetEase/') }}
+    if: ${{ startsWith(github.repository, 'apache/') }}
     strategy:
       matrix:
-        flink: [ "1.14.6", "1.15.3" ]
+        flink: [ "1.14.6", "1.18.1" ]
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK 8
@@ -124,14 +124,14 @@ jobs:
           flavor: |
             latest=auto
           images: |
-            name=arctic163/optimizer-flink
+            name=apache/amoro-flink-optimizer
           tags: |
             type=ref,event=branch,enable=${{ matrix.flink == '1.14.6' }},suffix=-snapshot
             type=ref,event=branch,enable=${{ matrix.flink == '1.14.6' }},suffix=-snapshot-flink1.14
-            type=ref,event=branch,enable=${{ matrix.flink == '1.15.3' }},suffix=-snapshot-flink1.15
+            type=ref,event=branch,enable=${{ matrix.flink == '1.18.1' }},suffix=-snapshot-flink1.18
             type=semver,enable=${{ matrix.flink == '1.14.6' }},pattern={{version}}
             type=semver,enable=${{ matrix.flink == '1.14.6' }},pattern={{version}}-flink1.14
-            type=semver,enable=${{ matrix.flink == '1.15.3' }},pattern={{version}}-flink1.15
+            type=semver,enable=${{ matrix.flink == '1.18.1' }},pattern={{version}}-flink1.18
 
       - name: Print tags
         run: echo '${{ steps.meta.outputs.tags }}'
@@ -145,7 +145,11 @@ jobs:
       - name: Set optimizer flink version
         run: |
           OPTIMIZER_FLINK=${{ matrix.flink }} && \
-          echo "OPTIMIZER_FLINK=-Doptimizer.flink${OPTIMIZER_FLINK%.*}" >> $GITHUB_ENV
+          echo "OPTIMIZER_FLINK=-Dflink-optimizer.flink-version${OPTIMIZER_FLINK}" >> $GITHUB_ENV
+          if [[ "$OPTIMIZER_FLINK" < "1.15" ]]; then
+            echo "Adding -Pflink-pre-1.15 for Flink version less than 1.15"
+            echo "OPTIMIZER_FLINK=-Pflink-pre-1.15 -Dflink-optimizer.flink-version=${OPTIMIZER_FLINK}" >> $GITHUB_ENV
+          fi
 
       - name: Set ENV Amoro version
         id: version
@@ -156,7 +160,7 @@ jobs:
           && echo "AMORO_VERSION=${AMORO_VERSION}" >> $GITHUB_OUTPUT
 
       - name: Build optimizer module with Maven
-        run: mvn clean package -pl 'ams/optimizer/flink-optimizer' -am -e ${OPTIMIZER_FLINK} -DskipTests -B -ntp
+        run: mvn clean package -pl 'amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink' -am -e ${OPTIMIZER_FLINK} -DskipTests -B -ntp
 
       - name: Build and Push Flink Optimizer Docker Image
         uses: docker/build-push-action@v4
@@ -170,13 +174,15 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           build-args: |
             FLINK_VERSION=${{ matrix.flink }}
-            OPTIMIZER_JOB=ams/optimizer/flink-optimizer/target/flink-optimizer-${{ env.AMORO_VERSION }}-jar-with-dependencies.jar
+            OPTIMIZER_JOB=amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/target/amoro-optimizer-flink-${{ env.AMORO_VERSION }}-jar-with-dependencies.jar
 
-  docker-quickdemo:
-    name: Push Amoro Quick-Demo Docker Image to Docker Hub
+  docker-optimizer-spark:
+    name: Push Amoro Optimizer-Spark Docker Image to Docker Hub
     runs-on: ubuntu-latest
-    needs: docker-amoro
-    if: ${{ startsWith(github.repository, 'NetEase/') && startsWith(github.ref, 'refs/tags/' )}}
+    if: ${{ startsWith(github.repository, 'apache/') }}
+    strategy:
+      matrix:
+        spark: [ "3.3.3" ]
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK 8
@@ -198,37 +204,49 @@ jobs:
           flavor: |
             latest=auto
           images: |
-            name=arctic163/quickdemo
+            name=apache/amoro-spark-optimizer
           tags: |
-            type=ref,event=branch,suffix=-snapshot
-            type=semver,pattern={{version}}
+            type=ref,event=branch,enable=${{ matrix.spark == '3.3.3' }},suffix=-snapshot
+            type=ref,event=branch,enable=${{ matrix.spark == '3.3.3' }},suffix=-snapshot-spark3.3
+            type=semver,enable=${{ matrix.spark == '3.3.3' }},pattern={{version}}
 
       - name: Print tags
         run: echo '${{ steps.meta.outputs.tags }}'
 
-      - name: Set Amoro Tag
-        id: tag
-        run: |
-          AMORO_TAG=${{ steps.meta.outputs.tags }} && echo "AMORO_TAG=${AMORO_TAG#*:}" >> $GITHUB_OUTPUT
-
       - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Set optimizer spark version
+        run: |
+          OPTIMIZER_SPARK=${{ matrix.spark }} && \
+          echo "OPTIMIZER_SPARK=-Dspark-optimizer.spark-version${OPTIMIZER_SPARK}" >> $GITHUB_ENV
+
+      - name: Set ENV Amoro version
+        id: version
+        run: |
+          AMORO_VERSION=`cat pom.xml | grep 'amoro-parent' -C 3 | grep -Eo '<version>.*</version>' | awk -F'[><]' '{print $3}'` \
+          && echo "$AMORO_VERSION" \
+          && echo "AMORO_VERSION=${AMORO_VERSION}" >> $GITHUB_ENV \
+          && echo "AMORO_VERSION=${AMORO_VERSION}" >> $GITHUB_OUTPUT
+
       - name: Build optimizer module with Maven
-        run: mvn clean package -pl 'mixed/flink/v1.17/flink-runtime' -am -e -DskipTests -B -ntp
+        run: mvn clean package -pl 'amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark' -am -e ${OPTIMIZER_SPARK} -DskipTests -B -ntp
 
-      - name: Build and Push Flink Optimizer Docker Image
+      - name: Build and Push Spark Optimizer Docker Image
         uses: docker/build-push-action@v4
+        env:
+          AMORO_VERSION: ${{ steps.version.outputs.AMORO_VERSION }}
         with:
           context: .
           push: true
-          file: docker/quickdemo/Dockerfile
+          file: docker/optimizer-spark/Dockerfile
           platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
           build-args: |
-            AMORO_TAG=${{ steps.tag.outputs.AMORO_TAG }}
-            FLINK_VERSION=1.17.1
-            ICEBERG_VERSION=1.3.1
+            SPARK_VERSION=${{ matrix.spark }}
+            OPTIMIZER_JOB=amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark/target/amoro-optimizer-spark-${{ env.AMORO_VERSION }}-jar-with-dependencies.jar
+
+

.github/workflows/helm-charts-ci.yml

@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+# This workflow will execute Tests when submit a PR.
+
+name: "CI for Helm Charts"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    paths:
+      - 'charts/**'
+  push:
+    paths:
+      - 'charts/**'
+  merge_group:
+  workflow_dispatch:
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+
+      - name: Install helm-unittests
+        run: |
+          helm plugin install https://github.com/helm-unittest/helm-unittest.git
+
+      - name: Rebuild Chart with Dependency
+        run: |
+          cd charts/amoro
+          helm dependency build
+
+      - name: Run Unit Tests
+        run: |
+          helm unittest charts/amoro

.github/workflows/trino-ci.yml

@@ -18,11 +18,12 @@ on:
       - "master"
       - "0.**"
     paths:
-      - "ams/**"
-      - "core/**"
-      - "mixed/hive/**"
-      - "mixed/trino/**"
+      - "amoro-ams/**"
+      - "amoro-core/**"
+      - "amoro-mixed-format/amoro-mixed-format-hive/**"
+      - "amoro-mixed-format/amoro-mixed-format-trino/**"
       - "pom.xml"
+      - "tools/maven/**"
 
 jobs:
   build:
@@ -40,10 +41,10 @@ jobs:
         run: mvn validate -P trino-spotless
 
       - name: Install dependency with Maven
-        run: mvn clean install -DskipTests -pl 'ams/api,core,mixed/hive' -Dhadoop=v2 -am -B -P trino-spotless
+        run: mvn clean install -DskipTests -pl 'amoro-ams/amoro-ams-api,amoro-core,amoro-mixed-format/amoro-mixed-format-hive' -Dhadoop=v2 -am -B -P trino-spotless
 
       - name: Build with Maven
-        run: mvn clean test -pl 'mixed/trino' -B -ntp -Dhadoop=v2 -P trino-spotless
+        run: mvn clean test -pl 'amoro-mixed-format/amoro-mixed-format-trino' -B -ntp -Dhadoop=v2 -P trino-spotless
 
       - name: Code coverage
         uses: codecov/codecov-action@v3