[#4633] refactor(integration-test): move the tests in integration-tes…

…t module into their corresponding modules (#4628) ### What changes were proposed in this pull request? - move tests from module integration-test to other corresponding module - add CI for TrinoIT ### Why are the changes needed? Fix: #4633 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? CI passed
apache · Sep 9, 2024 · 6c0ecf5 · 6c0ecf5
1 parent 60f4beb
commit 6c0ecf5
Show file tree

Hide file tree

Showing 1,170 changed files with 649 additions and 388 deletions.
diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
@@ -105,7 +105,7 @@ jobs:
           -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test 
           -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
           -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
-          -x :authorizations:authorization-ranger:test
+          -x :authorizations:authorization-ranger:test -x :web:integration-test:test -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3
@@ -178,7 +178,7 @@ jobs:
           -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test 
           -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
           -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
-          -x :authorizations:authorization-ranger:test
+          -x :authorizations:authorization-ranger:test -x :web:integration-test:test -x :trino-connector:integration-test:test -x :trino-connector:test
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3
@@ -188,9 +188,6 @@ jobs:
           path: |
             build/reports
             iceberg/iceberg-rest-server/build/*.log
-            integration-test/build/*.log
-            integration-test/build/*.tar
-            integration-test/build/trino-ci-container-log
             distribution/package/logs/*.out
             distribution/package/logs/*.log
             catalogs/**/*.log

diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml
@@ -89,8 +89,8 @@ jobs:
       - name: Frontend Integration Test
         id: integrationTest
         run: |
-          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**"
-          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**"
+          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :web:integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**"
+          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :web:integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**"
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3
@@ -99,9 +99,9 @@ jobs:
           name: integrate-test-reports-${{ matrix.java-version }}
           path: |
             build/reports
-            integration-test/build/integration-test-integration-test.log
-            integration-test/build/*.tar
+            integration-test-common/build/*.log
             distribution/package/logs/gravitino-server.out
             distribution/package/logs/gravitino-server.log
             catalogs/**/*.log
-            catalogs/**/*.tar
+            catalogs/**/*.tar
+            web/integration-test/build/*.log
diff --git a/.github/workflows/trino-integration-test.yml b/.github/workflows/trino-integration-test.yml
@@ -0,0 +1,101 @@
+name: Trino Integration Test
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the "main" branch
+  push:
+    branches: [ "main", "branch-*" ]
+  pull_request:
+    branches: [ "main", "branch-*" ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            source_changes:
+              - .github/**
+              - api/**
+              - bin/**
+              - catalogs/**
+              - clients/client-java/**
+              - clients/client-java-runtime/**
+              - common/**
+              - conf/**
+              - core/**
+              - dev/**
+              - gradle/**
+              - meta/**
+              - server/**
+              - server-common/**
+              - trino-connector/**
+              - build.gradle.kts
+              - gradle.properties
+              - gradlew
+              - setting.gradle.kts
+    outputs:
+      source_changes: ${{ steps.filter.outputs.source_changes }}
+
+  # Integration test for AMD64 architecture
+  test-amd64-arch:
+    needs: changes
+    if: needs.changes.outputs.source_changes == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        architecture: [linux/amd64]
+        java-version: [ 17 ]
+    env:
+      PLATFORM: ${{ matrix.architecture }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Check required command
+        run: |
+          dev/ci/check_commands.sh
+
+      - name: Package Gravitino
+        run: |
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
+
+      - name: Free up disk space
+        run: |
+          dev/ci/util_free_space.sh
+
+      - name: Trino Integration Test
+        id: integrationTest
+        run: |
+          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :trino-connector:integration-test:test
+          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :trino-connector:integration-test:test
+
+      - name: Upload integrate tests reports
+        uses: actions/upload-artifact@v3
+        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
+        with:
+          name: trino-connector-integrate-test-reports-${{ matrix.java-version }}
+          path: |
+            build/reports
+            trino-connector/integrate-test/build/*.log
+            integration-test-common/build/trino-ci-container-log
+            distribution/package/logs/gravitino-server.out
+            distribution/package/logs/gravitino-server.log
+            catalogs/**/*.log
+            catalogs/**/*.tar
diff --git a/LICENSE b/LICENSE
@@ -260,7 +260,7 @@
    Apache Zeppelin
    ./integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CommandExecutor.java
    ./integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ProcessData.java
-   ./web/WEB-INF/web.xml
+   ./web/web/WEB-INF/web.xml
 
    Apache Hadoop
    ./dev/docker/hive/hadoop-env.sh

diff --git a/NOTICE b/NOTICE
@@ -7,7 +7,7 @@ The Apache Software Foundation (http://www.apache.org/).
 The initial code for the Gravitino project was donated
 to the ASF by Datastrato (https://datastrato.ai/) copyright 2023-2024.
 
-The Web UI also has a NOTICE file please see web/NOTICE
+The Web UI also has a NOTICE file please see web/web/NOTICE
 for it's contents.
 
 Apache Iceberg

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -198,7 +198,7 @@ allprojects {
       } else if (testMode == "embedded") {
         param.environment("GRAVITINO_HOME", project.rootDir.path)
         param.environment("GRAVITINO_TEST", "true")
-        param.environment("GRAVITINO_WAR", project.rootDir.path + "/web/dist/")
+        param.environment("GRAVITINO_WAR", project.rootDir.path + "/web/web/dist/")
         param.systemProperty("testMode", "embedded")
       } else {
         throw GradleException("Gravitino integration tests only support [-PtestMode=embedded] or [-PtestMode=deploy] mode!")
@@ -487,17 +487,17 @@ tasks.rat {
     "integration-test/**/*.txt",
     "docs/**/*.md",
     "spark-connector/spark-common/src/test/resources/**",
-    "web/.**",
-    "web/next-env.d.ts",
-    "web/dist/**/*",
-    "web/node_modules/**/*",
-    "web/src/lib/utils/axios/**/*",
-    "web/src/lib/enums/httpEnum.js",
-    "web/src/types/axios.d.ts",
-    "web/yarn.lock",
-    "web/package-lock.json",
-    "web/pnpm-lock.yaml",
-    "web/src/lib/icons/svg/**/*.svg",
+    "web/web/.**",
+    "web/web/next-env.d.ts",
+    "web/web/dist/**/*",
+    "web/web/node_modules/**/*",
+    "web/web/src/lib/utils/axios/**/*",
+    "web/web/src/lib/enums/httpEnum.js",
+    "web/web/src/types/axios.d.ts",
+    "web/web/yarn.lock",
+    "web/web/package-lock.json",
+    "web/web/pnpm-lock.yaml",
+    "web/web/src/lib/icons/svg/**/*.svg",
     "**/LICENSE.*",
     "**/NOTICE.*",
     "DISCLAIMER_WIP.txt",
@@ -520,8 +520,6 @@ tasks.rat {
     exclusions.addAll(gitIgnoreExcludes)
   }
 
-  dependsOn(":web:nodeSetup")
-
   verbose.set(true)
   failOnError.set(true)
   setExcludes(exclusions)
@@ -543,15 +541,15 @@ tasks {
   val outputDir = projectDir.dir("distribution")
 
   val compileDistribution by registering {
-    dependsOn("copySubprojectDependencies", "copyCatalogLibAndConfigs", "copyAuthorizationLibAndConfigs", "copySubprojectLib", "iceberg:iceberg-rest-server:copyLibAndConfigs")
+    dependsOn(":web:web:build", "copySubprojectDependencies", "copyCatalogLibAndConfigs", "copyAuthorizationLibAndConfigs", "copySubprojectLib", "iceberg:iceberg-rest-server:copyLibAndConfigs")
 
     group = "gravitino distribution"
     outputs.dir(projectDir.dir("distribution/package"))
     doLast {
       copy {
         from(projectDir.dir("conf")) { into("package/conf") }
         from(projectDir.dir("bin")) { into("package/bin") }
-        from(projectDir.dir("web/build/libs/${rootProject.name}-web-$version.war")) { into("package/web") }
+        from(projectDir.dir("web/web/build/libs/${rootProject.name}-web-$version.war")) { into("package/web") }
         from(projectDir.dir("scripts")) { into("package/scripts") }
         into(outputDir)
         rename { fileName ->
@@ -627,7 +625,7 @@ tasks {
   }
 
   val compileTrinoConnector by registering {
-    dependsOn("trino-connector:copyLibs")
+    dependsOn("trino-connector:trino-connector:copyLibs")
     group = "gravitino distribution"
     outputs.dir(projectDir.dir("distribution/${rootProject.name}-trino-connector"))
     doLast {
@@ -754,7 +752,7 @@ tasks {
         !it.name.startsWith("iceberg") &&
         !it.name.startsWith("integration-test") &&
         !it.name.startsWith("flink") &&
-        it.name != "trino-connector" &&
+        !it.name.startsWith("trino-connector") &&
         it.name != "bundled-catalog"
       ) {
         dependsOn("${it.name}:build")

diff --git a/clients/client-java/build.gradle.kts b/clients/client-java/build.gradle.kts
@@ -42,12 +42,24 @@ dependencies {
   testCompileOnly(libs.lombok)
   testAnnotationProcessor(libs.lombok)
 
+  testImplementation(project(":core"))
+  testImplementation(project(":integration-test-common", "testArtifacts"))
+  testImplementation(project(":server"))
+  testImplementation(project(":server-common"))
+
+  testImplementation(libs.bundles.jersey)
+  testImplementation(libs.bundles.jwt)
+  testImplementation(libs.commons.lang3)
+  testImplementation(libs.hadoop3.client)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
+  testImplementation(libs.minikdc)
   testImplementation(libs.mockito.core)
   testImplementation(libs.mockserver.netty)
   testImplementation(libs.mockserver.client.java)
-  testImplementation(libs.bundles.jwt)
+  testImplementation(libs.mysql.driver)
+  testImplementation(libs.postgresql.driver)
+  testImplementation(libs.testcontainers)
 
   testRuntimeOnly(libs.junit.jupiter.engine)
 }
@@ -56,6 +68,16 @@ tasks.build {
   dependsOn("javadoc")
 }
 
+tasks.test {
+  val skipITs = project.hasProperty("skipITs")
+  if (skipITs) {
+    exclude("**/integration/test/**")
+  } else {
+    dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars")
+    dependsOn(":catalogs:catalog-kafka:jar", ":catalogs:catalog-kafka:runtimeJars")
+  }
+}
+
 tasks.javadoc {
   dependsOn(":api:javadoc", ":common:javadoc")
   source =
@@ -67,3 +89,8 @@ tasks.javadoc {
     project(":api").configurations["runtimeClasspath"] +
     project(":common").configurations["runtimeClasspath"]
 }
+
+tasks.clean {
+  delete("target")
+  delete("tmp")
+}
diff --git a/...tino/integration/test/client/AuditIT.java → ...tino/client/integration/test/AuditIT.java b/...tino/integration/test/client/AuditIT.java → ...tino/client/integration/test/AuditIT.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.gravitino.integration.test.client;
+package org.apache.gravitino.client.integration.test;
 
 import com.google.common.collect.Maps;
 import java.util.Collections;

diff --git a/...no/integration/test/client/CatalogIT.java → ...no/client/integration/test/CatalogIT.java b/...no/integration/test/client/CatalogIT.java → ...no/client/integration/test/CatalogIT.java
@@ -17,14 +17,14 @@
  * under the License.
  */
 
-package org.apache.gravitino.integration.test.client;
+package org.apache.gravitino.client.integration.test;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import java.io.File;
 import java.util.Collections;
 import java.util.Map;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.gravitino.Catalog;
 import org.apache.gravitino.CatalogChange;
 import org.apache.gravitino.client.GravitinoMetalake;

diff --git a/...o/integration/test/client/MetalakeIT.java → ...o/client/integration/test/MetalakeIT.java b/...o/integration/test/client/MetalakeIT.java → ...o/client/integration/test/MetalakeIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.integration.test.client;
+package org.apache.gravitino.client.integration.test;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;

diff --git a/...che/gravitino/integration/test/TagIT.java → ...vitino/client/integration/test/TagIT.java b/...che/gravitino/integration/test/TagIT.java → ...vitino/client/integration/test/TagIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.integration.test;
+package org.apache.gravitino.client.integration.test;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Sets;

diff --git a/...on/test/web/rest/VersionOperationsIT.java → ...integration/test/VersionOperationsIT.java b/...on/test/web/rest/VersionOperationsIT.java → ...integration/test/VersionOperationsIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.integration.test.web.rest;
+package org.apache.gravitino.client.integration.test;
 
 import org.apache.gravitino.client.GravitinoVersion;
 import org.apache.gravitino.integration.test.util.AbstractIT;

diff --git a/...n/test/authorization/AccessControlIT.java → ...n/test/authorization/AccessControlIT.java b/...n/test/authorization/AccessControlIT.java → ...n/test/authorization/AccessControlIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.integration.test.authorization;
+package org.apache.gravitino.client.integration.test.authorization;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;

diff --git a/...uthorization/AccessControlNotAllowIT.java → ...uthorization/AccessControlNotAllowIT.java b/...uthorization/AccessControlNotAllowIT.java → ...uthorization/AccessControlNotAllowIT.java
diff --git a/...n/test/web/rest/KerberosOperationsIT.java → ...t/authorization/KerberosOperationsIT.java b/...n/test/web/rest/KerberosOperationsIT.java → ...t/authorization/KerberosOperationsIT.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.gravitino.integration.test.web.rest;
+package org.apache.gravitino.client.integration.test.authorization;
 
 import static org.apache.gravitino.server.authentication.KerberosConfig.KEYTAB;
 import static org.apache.gravitino.server.authentication.KerberosConfig.PRINCIPAL;

diff --git a/.../test/web/rest/MultiAuthOperationsIT.java → .../authorization/MultiAuthOperationsIT.java b/.../test/web/rest/MultiAuthOperationsIT.java → .../authorization/MultiAuthOperationsIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.integration.test.web.rest;
+package org.apache.gravitino.client.integration.test.authorization;
 
 import static org.apache.gravitino.server.GravitinoServer.WEBSERVER_CONF_PREFIX;
 import static org.apache.gravitino.server.authentication.KerberosConfig.KEYTAB;

diff --git a/...ion/test/web/rest/OAuth2OperationsIT.java → ...est/authorization/OAuth2OperationsIT.java b/...ion/test/web/rest/OAuth2OperationsIT.java → ...est/authorization/OAuth2OperationsIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.integration.test.web.rest;
+package org.apache.gravitino.client.integration.test.authorization;
 
 import com.google.common.collect.Maps;
 import io.jsonwebtoken.Jwts;