remove integration-test module

apache · Aug 22, 2024 · 5635159 · 5635159
1 parent 92b8348
commit 5635159
Show file tree

Hide file tree

Showing 387 changed files with 462 additions and 242 deletions.
diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
@@ -171,6 +171,7 @@ jobs:
           -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test 
           -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
           -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
+          -x :web:integration-test:test -x :trino-connector:integration-test:test -x :trino-connector:test
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml
@@ -89,8 +89,8 @@ jobs:
       - name: Frontend Integration Test
         id: integrationTest
         run: |
-          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**"
-          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**"
+          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :web:integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**"
+          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :web:integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**"
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3
@@ -99,9 +99,10 @@ jobs:
           name: integrate-test-reports-${{ matrix.java-version }}
           path: |
             build/reports
-            integration-test/build/integration-test-integration-test.log
-            integration-test/build/*.tar
+            integration-test-common/build/integration-test-integration-test.log
+            integration-test-common/build/*.tar
             distribution/package/logs/gravitino-server.out
             distribution/package/logs/gravitino-server.log
             catalogs/**/*.log
-            catalogs/**/*.tar
+            catalogs/**/*.tar
+            web/integration-test/build/*.log
diff --git a/.github/workflows/trino-integration-test.yml b/.github/workflows/trino-integration-test.yml
@@ -0,0 +1,104 @@
+name: Trino Integration Test
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the "main" branch
+  push:
+    branches: [ "main", "branch-*" ]
+  pull_request:
+    branches: [ "main", "branch-*" ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            source_changes:
+              - .github/**
+              - api/**
+              - bin/**
+              - catalogs/**
+              - clients/client-java/**
+              - clients/client-java-runtime/**
+              - clients/filesystem-hadoop3/**
+              - clients/filesystem-hadoop3-runtime/**
+              - common/**
+              - conf/**
+              - core/**
+              - dev/**
+              - gradle/**
+              - meta/**
+              - server/**
+              - server-common/**
+              - trino-connector/**
+              - docs/open-api/**
+              - build.gradle.kts
+              - gradle.properties
+              - gradlew
+              - setting.gradle.kts
+    outputs:
+      source_changes: ${{ steps.filter.outputs.source_changes }}
+
+  # Integration test for AMD64 architecture
+  test-amd64-arch:
+    needs: changes
+    if: needs.changes.outputs.source_changes == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        architecture: [linux/amd64]
+        java-version: [ 17 ]
+    env:
+      PLATFORM: ${{ matrix.architecture }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Check required command
+        run: |
+          dev/ci/check_commands.sh
+
+      - name: Package Gravitino
+        run: |
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
+
+      - name: Free up disk space
+        run: |
+          dev/ci/util_free_space.sh
+
+      - name: Trino Integration Test
+        id: integrationTest
+        run: |
+          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :trino-connector:integration-test:test
+          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :trino-connector:integration-test:test
+
+      - name: Upload integrate tests reports
+        uses: actions/upload-artifact@v3
+        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
+        with:
+          name: trino-connector-integrate-test-reports-${{ matrix.java-version }}
+          path: |
+            build/reports
+            trino-connector/integrate-test/build/*.log
+            trino-connector/integrate-test/build/*.tar
+            distribution/package/logs/gravitino-server.out
+            distribution/package/logs/gravitino-server.log
+            catalogs/**/*.log
+            catalogs/**/*.tar
diff --git a/clients/client-java/build.gradle.kts b/clients/client-java/build.gradle.kts
@@ -42,12 +42,22 @@ dependencies {
   testCompileOnly(libs.lombok)
   testAnnotationProcessor(libs.lombok)
 
+  testImplementation(project(":core"))
+  testImplementation(project(":integration-test-common", "testArtifacts"))
+  testImplementation(project(":server"))
+  testImplementation(project(":server-common"))
+
+  testImplementation(libs.bundles.jersey)
+  testImplementation(libs.bundles.jwt)
+  testImplementation(libs.commons.lang3)
+  testImplementation(libs.hadoop3.client)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
+  testImplementation(libs.minikdc)
   testImplementation(libs.mockito.core)
   testImplementation(libs.mockserver.netty)
   testImplementation(libs.mockserver.client.java)
-  testImplementation(libs.bundles.jwt)
+  testImplementation(libs.testcontainers)
 
   testRuntimeOnly(libs.junit.jupiter.engine)
 }
@@ -56,6 +66,16 @@ tasks.build {
   dependsOn("javadoc")
 }
 
+tasks.test {
+  val skipITs = project.hasProperty("skipITs")
+  if (skipITs) {
+    exclude("**/integration/test/**")
+  } else {
+    dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars")
+    dependsOn(":catalogs:catalog-kafka:jar", ":catalogs:catalog-kafka:runtimeJars")
+  }
+}
+
 tasks.javadoc {
   dependsOn(":api:javadoc", ":common:javadoc")
   source =
@@ -67,3 +87,8 @@ tasks.javadoc {
     project(":api").configurations["runtimeClasspath"] +
     project(":common").configurations["runtimeClasspath"]
 }
+
+tasks.clean {
+  delete("target")
+  delete("tmp")
+}
diff --git a/...tino/integration/test/client/AuditIT.java → ...tino/client/integration/test/AuditIT.java b/...tino/integration/test/client/AuditIT.java → ...tino/client/integration/test/AuditIT.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.gravitino.integration.test.client;
+package org.apache.gravitino.client.integration.test;
 
 import com.google.common.collect.Maps;
 import java.util.Collections;

diff --git a/...no/integration/test/client/CatalogIT.java → ...no/client/integration/test/CatalogIT.java b/...no/integration/test/client/CatalogIT.java → ...no/client/integration/test/CatalogIT.java
@@ -17,14 +17,14 @@
  * under the License.
  */
 
-package org.apache.gravitino.integration.test.client;
+package org.apache.gravitino.client.integration.test;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import java.io.File;
 import java.util.Collections;
 import java.util.Map;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.gravitino.Catalog;
 import org.apache.gravitino.client.GravitinoMetalake;
 import org.apache.gravitino.integration.test.container.ContainerSuite;

diff --git a/...o/integration/test/client/MetalakeIT.java → ...o/client/integration/test/MetalakeIT.java b/...o/integration/test/client/MetalakeIT.java → ...o/client/integration/test/MetalakeIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.integration.test.client;
+package org.apache.gravitino.client.integration.test;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;

diff --git a/...che/gravitino/integration/test/TagIT.java → ...vitino/client/integration/test/TagIT.java b/...che/gravitino/integration/test/TagIT.java → ...vitino/client/integration/test/TagIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.integration.test;
+package org.apache.gravitino.client.integration.test;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Sets;

diff --git a/...on/test/web/rest/VersionOperationsIT.java → ...integration/test/VersionOperationsIT.java b/...on/test/web/rest/VersionOperationsIT.java → ...integration/test/VersionOperationsIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.integration.test.web.rest;
+package org.apache.gravitino.client.integration.test;
 
 import org.apache.gravitino.client.GravitinoVersion;
 import org.apache.gravitino.integration.test.util.AbstractIT;

diff --git a/...n/test/authorization/AccessControlIT.java → ...n/test/authorization/AccessControlIT.java b/...n/test/authorization/AccessControlIT.java → ...n/test/authorization/AccessControlIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.integration.test.authorization;
+package org.apache.gravitino.client.integration.test.authorization;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;

diff --git a/...n/test/web/rest/KerberosOperationsIT.java → ...t/authorization/KerberosOperationsIT.java b/...n/test/web/rest/KerberosOperationsIT.java → ...t/authorization/KerberosOperationsIT.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.gravitino.integration.test.web.rest;
+package org.apache.gravitino.client.integration.test.authorization;
 
 import static org.apache.gravitino.server.authentication.KerberosConfig.KEYTAB;
 import static org.apache.gravitino.server.authentication.KerberosConfig.PRINCIPAL;

diff --git a/.../test/web/rest/MultiAuthOperationsIT.java → .../authorization/MultiAuthOperationsIT.java b/.../test/web/rest/MultiAuthOperationsIT.java → .../authorization/MultiAuthOperationsIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.integration.test.web.rest;
+package org.apache.gravitino.client.integration.test.authorization;
 
 import static org.apache.gravitino.server.GravitinoServer.WEBSERVER_CONF_PREFIX;
 import static org.apache.gravitino.server.authentication.KerberosConfig.KEYTAB;

diff --git a/...ion/test/web/rest/OAuth2OperationsIT.java → ...est/authorization/OAuth2OperationsIT.java b/...ion/test/web/rest/OAuth2OperationsIT.java → ...est/authorization/OAuth2OperationsIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.integration.test.web.rest;
+package org.apache.gravitino.client.integration.test.authorization;
 
 import com.google.common.collect.Maps;
 import io.jsonwebtoken.Jwts;

diff --git a/...tegration/test/authorization/OwnerIT.java → ...tegration/test/authorization/OwnerIT.java b/...tegration/test/authorization/OwnerIT.java → ...tegration/test/authorization/OwnerIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.integration.test.authorization;
+package org.apache.gravitino.client.integration.test.authorization;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;

diff --git a/clients/client-java/src/test/java/resources/log4j2.properties b/clients/client-java/src/test/java/resources/log4j2.properties
diff --git a/clients/filesystem-hadoop3-runtime/build.gradle.kts b/clients/filesystem-hadoop3-runtime/build.gradle.kts
@@ -27,6 +27,7 @@ plugins {
 
 dependencies {
   implementation(project(":clients:filesystem-hadoop3"))
+  implementation(project(":clients:client-java-runtime", configuration = "shadow"))
 }
 
 tasks.withType<ShadowJar>(ShadowJar::class.java) {

diff --git a/clients/filesystem-hadoop3/build.gradle.kts b/clients/filesystem-hadoop3/build.gradle.kts
@@ -24,15 +24,30 @@ plugins {
 }
 
 dependencies {
-  compileOnly(libs.hadoop3.common)
-  implementation(project(":clients:client-java-runtime", configuration = "shadow"))
+  compileOnly(project(":clients:client-java-runtime", configuration = "shadow"))
+  implementation(libs.hadoop3.common)
   implementation(libs.caffeine)
 
+  testImplementation(project(":api"))
   testImplementation(project(":core"))
+  testImplementation(project(":common"))
+  testImplementation(project(":server"))
   testImplementation(project(":server-common"))
+  testImplementation(project(":clients:client-java"))
+  testImplementation(project(":integration-test-common", "testArtifacts"))
   testImplementation(libs.awaitility)
+  testImplementation(libs.bundles.jetty)
+  testImplementation(libs.bundles.jersey)
   testImplementation(libs.bundles.jwt)
-  testImplementation(libs.hadoop3.common)
+  testImplementation(libs.testcontainers)
+  testImplementation(libs.guava)
+  testImplementation(libs.hadoop3.common) {
+    exclude("com.sun.jersey", "jersey-core")
+    exclude("com.sun.jersey", "jersey-servlet")
+    exclude("com.sun.jersey", "jersey-json")
+    exclude("com.sun.jersey", "jersey-server")
+  }
+  testImplementation(libs.httpclient5)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
   testImplementation(libs.minikdc)
@@ -47,8 +62,21 @@ tasks.build {
   dependsOn("javadoc")
 }
 
+tasks.test {
+  val skipITs = project.hasProperty("skipITs")
+  if (skipITs) {
+    exclude("**/integration/test/**")
+  } else {
+    dependsOn(":catalogs:catalog-hadoop:jar", ":catalogs:catalog-hadoop:runtimeJars")
+  }
+}
+
 tasks.javadoc {
   dependsOn(":clients:client-java-runtime:javadoc")
   source = sourceSets["main"].allJava +
     project(":clients:client-java-runtime").sourceSets["main"].allJava
 }
+
+tasks.clean {
+  delete("target")
+}