Skip to content

Commit

Permalink
Merge branch 'main' into issue-3751
Browse files Browse the repository at this point in the history
  • Loading branch information
unknowntpo authored Jun 28, 2024
2 parents a3947c2 + ef91358 commit 5985999
Show file tree
Hide file tree
Showing 98 changed files with 279 additions and 213 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/backend-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ jobs:
- name: Backend Integration Test
id: integrationTest
run: >
./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -P${{ matrix.backend }} -PskipWebITs
./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -P${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
-x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test
-x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test
-x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ jobs:
dev/ci/util_free_space.sh
- name: Build with Gradle
run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} -x :clients:client-python:build
run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false -x :clients:client-python:build

- name: Upload unit tests report
uses: actions/upload-artifact@v3
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/cron-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ jobs:
- name: Integration Test
id: integrationTest
run: |
./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }}
./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/flink-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,8 @@ jobs:
- name: Flink Integration Test
id: integrationTest
run: |
./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/frontend-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,8 @@ jobs:
- name: Frontend Integration Test
id: integrationTest
run: |
./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/spark-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,10 @@ jobs:
id: integrationTest
run: |
if [ "${{ matrix.scala-version }}" == "2.12" ];then
./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.3:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
fi
./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.4:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.5:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.4:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
Expand Down
26 changes: 13 additions & 13 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,10 @@ import com.github.jk1.license.render.InventoryHtmlReportRenderer
import com.github.jk1.license.render.ReportRenderer
import com.github.vlsi.gradle.dsl.configureEach
import net.ltgt.gradle.errorprone.errorprone
import org.gradle.api.tasks.testing.Test
import org.gradle.api.tasks.testing.logging.TestExceptionFormat
import org.gradle.internal.hash.ChecksumService
import org.gradle.internal.os.OperatingSystem
import org.gradle.kotlin.dsl.support.serviceOf
import java.io.File
import java.io.IOException
import java.util.Locale

Expand Down Expand Up @@ -186,9 +184,9 @@ allprojects {
}

param.useJUnitPlatform {
val DOCKER_IT_TEST = project.rootProject.extra["docker_it_test"] as? Boolean ?: false
if (!DOCKER_IT_TEST) {
excludeTags("gravitino-docker-it")
val dockerTest = project.rootProject.extra["dockerTest"] as? Boolean ?: false
if (!dockerTest) {
excludeTags("gravitino-docker-test")
}
}
}
Expand Down Expand Up @@ -657,7 +655,7 @@ tasks {

apply(plugin = "com.dorongold.task-tree")

project.extra["docker_it_test"] = false
project.extra["dockerTest"] = false
project.extra["dockerRunning"] = false
project.extra["macDockerConnector"] = false
project.extra["isOrbStack"] = false
Expand All @@ -676,13 +674,15 @@ fun printDockerCheckInfo() {
val macDockerConnector = project.extra["macDockerConnector"] as? Boolean ?: false
val isOrbStack = project.extra["isOrbStack"] as? Boolean ?: false

if (OperatingSystem.current().isMacOsX() &&
if (extra["skipDockerTests"].toString().toBoolean()) {
project.extra["dockerTest"] = false
} else if (OperatingSystem.current().isMacOsX() &&
dockerRunning &&
(macDockerConnector || isOrbStack)
) {
project.extra["docker_it_test"] = true
project.extra["dockerTest"] = true
} else if (OperatingSystem.current().isLinux() && dockerRunning) {
project.extra["docker_it_test"] = true
project.extra["dockerTest"] = true
}

println("------------------ Check Docker environment ---------------------")
Expand All @@ -692,11 +692,11 @@ fun printDockerCheckInfo() {
println("OrbStack status ................................................. [${if (dockerRunning && isOrbStack) "yes" else "no"}]")
}

val docker_it_test = project.extra["docker_it_test"] as? Boolean ?: false
if (!docker_it_test) {
println("Run test cases without `gravitino-docker-it` tag ................ [$testMode test]")
val dockerTest = project.extra["dockerTest"] as? Boolean ?: false
if (dockerTest) {
println("Using Docker container to run all tests. [$testMode test]")
} else {
println("Using Gravitino IT Docker container to run all integration tests. [$testMode test]")
println("Run test cases without `gravitino-docker-test` tag ................ [$testMode test]")
}
println("-----------------------------------------------------------------")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class HDFSKerberosIT {
private static final Logger LOG = LoggerFactory.getLogger(HDFSKerberosIT.class);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class HadoopCatalogIT extends AbstractIT {
private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogIT.class);
private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
import org.slf4j.LoggerFactory;
import sun.security.krb5.KrbException;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class HadoopUserAuthenticationIT extends AbstractIT {
private static final Logger LOG = LoggerFactory.getLogger(HadoopUserAuthenticationIT.class);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class HadoopUserImpersonationIT extends AbstractIT {
private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogIT.class);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class CatalogHiveIT extends AbstractIT {
private static final Logger LOG = LoggerFactory.getLogger(CatalogHiveIT.class);
public static final String metalakeName =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class HiveUserAuthenticationIT extends AbstractIT {
private static final Logger LOG = LoggerFactory.getLogger(HiveUserAuthenticationIT.class);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class ProxyCatalogHiveIT extends AbstractIT {

public static final String METALAKE_NAME =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import org.junit.jupiter.api.Tag;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class CatalogDorisDriverIT extends CatalogDorisIT {
public CatalogDorisDriverIT() {
super();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
import org.junit.jupiter.api.TestInstance.Lifecycle;
import org.testcontainers.shaded.org.awaitility.Awaitility;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
@TestInstance(Lifecycle.PER_CLASS)
public class CatalogDorisIT extends AbstractIT {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class TestDorisDatabaseOperations extends TestDoris {

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import org.junit.jupiter.api.Test;
import org.testcontainers.shaded.org.awaitility.Awaitility;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class TestDorisTableOperations extends TestDoris {
private static final Type VARCHAR_255 = Types.VarCharType.of(255);
private static final Type VARCHAR_1024 = Types.VarCharType.of(1024);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class AuditCatalogMysqlIT extends AbstractIT {
private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
public static final String metalakeName = GravitinoITUtils.genRandomName("audit_mysql_metalake");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import org.junit.jupiter.api.Tag;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class CatalogMysqlDriverIT extends CatalogMysqlIT {
public CatalogMysqlDriverIT() {
super();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@
import org.junit.jupiter.api.TestInstance.Lifecycle;
import org.junit.jupiter.api.condition.EnabledIf;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
@TestInstance(Lifecycle.PER_CLASS)
public class CatalogMysqlIT extends AbstractIT {
private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class CatalogMysqlVersion5IT extends CatalogMysqlIT {
public CatalogMysqlVersion5IT() {
super();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class TestMysqlDatabaseOperations extends TestMysql {

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class TestMysqlTableOperations extends TestMysql {
private static Type VARCHAR = Types.VarCharType.of(255);
private static Type INT = Types.IntegerType.get();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.TestInstance.Lifecycle;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
@TestInstance(Lifecycle.PER_CLASS)
public class CatalogPostgreSqlIT extends AbstractIT {
private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import com.datastrato.gravitino.integration.test.container.PGImageName;
import org.junit.jupiter.api.Tag;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class CatalogPostgreSqlVersion12IT extends CatalogPostgreSqlIT {
public CatalogPostgreSqlVersion12IT() {
postgreImageName = PGImageName.VERSION_12;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import com.datastrato.gravitino.integration.test.container.PGImageName;
import org.junit.jupiter.api.Tag;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class CatalogPostgreSqlVersion14IT extends CatalogPostgreSqlIT {
public CatalogPostgreSqlVersion14IT() {
postgreImageName = PGImageName.VERSION_14;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import com.datastrato.gravitino.integration.test.container.PGImageName;
import org.junit.jupiter.api.Tag;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class CatalogPostgreSqlVersion15IT extends CatalogPostgreSqlIT {
public CatalogPostgreSqlVersion15IT() {
postgreImageName = PGImageName.VERSION_15;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import com.datastrato.gravitino.integration.test.container.PGImageName;
import org.junit.jupiter.api.Tag;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class CatalogPostgreSqlVersion16IT extends CatalogPostgreSqlIT {
public CatalogPostgreSqlVersion16IT() {
postgreImageName = PGImageName.VERSION_16;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class TestMultipleJDBCLoad extends AbstractIT {
private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
private static final TestDatabaseName TEST_DB_NAME =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import org.junit.jupiter.api.Test;
import org.testcontainers.shaded.com.google.common.collect.Maps;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class TestPostgreSqlSchemaOperations extends TestPostgreSql {

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
import org.junit.jupiter.api.Test;
import org.testcontainers.shaded.com.google.common.collect.Maps;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class TestPostgreSqlTableOperations extends TestPostgreSql {

private static Type VARCHAR = Types.VarCharType.of(255);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class CatalogKafkaIT extends AbstractIT {
private static final Logger LOG = LoggerFactory.getLogger(CatalogKafkaIT.class);
private static final ContainerSuite CONTAINER_SUITE = ContainerSuite.getInstance();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.TestInstance;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class CatalogIcebergHiveIT extends CatalogIcebergBaseIT {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
public class CatalogIcebergKerberosHiveIT extends AbstractIT {

private static final Logger LOG = LoggerFactory.getLogger(CatalogIcebergKerberosHiveIT.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.TestInstance;

@Tag("gravitino-docker-it")
@Tag("gravitino-docker-test")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class CatalogIcebergRestIT extends CatalogIcebergBaseIT {

Expand Down
Loading

0 comments on commit 5985999

Please sign in to comment.