Merge branch 'main' into add_relational_gc

apache · Apr 19, 2024 · 9f086ab · 9f086ab
2 parents 511e3f0 + 5bc83cc
commit 9f086ab
Show file tree

Hide file tree

Showing 144 changed files with 3,158 additions and 1,183 deletions.
diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
@@ -95,7 +95,7 @@ jobs:
       - name: Backend Integration Test
         id: integrationTest
         run: |
-          ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }}
+          ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }} -PskipPyClientITs
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml
@@ -57,18 +57,18 @@ jobs:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
-      - name: Free up disk space
-        run: |
-          dev/ci/util_free_space.sh
-
       - name: Python Client Integration Test
         id: integrationTest
         run: |
-          #./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
-          #for pythonVersion in "3.8" "3.9" "3.10" "3.11"
-          #do
-          #  ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} :client:client-python:integrationTest
-          #done
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
+
+          for pythonVersion in "3.8" "3.9" "3.10" "3.11"
+          do
+            echo "Use Python version ${pythonVersion} to test the Python client."
+            ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} :client:client-python:test
+            # Clean Gravitino database to clean test data
+            rm -rf ./distribution/package/data
+          done
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

diff --git a/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java b/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java
@@ -72,7 +72,7 @@ public static final class IndexImpl implements Index {
      * @param name The name of the index
      * @param fieldNames The field names under the table contained in the index.
      */
-    public IndexImpl(IndexType indexType, String name, String[][] fieldNames) {
+    private IndexImpl(IndexType indexType, String name, String[][] fieldNames) {
       this.indexType = indexType;
       this.name = name;
       this.fieldNames = fieldNames;

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -282,100 +282,41 @@ subprojects {
 
     tasks.withType<JavaCompile>().configureEach {
       options.errorprone.isEnabled.set(true)
-      options.errorprone.disableAllChecks.set(true)
       options.errorprone.disableWarningsInGeneratedCode.set(true)
-      options.errorprone.enable(
-        "AnnotateFormatMethod",
-        "AlwaysThrows",
-        "ArrayEquals",
-        "ArrayToString",
-        "ArraysAsListPrimitiveArray",
-        "ArrayFillIncompatibleType",
-        "BadImport",
-        "BoxedPrimitiveEquality",
-        "ChainingConstructorIgnoresParameter",
-        "CheckNotNullMultipleTimes",
-        "ClassCanBeStatic",
-        "CollectionIncompatibleType",
-        "CollectionToArraySafeParameter",
-        "ComparingThisWithNull",
-        "ComparisonOutOfRange",
-        "CompatibleWithAnnotationMisuse",
-        "CompileTimeConstant",
-        "ConditionalExpressionNumericPromotion",
-        "DangerousLiteralNull",
-        "DeadException",
-        "DeadThread",
-        "DefaultCharset",
-        "DoNotCall",
-        "DoNotMock",
-        "DuplicateMapKeys",
-        "EqualsGetClass",
-        "EqualsNaN",
-        "EqualsNull",
-        "EqualsReference",
-        "EqualsWrongThing",
-        "ForOverride",
-        "FormatString",
-        "FormatStringAnnotation",
-        "GetClassOnAnnotation",
-        "GetClassOnClass",
-        "HashtableContains",
-        "IdentityBinaryExpression",
-        "IdentityHashMapBoxing",
-        "Immutable",
-        "ImmutableEnumChecker",
-        "Incomparable",
-        "IncompatibleArgumentType",
-        "IndexOfChar",
-        "InfiniteRecursion",
-        "InlineFormatString",
-        "InvalidJavaTimeConstant",
-        "InvalidPatternSyntax",
-        "IsInstanceIncompatibleType",
-        "JavaUtilDate",
-        "JUnit4ClassAnnotationNonStatic",
-        "JUnit4SetUpNotRun",
-        "JUnit4TearDownNotRun",
-        "JUnit4TestNotRun",
-        "JUnitAssertSameCheck",
-        "LockOnBoxedPrimitive",
-        "LoopConditionChecker",
-        "LossyPrimitiveCompare",
-        "MathRoundIntLong",
-        "MissingSuperCall",
-        "ModifyingCollectionWithItself",
-        "MutablePublicArray",
-        "NonCanonicalStaticImport",
-        "NonFinalCompileTimeConstant",
-        "NonRuntimeAnnotation",
-        "NullTernary",
-        "OptionalEquality",
-        "PackageInfo",
-        "ParametersButNotParameterized",
-        "RandomCast",
-        "RandomModInteger",
-        "ReferenceEquality",
-        "SelfAssignment",
-        "SelfComparison",
-        "SelfEquals",
-        "SizeGreaterThanOrEqualsZero",
-        "StaticGuardedByInstance",
-        "StreamToString",
-        "StringBuilderInitWithChar",
-        "SubstringOfZero",
-        "ThrowNull",
-        "TruthSelfEquals",
-        "TryFailThrowable",
-        "TypeParameterQualifier",
-        "UnnecessaryCheckNotNull",
-        "UnnecessaryTypeArgument",
-        "UnusedAnonymousClass",
-        "UnusedCollectionModifiedInPlace",
-        "UnusedVariable",
-        "UseCorrectAssertInTests",
-        "VarTypeName",
-        "XorPower"
+      options.errorprone.disable(
+        "AlmostJavadoc",
+        "CanonicalDuration",
+        "CheckReturnValue",
+        "ComparableType",
+        "ConstantOverflow",
+        "DoubleBraceInitialization",
+        "EqualsUnsafeCast",
+        "EmptyBlockTag",
+        "FutureReturnValueIgnored",
+        "InconsistentCapitalization",
+        "InconsistentHashCode",
+        "JavaTimeDefaultTimeZone",
+        "JdkObsolete",
+        "LockNotBeforeTry",
+        "MissingSummary",
+        "MissingOverride",
+        "MutableConstantField",
+        "NonOverridingEquals",
+        "ObjectEqualsForPrimitives",
+        "OperatorPrecedence",
+        "ReturnValueIgnored",
+        "SameNameButDifferent",
+        "StaticAssignmentInConstructor",
+        "StringSplitter",
+        "ThreadPriorityCheck",
+        "ThrowIfUncheckedKnownChecked",
+        "TypeParameterUnusedInFormals",
+        "UnicodeEscape",
+        "UnnecessaryParentheses",
+        "UnsafeReflectiveConstructionCast",
+        "UnusedMethod",
+        "VariableNameSameAsType",
+        "WaitNotInLoop"
       )
     }
   }
@@ -407,6 +348,7 @@ subprojects {
     plugins.apply(NodePlugin::class)
     configure<NodeExtension> {
       version.set("20.9.0")
+      pnpmVersion.set("9.x")
       nodeProjectDir.set(file("$rootDir/.node"))
       download.set(true)
     }

diff --git a/...talog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java b/...talog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogCapability.java
@@ -25,4 +25,18 @@ public CapabilityResult columnDefaultValue() {
         "The DEFAULT constraint for column is only supported since Hive 3.0, "
             + "but the current Gravitino Hive catalog only supports Hive 2.x.");
   }
+
+  @Override
+  public CapabilityResult caseSensitiveOnName(Scope scope) {
+    switch (scope) {
+      case SCHEMA:
+      case TABLE:
+      case COLUMN:
+        // Hive is case insensitive, see
+        // https://cwiki.apache.org/confluence/display/Hive/User+FAQ#UserFAQ-AreHiveSQLidentifiers(e.g.tablenames,columnnames,etc)casesensitive?
+        return CapabilityResult.unsupported("Hive is case insensitive.");
+      default:
+        return CapabilityResult.SUPPORTED;
+    }
+  }
 }
diff --git a/...e/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java b/...e/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java
@@ -584,7 +584,7 @@ public void testHiveTableProperties() throws TException, InterruptedException {
     Assertions.assertEquals(TEXT_INPUT_FORMAT_CLASS, actualTable2.getSd().getInputFormat());
     Assertions.assertEquals(IGNORE_KEY_OUTPUT_FORMAT_CLASS, actualTable2.getSd().getOutputFormat());
     Assertions.assertEquals(EXTERNAL_TABLE.name(), actualTable2.getTableType());
-    Assertions.assertEquals(table2, actualTable2.getSd().getSerdeInfo().getName());
+    Assertions.assertEquals(table2.toLowerCase(), actualTable2.getSd().getSerdeInfo().getName());
     Assertions.assertEquals(TABLE_COMMENT, actualTable2.getParameters().get(COMMENT));
     Assertions.assertEquals(
         ((Boolean) tablePropertiesMetadata.getDefaultValue(EXTERNAL)).toString().toUpperCase(),
@@ -1224,7 +1224,7 @@ private void assertDefaultTableProperties(
     Assertions.assertEquals(
         ((TableType) tablePropertiesMetadata.getDefaultValue(TABLE_TYPE)).name(),
         actualTable.getTableType());
-    Assertions.assertEquals(tableName, actualTable.getSd().getSerdeInfo().getName());
+    Assertions.assertEquals(tableName.toLowerCase(), actualTable.getSd().getSerdeInfo().getName());
     Assertions.assertEquals(
         ((Boolean) tablePropertiesMetadata.getDefaultValue(EXTERNAL)).toString().toUpperCase(),
         actualTable.getParameters().get(EXTERNAL));

diff --git a/.../src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java b/.../src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java
@@ -255,7 +255,7 @@ protected void correctJdbcTableFields(
     }
 
     // Doris Cannot get comment from JDBC 8.x, so we need to get comment from sql
-    String comment = "";
+    StringBuilder comment = new StringBuilder();
     String sql =
         "SELECT TABLE_COMMENT FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?";
     try (PreparedStatement preparedStatement = connection.prepareStatement(sql)) {
@@ -264,10 +264,10 @@ protected void correctJdbcTableFields(
 
       try (ResultSet resultSet = preparedStatement.executeQuery()) {
         while (resultSet.next()) {
-          comment += resultSet.getString("TABLE_COMMENT");
+          comment.append(resultSet.getString("TABLE_COMMENT"));
         }
       }
-      tableBuilder.withComment(comment);
+      tableBuilder.withComment(comment.toString());
     } catch (SQLException e) {
       throw exceptionMapper.toGravitinoException(e);
     }

diff --git a/.../src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java b/.../src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java
@@ -85,7 +85,7 @@ protected String generateCreateTableSql(
     }
 
     Preconditions.checkArgument(
-        distribution == Distributions.NONE, "MySQL does not support distribution");
+        Distributions.NONE.equals(distribution), "MySQL does not support distribution");
 
     validateIncrementCol(columns, indexes);
     StringBuilder sqlBuilder = new StringBuilder();

diff --git a/...java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java b/...java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java
@@ -84,7 +84,7 @@ protected String generateCreateTableSql(
           "Currently we do not support Partitioning in PostgreSQL");
     }
     Preconditions.checkArgument(
-        distribution == Distributions.NONE, "PostgreSQL does not support distribution");
+        Distributions.NONE.equals(distribution), "PostgreSQL does not support distribution");
 
     StringBuilder sqlBuilder = new StringBuilder();
     sqlBuilder

diff --git a/...og-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java b/...og-kafka/src/main/java/com/datastrato/gravitino/catalog/kafka/KafkaCatalogOperations.java
@@ -216,7 +216,7 @@ public Topic createTopic(
       LOG.info(
           "Created topic {}[id: {}] with {} partitions and replication factor {}",
           ident,
-          topicId.toString(),
+          topicId,
           createTopicsResult.numPartitions(ident.name()).get(),
           createTopicsResult.replicationFactor(ident.name()).get());
 

diff --git a/...ain/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java b/...ain/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java
@@ -128,7 +128,7 @@ public NameIdentifier[] listSchemas(Namespace namespace) throws NoSuchCatalogExc
           icebergTableOps.listNamespace(IcebergTableOpsHelper.getIcebergNamespace()).namespaces();
 
       return namespaces.stream()
-          .map(icebergNamespace -> NameIdentifier.of(icebergNamespace.levels()))
+          .map(icebergNamespace -> NameIdentifier.of(namespace, icebergNamespace.toString()))
           .toArray(NameIdentifier[]::new);
     } catch (NoSuchNamespaceException e) {
       throw new NoSuchSchemaException(

diff --git a/...g/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergSchema.java b/...g/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergSchema.java
@@ -65,6 +65,18 @@ public void testCreateIcebergSchema() {
     Assertions.assertTrue(exception.getMessage().contains("already exists"));
   }
 
+  @Test
+  public void testListSchema() {
+    IcebergCatalog icebergCatalog = initIcebergCatalog("testListIcebergSchema");
+    NameIdentifier ident = NameIdentifier.of("metalake", icebergCatalog.name(), "test");
+    icebergCatalog.asSchemas().createSchema(ident, COMMENT_VALUE, Maps.newHashMap());
+
+    NameIdentifier[] schemas = icebergCatalog.asSchemas().listSchemas(ident.namespace());
+    Assertions.assertEquals(1, schemas.length);
+    Assertions.assertEquals(ident.name(), schemas[0].name());
+    Assertions.assertEquals(ident.namespace(), schemas[0].namespace());
+  }
+
   @Test
   public void testAlterSchema() {
     IcebergCatalog icebergCatalog = initIcebergCatalog("testAlterSchema");