From 42245d3cf7fd1ab4f3caaeab643a5a6233bf0ce7 Mon Sep 17 00:00:00 2001 From: SteNicholas Date: Wed, 7 Feb 2024 19:27:28 +0800 Subject: [PATCH] [#2117] improvement(api): Type adds UNKNOWN column data type to handle an unresolvable type from the catalog --- .../datastrato/gravitino/rel/types/Type.java | 5 ++++- .../datastrato/gravitino/rel/types/Types.java | 22 +++++++++++++++++++ .../datastrato/gravitino/rel/TestTypes.java | 5 +++++ .../catalog/hive/converter/FromHiveType.java | 8 +++---- .../catalog/converter/TestTypeConverter.java | 11 ++++++++++ .../mysql/converter/MysqlTypeConverter.java | 2 +- .../converter/TestMysqlTypeConverter.java | 2 ++ .../converter/PostgreSqlTypeConverter.java | 2 +- .../TestPostgreSqlTypeConverter.java | 2 ++ 9 files changed, 51 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/com/datastrato/gravitino/rel/types/Type.java b/api/src/main/java/com/datastrato/gravitino/rel/types/Type.java index 0be8ecf83c2..f8a97c16bf5 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/types/Type.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/types/Type.java @@ -73,7 +73,10 @@ enum Name { UNION, /** The null type. A null type represents a value that is null. */ - NULL + NULL, + + /** The unknown type. An unknown type represents an unresolvable type. */ + UNKNOWN } /** The base type of all primitive types. */ diff --git a/api/src/main/java/com/datastrato/gravitino/rel/types/Types.java b/api/src/main/java/com/datastrato/gravitino/rel/types/Types.java index 40622d7d140..65441afbb24 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/types/Types.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/types/Types.java @@ -34,6 +34,28 @@ public String simpleString() { } } + /** The data type representing `UNKNOWN` values. */ + public static class UnknownType implements Type { + private static final UnknownType INSTANCE = new UnknownType(); + + /** @return The singleton instance of {@link UnknownType}. */ + public static UnknownType get() { + return INSTANCE; + } + + private UnknownType() {} + + @Override + public Name name() { + return Name.UNKNOWN; + } + + @Override + public String simpleString() { + return "unknown"; + } + } + /** The boolean type in Gravitino. */ public static class BooleanType extends Type.PrimitiveType { private static final BooleanType INSTANCE = new BooleanType(); diff --git a/api/src/test/java/com/datastrato/gravitino/rel/TestTypes.java b/api/src/test/java/com/datastrato/gravitino/rel/TestTypes.java index 830e44851de..ee0e8c6abe8 100644 --- a/api/src/test/java/com/datastrato/gravitino/rel/TestTypes.java +++ b/api/src/test/java/com/datastrato/gravitino/rel/TestTypes.java @@ -24,6 +24,11 @@ public void testPrimitiveTypes() { Assertions.assertSame(nullType, Types.NullType.get()); Assertions.assertEquals("null", nullType.simpleString()); + Types.UnknownType unknownType = Types.UnknownType.get(); + Assertions.assertEquals(Type.Name.UNKNOWN, unknownType.name()); + Assertions.assertSame(unknownType, Types.UnknownType.get()); + Assertions.assertEquals("unknown", unknownType.simpleString()); + Types.ByteType byteType = Types.ByteType.get(); Assertions.assertEquals(Type.Name.BYTE, byteType.name()); Assertions.assertSame(byteType, Types.ByteType.get()); diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/FromHiveType.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/FromHiveType.java index 60a7214b9f2..fd6db3c12c5 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/FromHiveType.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/converter/FromHiveType.java @@ -54,7 +54,7 @@ public static Type convert(String hiveType) throws IllegalArgumentException { * @return The equivalent Gravitino Type. * @throws IllegalArgumentException if the Hive data type category is unknown or unsupported. */ - private static Type toGravitinoType(TypeInfo hiveTypeInfo) throws IllegalArgumentException { + public static Type toGravitinoType(TypeInfo hiveTypeInfo) throws IllegalArgumentException { switch (hiveTypeInfo.getCategory()) { case PRIMITIVE: switch (hiveTypeInfo.getTypeName()) { @@ -98,8 +98,7 @@ private static Type toGravitinoType(TypeInfo hiveTypeInfo) throws IllegalArgumen return Types.DecimalType.of(decimalTypeInfo.precision(), decimalTypeInfo.scale()); } - throw new IllegalArgumentException( - "Unknown Hive type: " + hiveTypeInfo.getQualifiedName()); + return Types.UnknownType.get(); } case LIST: return Types.ListType.nullable( @@ -128,8 +127,7 @@ private static Type toGravitinoType(TypeInfo hiveTypeInfo) throws IllegalArgumen .map(FromHiveType::toGravitinoType) .toArray(Type[]::new)); default: - throw new IllegalArgumentException( - "Unknown category of Hive type: " + hiveTypeInfo.getCategory()); + return Types.UnknownType.get(); } } diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/converter/TestTypeConverter.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/converter/TestTypeConverter.java index b4ef1bb5b66..89f562b0f93 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/converter/TestTypeConverter.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/converter/TestTypeConverter.java @@ -29,7 +29,9 @@ import com.datastrato.gravitino.catalog.hive.converter.FromHiveType; import com.datastrato.gravitino.catalog.hive.converter.ToHiveType; +import com.datastrato.gravitino.rel.types.Types; import java.util.Arrays; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -69,6 +71,8 @@ public void testTypeConverter() { Arrays.asList( getPrimitiveTypeInfo(STRING_TYPE_NAME), getPrimitiveTypeInfo(INT_TYPE_NAME))) .getTypeName()); + Assertions.assertEquals( + Types.UnknownType.get(), FromHiveType.toGravitinoType(new UnknownTypeInfo())); } private void testConverter(String typeName) { @@ -76,4 +80,11 @@ private void testConverter(String typeName) { TypeInfo convertedType = ToHiveType.convert(FromHiveType.convert(hiveType.getTypeName())); Assertions.assertEquals(hiveType, convertedType); } + + static class UnknownTypeInfo extends PrimitiveTypeInfo { + @Override + public String getTypeName() { + return Types.UnknownType.get().simpleString(); + } + } } diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlTypeConverter.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlTypeConverter.java index 3e18cabb2f2..16acef1aae7 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlTypeConverter.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/converter/MysqlTypeConverter.java @@ -55,7 +55,7 @@ public Type toGravitinoType(JdbcTypeBean typeBean) { case BINARY: return Types.BinaryType.get(); default: - throw new IllegalArgumentException("Not a supported type: " + typeBean.getTypeName()); + return Types.UnknownType.get(); } } diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/converter/TestMysqlTypeConverter.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/converter/TestMysqlTypeConverter.java index 8059896bc9f..536621a1559 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/converter/TestMysqlTypeConverter.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/converter/TestMysqlTypeConverter.java @@ -44,6 +44,8 @@ public void testToGravitinoType() { checkJdbcTypeToGravitinoType(Types.FixedCharType.of(20), CHAR, "20", null); checkJdbcTypeToGravitinoType(Types.StringType.get(), TEXT, null, null); checkJdbcTypeToGravitinoType(Types.BinaryType.get(), BINARY, null, null); + checkJdbcTypeToGravitinoType( + Types.UnknownType.get(), Types.UnknownType.get().simpleString(), null, null); } @Test diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java index bc3f295f00f..fdb54d8d613 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java @@ -59,7 +59,7 @@ public Type toGravitinoType(JdbcTypeBean typeBean) { case BYTEA: return Types.BinaryType.get(); default: - throw new IllegalArgumentException("Not a supported type: " + typeBean); + return Types.UnknownType.get(); } } diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java index 667f4d19262..3177dc47ba3 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java @@ -47,6 +47,8 @@ public void testToGravitinoType() { checkJdbcTypeToGravitinoType(Types.FixedCharType.of(20), BPCHAR, "20", null); checkJdbcTypeToGravitinoType(Types.StringType.get(), TEXT, null, null); checkJdbcTypeToGravitinoType(Types.BinaryType.get(), BYTEA, null, null); + checkJdbcTypeToGravitinoType( + Types.UnknownType.get(), Types.UnknownType.get().simpleString(), null, null); } @Test