From 1b4c9c1513951cd50baf884171708de774345cc8 Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 4 Jan 2024 10:00:11 +0800 Subject: [PATCH 1/2] Support Fixed type conversion --- integration-test/build.gradle.kts | 2 +- .../gravitino/integration/test/trino/TrinoConnectorIT.java | 2 +- .../trino/connector/catalog/hive/HiveDataTypeTransformer.java | 2 +- .../trino/connector/util/GeneralDataTypeTransformer.java | 2 ++ 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index ee7b14860c9..ac5e7af52f9 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -283,7 +283,7 @@ tasks.test { // Gravitino CI Docker image environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.7") - environment("GRAVITINO_CI_TRINO_DOCKER_IMAGE", "datastrato/gravitino-ci-trino:0.1.2") + environment("GRAVITINO_CI_TRINO_DOCKER_IMAGE", "datastrato/gravitino-ci-trino:0.1.3") val testMode = project.properties["testMode"] as? String ?: "embedded" systemProperty("gravitino.log.path", buildDir.path + "/integration-test.log") diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java index 1edca4109ed..df7b358f42e 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java @@ -505,7 +505,7 @@ private ColumnDTO[] createIcebergFullTypeColumns() { ColumnDTO[] columnDTO = createFullTypeColumns(); Set unsupportedType = - Sets.newHashSet("ByteType", "ShortType", "VarCharType", "FixedCharType", "FixedType"); + Sets.newHashSet("ByteType", "ShortType", "VarCharType", "FixedCharType"); return Arrays.stream(columnDTO) .filter(c -> !unsupportedType.contains(c.name())) .toArray(ColumnDTO[]::new); diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveDataTypeTransformer.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveDataTypeTransformer.java index 701d9adcb21..18d5c07ac89 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveDataTypeTransformer.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveDataTypeTransformer.java @@ -12,7 +12,7 @@ /** Type transformer between Hive and Trino */ public class HiveDataTypeTransformer extends GeneralDataTypeTransformer { - // Hive varchar max length of 65535 + // Max length of Hive varchar is 65535 private static final int HIVE_VARCHAR_MAX_LENGTH = 65535; @Override diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/GeneralDataTypeTransformer.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/GeneralDataTypeTransformer.java index 888c1fd81b1..5eeb6f14975 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/GeneralDataTypeTransformer.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/GeneralDataTypeTransformer.java @@ -69,6 +69,8 @@ public Type getTrinoType(com.datastrato.gravitino.rel.types.Type type) { return VarcharType.createVarcharType(((Types.VarCharType) type).length()); case BINARY: return VARBINARY; + case FIXED: + return VARBINARY; case DATE: return DATE; case TIME: From 156ef46e84888af3bb41660e4c09d6dad4397035 Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 5 Jan 2024 18:51:26 +0800 Subject: [PATCH 2/2] Fix discussion --- .../catalog/iceberg/IcebergDataTypeTransformer.java | 9 +++++++++ .../trino/connector/util/GeneralDataTypeTransformer.java | 2 -- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergDataTypeTransformer.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergDataTypeTransformer.java index a7633b22c6e..f6f815cda98 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergDataTypeTransformer.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergDataTypeTransformer.java @@ -9,6 +9,7 @@ import com.datastrato.gravitino.rel.types.Type.Name; import com.datastrato.gravitino.rel.types.Types; import com.datastrato.gravitino.trino.connector.util.GeneralDataTypeTransformer; +import io.trino.spi.type.VarbinaryType; /** Type transformer between Iceberg and Trino */ public class IcebergDataTypeTransformer extends GeneralDataTypeTransformer { @@ -21,4 +22,12 @@ public Type getGravitinoType(io.trino.spi.type.Type type) { } return gravitinoType; } + + @Override + public io.trino.spi.type.Type getTrinoType(Type type) { + if (Name.FIXED == type.name()) { + return VarbinaryType.VARBINARY; + } + return super.getTrinoType(type); + } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/GeneralDataTypeTransformer.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/GeneralDataTypeTransformer.java index 5eeb6f14975..888c1fd81b1 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/GeneralDataTypeTransformer.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/GeneralDataTypeTransformer.java @@ -69,8 +69,6 @@ public Type getTrinoType(com.datastrato.gravitino.rel.types.Type type) { return VarcharType.createVarcharType(((Types.VarCharType) type).length()); case BINARY: return VARBINARY; - case FIXED: - return VARBINARY; case DATE: return DATE; case TIME: