diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.sql index 8e6e4f95a77..c3d7890550a 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.sql @@ -4,8 +4,7 @@ USE "test.gt_iceberg".gt_db2; -- Unsupported Type: TINYINT, SMALLINT CREATE TABLE tb01 ( - f1 VARCHAR(200), - f2 CHAR(20), + f1 VARCHAR, f3 VARBINARY, f4 DECIMAL(10, 3), f5 REAL, @@ -21,17 +20,16 @@ CREATE TABLE tb01 ( SHOW CREATE TABLE tb01; -INSERT INTO tb01 (f1, f2, f3, f4, f5, f6, f7, f10, f11, f12, f13, f14, f15) -VALUES ('Sample text 1', 'Text1', x'65', 123.456, 7.89, 12.34, true, 1000, 1000, 100000, DATE '2024-01-01', TIME '08:00:00', TIMESTAMP '2024-01-01 08:00:00'); +INSERT INTO tb01 (f1, f3, f4, f5, f6, f7, f10, f11, f12, f13, f14, f15) +VALUES ('Sample text 1', x'65', 123.456, 7.89, 12.34, true, 1000, 1000, 100000, DATE '2024-01-01', TIME '08:00:00', TIMESTAMP '2024-01-01 08:00:00'); -INSERT INTO tb01 (f1, f2, f3, f4, f5, f6, f7, f10, f11, f12, f13, f14, f15) -VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO tb01 (f1, f3, f4, f5, f6, f7, f10, f11, f12, f13, f14, f15) +VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); select * from tb01 order by f1; CREATE TABLE tb02 ( - f1 VARCHAR(200) not null , - f2 CHAR(20) not null , + f1 VARCHAR not null , f3 VARBINARY not null , f4 DECIMAL(10, 3) not null , f5 REAL not null , @@ -47,20 +45,20 @@ CREATE TABLE tb02 ( show create table tb02; -INSERT INTO tb02 (f1, f2, f3, f4, f5, f6, f7, f10, f11, f12, f13, f14, f15) -VALUES ('Sample text 1', 'Text1', x'65', 123.456, 7.89, 12.34, true, 1000, 1000, 100000, DATE '2024-01-01', TIME '08:00:00', TIMESTAMP '2024-01-01 08:00:00'); +INSERT INTO tb02 (f1, f3, f4, f5, f6, f7, f10, f11, f12, f13, f14, f15) +VALUES ('Sample text 1', x'65', 123.456, 7.89, 12.34, true, 1000, 1000, 100000, DATE '2024-01-01', TIME '08:00:00', TIMESTAMP '2024-01-01 08:00:00'); -INSERT INTO tb02 (f1, f2, f3, f4, f5, f6, f7, f10, f11, f12, f13, f14, f15) -VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO tb02 (f1, f3, f4, f5, f6, f7, f10, f11, f12, f13, f14, f15) +VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); -INSERT INTO tb02 (f1, f2, f3, f4, f5, f6, f7, f10, f11, f12, f13, f14, f15) -VALUES ('Sample text 1', NULL, x'65', 123.456, 7.89, 12.34, true, 1000, 1000, 100000, DATE '2024-01-01', TIME '08:00:00', TIMESTAMP '2024-01-01 08:00:00'); +INSERT INTO tb02 (f1, f3, f4, f5, f6, f10, f11, f12, f13, f14, f15) +VALUES ('Sample text 1', x'65', 123.456, 7.89, 12.34, 1000, 1000, 100000, DATE '2024-01-01', TIME '08:00:00', TIMESTAMP '2024-01-01 08:00:00'); -INSERT INTO tb02 (f1, f2, f3, f4, f5, f6, f7, f10, f11, f12, f13, f14, f15) -VALUES ('Sample text 1', 'same3', x'65', 123.456, 7.89, 12.34, true, 1000, 1000, NULl, DATE '2024-01-01', TIME '08:00:00', TIMESTAMP '2024-01-01 08:00:00'); +INSERT INTO tb02 (f1, f3, f4, f5, f6, f7, f10, f11, f12, f13, f14, f15) +VALUES ('Sample text 1', x'65', 123.456, 7.89, 12.34, true, 1000, 1000, NULL, DATE '2024-01-01', TIME '08:00:00', TIMESTAMP '2024-01-01 08:00:00'); -INSERT INTO tb02 (f1, f2, f3, f4, f5, f6, f7, f10, f11, f12, f13, f14, f15) -VALUES ('Sample text 1', 'same9', x'65', 123.456, 7.89, 12.34, true, 1000, 1000, 1992382342, DATE '2024-01-01', NULL, TIMESTAMP '2024-01-01 08:00:00'); +INSERT INTO tb02 (f1, f3, f4, f5, f6, f7, f10, f11, f12, f13, f14, f15) +VALUES ('Sample text 1', x'65', 123.456, 7.89, 12.34, true, 1000, 1000, 1992382342, DATE '2024-01-01', NULL, TIMESTAMP '2024-01-01 08:00:00'); drop table tb01; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt index aa6b6901c5d..8ff0979aa96 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt @@ -6,7 +6,6 @@ CREATE TABLE "CREATE TABLE ""test.gt_iceberg"".gt_db2.tb01 ( f1 varchar, - f2 varchar, f3 varbinary, f4 decimal(10, 3), f5 real, @@ -26,14 +25,13 @@ INSERT: 1 row INSERT: 1 row -"Sample text 1","Text1","65","123.456","7.89","12.34","true","1000","1000","100000","2024-01-01","08:00:00.000","2024-01-01 08:00:00.000" -"","","","","","","","","","","","","" +"Sample text 1","65","123.456","7.89","12.34","true","1000","1000","100000","2024-01-01","08:00:00.000","2024-01-01 08:00:00.000" +"","","","","","","","","","","","" CREATE TABLE "CREATE TABLE ""test.gt_iceberg"".gt_db2.tb02 ( f1 varchar NOT NULL, - f2 varchar NOT NULL, f3 varbinary NOT NULL, f4 decimal(10, 3) NOT NULL, f5 real NOT NULL, @@ -52,7 +50,7 @@ INSERT: 1 row NULL value not allowed for NOT NULL column: f1 - NULL value not allowed for NOT NULL column: f2 + NULL value not allowed for NOT NULL column: f7 NULL value not allowed for NOT NULL column: f12 diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.sql new file mode 100644 index 00000000000..91a3ea10906 --- /dev/null +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.sql @@ -0,0 +1,18 @@ +CREATE SCHEMA "test.gt_iceberg".varchar_db1; + +USE "test.gt_iceberg".varchar_db1; + +CREATE TABLE tb01 (id int, name char(20)); + +CREATE TABLE tb02 (id int, name char); + +CREATE TABLE tb03 (id int, name varchar(233)); + +CREATE TABLE tb04 (id int, name varchar); + +SHOW CREATE TABLE "test.gt_iceberg".varchar_db1.tb04; + +drop table "test.gt_iceberg".varchar_db1.tb04; + +drop schema "test.gt_iceberg".varchar_db1; + diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt new file mode 100644 index 00000000000..260e856e077 --- /dev/null +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt @@ -0,0 +1,21 @@ +CREATE SCHEMA + +USE + + Iceberg does not support the datatype CHAR + + Iceberg does not support the datatype CHAR + + Iceberg does not support the datatype VARCHAR with length + +CREATE TABLE + +"CREATE TABLE ""test.gt_iceberg"".varchar_db1.tb04 ( + id integer, + name varchar +) +COMMENT ''" + +DROP TABLE + +DROP SCHEMA diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_iceberg_prepare.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_iceberg_prepare.sql index 1f5029a8297..dfd439612d3 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_iceberg_prepare.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/tpch/catalog_iceberg_prepare.sql @@ -12,13 +12,13 @@ use "test.gt_iceberg2".gt_tpch2; CREATE TABLE customer ( custkey bigint, - name varchar(25), - address varchar(40), + name varchar, + address varchar, nationkey bigint, - phone varchar(15), + phone varchar, acctbal decimal(12, 2), - mktsegment varchar(10), - comment varchar(117) + mktsegment varchar, + comment varchar ); CREATE TABLE lineitem ( @@ -30,45 +30,45 @@ CREATE TABLE lineitem ( extendedprice decimal(12, 2), discount decimal(12, 2), tax decimal(12, 2), - returnflag varchar(1), - linestatus varchar(1), + returnflag varchar, + linestatus varchar, shipdate date, commitdate date, receiptdate date, - shipinstruct varchar(25), - shipmode varchar(10), - comment varchar(44) + shipinstruct varchar, + shipmode varchar, + comment varchar ); CREATE TABLE nation ( nationkey bigint, - name varchar(25), + name varchar, regionkey bigint, - comment varchar(152) + comment varchar ); CREATE TABLE orders ( orderkey bigint, custkey bigint, - orderstatus varchar(1), + orderstatus varchar, totalprice decimal(12, 2), orderdate date, - orderpriority varchar(15), - clerk varchar(15), + orderpriority varchar, + clerk varchar, shippriority integer, - comment varchar(79) + comment varchar ); CREATE TABLE part ( partkey bigint, - name varchar(55), - mfgr varchar(25), - brand varchar(10), - type varchar(25), + name varchar, + mfgr varchar, + brand varchar, + type varchar, size integer, - container varchar(10), + container varchar, retailprice decimal(12, 2), - comment varchar(23) + comment varchar ); CREATE TABLE partsupp ( @@ -76,23 +76,23 @@ CREATE TABLE partsupp ( suppkey bigint, availqty integer, supplycost decimal(12, 2), - comment varchar(199) + comment varchar ); CREATE TABLE region ( regionkey bigint, - name varchar(25), - comment varchar(152) + name varchar, + comment varchar ); CREATE TABLE supplier ( suppkey bigint, - name varchar(25), - address varchar(40), + name varchar, + address varchar, nationkey bigint, - phone varchar(15), + phone varchar, acctbal decimal(12, 2), - comment varchar(101) + comment varchar ); insert into customer select * from tpch.tiny.customer; diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergDataTypeTransformer.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergDataTypeTransformer.java index f6f815cda98..21180cd1cf8 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergDataTypeTransformer.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergDataTypeTransformer.java @@ -8,19 +8,34 @@ import com.datastrato.gravitino.rel.types.Type; import com.datastrato.gravitino.rel.types.Type.Name; import com.datastrato.gravitino.rel.types.Types; +import com.datastrato.gravitino.trino.connector.GravitinoErrorCode; import com.datastrato.gravitino.trino.connector.util.GeneralDataTypeTransformer; +import io.trino.spi.TrinoException; import io.trino.spi.type.VarbinaryType; +import io.trino.spi.type.VarcharType; /** Type transformer between Iceberg and Trino */ public class IcebergDataTypeTransformer extends GeneralDataTypeTransformer { @Override public Type getGravitinoType(io.trino.spi.type.Type type) { - Type gravitinoType = super.getGravitinoType(type); - if (gravitinoType.name() == Name.VARCHAR || gravitinoType.name() == Name.FIXEDCHAR) { + Class typeClass = type.getClass(); + if (typeClass == io.trino.spi.type.CharType.class) { + throw new TrinoException( + GravitinoErrorCode.GRAVITINO_ILLEGAL_ARGUMENT, + "Iceberg does not support the datatype CHAR"); + } else if (typeClass == io.trino.spi.type.VarcharType.class) { + VarcharType varCharType = (VarcharType) type; + if (varCharType.getLength().isPresent()) { + throw new TrinoException( + GravitinoErrorCode.GRAVITINO_ILLEGAL_ARGUMENT, + "Iceberg does not support the datatype VARCHAR with length"); + } + return Types.StringType.get(); } - return gravitinoType; + + return super.getGravitinoType(type); } @Override diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergDataTypeTransformer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergDataTypeTransformer.java new file mode 100644 index 00000000000..b69c5d0a31d --- /dev/null +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergDataTypeTransformer.java @@ -0,0 +1,41 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.trino.connector.catalog.iceberg; + +import com.datastrato.gravitino.rel.types.Types; +import com.datastrato.gravitino.trino.connector.util.GeneralDataTypeTransformer; +import io.trino.spi.TrinoException; +import io.trino.spi.type.VarcharType; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class TestIcebergDataTypeTransformer { + + @Test + public void testTrinoTypeToGravitinoType() { + GeneralDataTypeTransformer generalDataTypeTransformer = new IcebergDataTypeTransformer(); + io.trino.spi.type.Type charTypeWithLengthOne = io.trino.spi.type.CharType.createCharType(1); + + Exception e = + Assert.expectThrows( + TrinoException.class, + () -> generalDataTypeTransformer.getGravitinoType(charTypeWithLengthOne)); + Assert.assertTrue(e.getMessage().contains("Iceberg does not support the datatype CHAR")); + + io.trino.spi.type.Type varcharType = io.trino.spi.type.VarcharType.createVarcharType(1); + e = + Assert.expectThrows( + TrinoException.class, () -> generalDataTypeTransformer.getGravitinoType(varcharType)); + Assert.assertTrue( + e.getMessage().contains("Iceberg does not support the datatype VARCHAR with length")); + + io.trino.spi.type.Type varcharTypeWithoutLength = VarcharType.VARCHAR; + + Assert.assertEquals( + generalDataTypeTransformer.getGravitinoType(varcharTypeWithoutLength), + Types.StringType.get()); + } +}