Skip to content

Commit

Permalink
[fix](jdbc catalog) fix and add mysql and doris extremum test apache#…
Browse files Browse the repository at this point in the history 
…41679 (apache#42122)

cherry pick from apache#41679

---------

Co-authored-by: zy-kkk <[email protected]>
  • Loading branch information
@morningman @zy-kkk
morningman and zy-kkk authored Oct 21, 2024
1 parent b9e2738 commit a150d16
Show file tree
Hide file tree
Showing 16 changed files with 980 additions and 46 deletions.
153 changes: 153 additions & 0 deletions docker/thirdparties/docker-compose/mysql/init/03-create-table.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,3 +352,156 @@ col_int_undef_signed2 int

create table doris_test.text_push (pk varchar(10));


create table doris_test.all_types_nullable (
`tinyint_u` tinyint unsigned,
`smallint_u` smallint unsigned,
`mediumint_u` mediumint unsigned,
`int_u` int unsigned,
`bigint_u` bigint unsigned,
`decimal1_u` decimal unsigned,
`decimal2_u` decimal(9, 2) unsigned,
`decimal3_u` decimal(18, 5) unsigned,
`decimal4_u` decimal(38, 10) unsigned,
`decimal5_u` decimal(65, 30) unsigned,
`double_u` double unsigned,
`float_u` float unsigned,
`boolean` boolean,
`tinyint` tinyint,
`smallint` smallint,
`mediumint` mediumint,
`int` int,
`bigint` bigint,
`double` double,
`float` float,
`decimal1` decimal,
`decimal2` decimal(9, 2),
`decimal3` decimal(18, 5) ,
`decimal4` decimal(38, 10),
`decimal5` decimal(65, 30),
`year` year,
`time1` time,
`time2` time(3),
`time3` time(6),
`date` date,
`datetime` datetime,
`timestamp1` timestamp null,
`timestamp2` timestamp(3) null,
`timestamp3` timestamp(6) null,
`char` char(5),
`varchar` varchar(10),
`text` text,
`blob` blob,
`json` json,
`set` set('Option1', 'Option2', 'Option3'),
`bit` bit(6),
`binary` binary(12),
`varbinary` varbinary(12),
`enum` enum('Value1', 'Value2', 'Value3')
) engine=innodb charset=utf8;


create table doris_test.all_types_non_nullable (
`tinyint_u` tinyint unsigned NOT NULL,
`smallint_u` smallint unsigned NOT NULL,
`mediumint_u` mediumint unsigned NOT NULL,
`int_u` int unsigned NOT NULL,
`bigint_u` bigint unsigned NOT NULL,
`decimal1_u` decimal unsigned NOT NULL,
`decimal2_u` decimal(9, 2) unsigned NOT NULL,
`decimal3_u` decimal(18, 5) unsigned NOT NULL,
`decimal4_u` decimal(38, 10) unsigned NOT NULL,
`decimal5_u` decimal(65, 30) unsigned NOT NULL,
`double_u` double unsigned NOT NULL,
`float_u` float unsigned NOT NULL,
`boolean` boolean NOT NULL,
`tinyint` tinyint NOT NULL,
`smallint` smallint NOT NULL,
`mediumint` mediumint NOT NULL,
`int` int NOT NULL,
`bigint` bigint NOT NULL,
`double` double NOT NULL,
`float` float NOT NULL,
`decimal1` decimal NOT NULL,
`decimal2` decimal(9, 2) NOT NULL,
`decimal3` decimal(18, 5) NOT NULL,
`decimal4` decimal(38, 10) NOT NULL,
`decimal5` decimal(65, 30) NOT NULL,
`year` year NOT NULL,
`time1` time NOT NULL,
`time2` time(3) NOT NULL,
`time3` time(6) NOT NULL,
`date` date NOT NULL,
`datetime` datetime NOT NULL,
`timestamp1` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`timestamp2` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
`timestamp3` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
`char` char(5) NOT NULL,
`varchar` varchar(10) NOT NULL,
`text` text NOT NULL,
`blob` blob NOT NULL,
`json` json NOT NULL,
`set` set('Option1', 'Option2', 'Option3') NOT NULL,
`bit` bit(6) NOT NULL,
`binary` binary(12) NOT NULL,
`varbinary` varbinary(12) NOT NULL,
`enum` enum('Value1', 'Value2', 'Value3') NOT NULL
) engine=innodb charset=utf8;


create table doris_test.all_types_multi_block (
`tinyint_u` tinyint unsigned,
`smallint_u` smallint unsigned,
`mediumint_u` mediumint unsigned,
`int_u` int unsigned,
`bigint_u` bigint unsigned,
`decimal1_u` decimal unsigned,
`decimal2_u` decimal(9, 2) unsigned,
`decimal3_u` decimal(18, 5) unsigned,
`decimal4_u` decimal(38, 10) unsigned,
`decimal5_u` decimal(65, 30) unsigned,
`double_u` double unsigned,
`float_u` float unsigned,
`boolean` boolean,
`tinyint` tinyint,
`smallint` smallint,
`mediumint` mediumint,
`int` int,
`bigint` bigint,
`double` double,
`float` float,
`decimal1` decimal,
`decimal2` decimal(9, 2),
`decimal3` decimal(18, 5) ,
`decimal4` decimal(38, 10),
`decimal5` decimal(65, 30),
`year` year,
`time1` time,
`time2` time(3),
`time3` time(6),
`date` date,
`datetime` datetime,
`timestamp1` timestamp null,
`timestamp2` timestamp(3) null,
`timestamp3` timestamp(6) null,
`char` char(5),
`varchar` varchar(10),
`text` text,
`blob` blob,
`json` json,
`set` set('Option1', 'Option2', 'Option3'),
`bit` bit(6),
`binary` binary(12),
`varbinary` varbinary(12),
`enum` enum('Value1', 'Value2', 'Value3')
) engine=innodb charset=utf8;


CREATE TABLE doris_test.`t_varchar` (
`varchar_col` varchar(21844)
);

CREATE TABLE doris_test.`t_char` (
`char_col` char(255) COLLATE utf8_bin DEFAULT NULL
);

37 changes: 37 additions & 0 deletions docker/thirdparties/docker-compose/mysql/init/04-insert.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1168,3 +1168,40 @@ insert into Doris.doris values ('doris');
insert into doris_test.compoundpredicate_test(pk,col_int_undef_signed,col_int_undef_signed2) values (0,null,23868),(1,68,-18),(2,19030,-125),(3,16539,null),(4,null,null),(5,null,-127),(6,14680,-26424),(7,-22270,12722),(8,null,null),(9,null,null),(10,null,7744),(11,null,-94),(12,16970,95),(13,null,7023),(14,null,1),(15,3679,-11),(16,null,-1079),(17,-22,null),(18,30995,null),(19,null,-79);

insert into doris_test.text_push values('a'),('aa'),('aaa');

insert into doris_test.all_types_nullable
values(0,0,0,0,0, 0, 0.00, 0.00000, 0.0000000000, 0.000000000000000000000000000000,0,0,false,-128,-32768,-8388608,-2147483648,-9223372036854775808,-1.7976931348623157E+308,-3.4028234E+38,-9999999999,-9999999.99,-9999999999999.99999,-9999999999999999999999999999.9999999999,-99999999999999999999999999999999999.999999999999999999999999999999,1901,'-838:59:59','-838:59:59.000','-838:59:59.000000','1000-01-01','1000-01-01 00:00:00','1970-01-01 00:00:01','1970-01-01 00:00:01.000','1970-01-01 00:00:01.000000','','','', '', '{}', '', b'000000', '', '', 'Value1'),
(255,65535,16777215,4294967295,18446744073709551615,9999999999, 9999999.99, 9999999999999.99999, 9999999999999999999999999999.9999999999, 99999999999999999999999999999999999.999999999999999999999999999999,1.7976931348623157E+308,3.4028234E+38,true,127,32767,8388607,2147483647,9223372036854775807,1.7976931348623157E+308,3.4028234E+38,9999999999,9999999.99,9999999999999.99999,9999999999999999999999999999.9999999999,99999999999999999999999999999999999.999999999999999999999999999999,2155,'838:59:59.000','838:59:59.000','838:59:59.000000','9999-12-31','9999-12-31 23:59:59','2038-01-19 03:14:07','2038-01-19 03:14:07.499','2038-01-19 03:14:07.499999','zzzz','abcdefgh','max', 'maxBLOB', '{"key": "maxJSON"}', 'Option1,Option2,Option3', b'111111', 'maxBIN', 'maxVARBIN', 'Value3'),
(0,0,0,0,0, 0, 0.00, 0.00000, 0.0000000000, 0.000000000000000000000000000000, 0, 0, false, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1901, '00:00:00', '00:00:00.000', '00:00:00.000000','1000-01-01','1000-01-01 00:00:00','1970-01-01 00:00:01','1970-01-01 00:00:01.000','1970-01-01 00:00:01.000000', '', '', '', '', '{}', '', b'', '', '', 'Value1'),
(NULL,NULL,NULL,NULL,NULL, NULL, NULL, NULL, NULL, NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);

insert into doris_test.all_types_non_nullable
values(0,0,0,0,0, 0, 0.00, 0.00000, 0.0000000000, 0.000000000000000000000000000000,0,0,false,-128,-32768,-8388608,-2147483648,-9223372036854775808,-1.7976931348623157E+308,-3.4028234E+38,-9999999999,-9999999.99,-9999999999999.99999,-9999999999999999999999999999.9999999999,-99999999999999999999999999999999999.999999999999999999999999999999,1901,'-838:59:59','-838:59:59.000','-838:59:59.000000','1000-01-01','1000-01-01 00:00:00','1970-01-01 00:00:01','1970-01-01 00:00:01.000','1970-01-01 00:00:01.000000','','','', '', '{}', '', b'000000', '', '', 'Value1'),
(255,65535,16777215,4294967295,18446744073709551615,9999999999, 9999999.99, 9999999999999.99999, 9999999999999999999999999999.9999999999, 99999999999999999999999999999999999.999999999999999999999999999999,1.7976931348623157E+308,3.4028234E+38,true,127,32767,8388607,2147483647,9223372036854775807,1.7976931348623157E+308,3.4028234E+38,9999999999,9999999.99,9999999999999.99999,9999999999999999999999999999.9999999999,99999999999999999999999999999999999.999999999999999999999999999999,2155,'838:59:59.000','838:59:59.000','838:59:59.000000','9999-12-31','9999-12-31 23:59:59','2038-01-19 03:14:07','2038-01-19 03:14:07.499','2038-01-19 03:14:07.499999','zzzz','abcdefgh','max', 'maxBLOB', '{"key": "maxJSON"}', 'Option1,Option2,Option3', b'111111', 'maxBIN', 'maxVARBIN', 'Value3'),
(0,0,0,0,0, 0, 0.00, 0.00000, 0.0000000000, 0.000000000000000000000000000000, 0, 0, false, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1901, '00:00:00', '00:00:00.000', '00:00:00.000000','1000-01-01','1000-01-01 00:00:00','1970-01-01 00:00:01','1970-01-01 00:00:01.000','1970-01-01 00:00:01.000000', '', '', '', '', '{}', '', b'', '', '', 'Value1');

insert into doris_test.all_types_multi_block select * from doris_test.all_types_nullable;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_nullable;

INSERT INTO doris_test.t_varchar (varchar_col) VALUES ('a');

INSERT INTO doris_test.t_varchar (varchar_col) VALUES ('中');

INSERT INTO doris_test.t_varchar (varchar_col) VALUES (REPEAT('a', 21844));

INSERT INTO doris_test.t_varchar (varchar_col) VALUES (REPEAT('中', 21844));

INSERT INTO doris_test.t_char (char_col) VALUES (REPEAT('a', 255));

INSERT INTO doris_test.t_char (char_col) VALUES (REPEAT('中', 255));

2 changes: 1 addition & 1 deletion fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ public abstract class BaseJdbcExecutor implements JdbcExecutor {
private static final TBinaryProtocol.Factory PROTOCOL_FACTORY = new TBinaryProtocol.Factory();
private HikariDataSource hikariDataSource = null;
private final byte[] hikariDataSourceLock = new byte[0];
private JdbcDataSourceConfig config;
private Connection conn = null;
protected JdbcDataSourceConfig config;
protected PreparedStatement preparedStatement = null;
protected Statement stmt = null;
protected ResultSet resultSet = null;
Expand Down
40 changes: 35 additions & 5 deletions ...e-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.doris.common.jni.vec.ColumnValueConverter;
import org.apache.doris.common.jni.vec.VectorTable;
import org.apache.doris.thrift.TJdbcOperation;
import org.apache.doris.thrift.TOdbcTableType;

import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.MoreExecutors;
Expand All @@ -34,6 +35,7 @@
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatterBuilder;
Expand Down Expand Up @@ -133,8 +135,18 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla
case VARCHAR:
case ARRAY:
return resultSet.getObject(columnIndex + 1, String.class);
case STRING:
return resultSet.getObject(columnIndex + 1);
case STRING: {
int jdbcType = resultSetMetaData.getColumnType(columnIndex + 1);
// If it is a time type in mysql, or use mysql driver connect mariadb
// We need to obtain the string directly to ensure that we can obtain a time other than 24 hours.
// If it is another database, such as oceanbase, this processing will lose precision information,
// so the original processing method will be maintained for the time being.
if (jdbcType == Types.TIME && config.getTableType() == TOdbcTableType.MYSQL) {
return resultSet.getString(columnIndex + 1);
} else {
return resultSet.getObject(columnIndex + 1);
}
}
default:
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
}
Expand Down Expand Up @@ -192,6 +204,9 @@ protected ColumnValueConverter getOutputConverter(ColumnType columnType, String
}

private Object convertArray(Object input, ColumnType columnType) {
if (input == null) {
return null;
}
java.lang.reflect.Type listType = getListTypeForArray(columnType);
if (columnType.getType() == Type.BOOLEAN) {
List<?> list = gson.fromJson((String) input, List.class);
Expand Down Expand Up @@ -228,10 +243,25 @@ private Object convertArray(Object input, ColumnType columnType) {
throw new IllegalArgumentException("Cannot convert " + item + " to LocalDateTime.");
}
}).collect(Collectors.toList());
} else if (columnType.getType() == Type.LARGEINT) {
List<?> list = gson.fromJson((String) input, List.class);
return list.stream().map(item -> {
if (item instanceof Number) {
return new BigDecimal(item.toString()).toBigInteger();
} else if (item instanceof String) {
return new BigDecimal((String) item).toBigInteger();
} else {
throw new IllegalArgumentException("Cannot convert " + item + " to BigInteger.");
}
}).collect(Collectors.toList());
} else if (columnType.getType() == Type.ARRAY) {
List<?> list = gson.fromJson((String) input, listType);
return list.stream()
.map(item -> convertArray(gson.toJson(item), columnType.getChildTypes().get(0)))
ColumnType childType = columnType.getChildTypes().get(0);
List<?> rawList = gson.fromJson((String) input, List.class);
return rawList.stream()
.map(element -> {
String elementJson = gson.toJson(element);
return convertArray(elementJson, childType);
})
.collect(Collectors.toList());
} else {
return gson.fromJson((String) input, listType);
Expand Down
23 changes: 12 additions & 11 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,8 +195,8 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
case "BIGINT":
return Type.LARGEINT;
case "DECIMAL": {
int precision = fieldSchema.getColumnSize().orElse(0) + 1;
int scale = fieldSchema.getDecimalDigits().orElse(0);
int precision = fieldSchema.requiredColumnSize() + 1;
int scale = fieldSchema.requiredDecimalDigits();
return createDecimalOrStringType(precision, scale);
}
case "DOUBLE":
Expand Down Expand Up @@ -233,7 +233,7 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
case "DATETIME": {
// mysql can support microsecond
// use columnSize to calculate the precision of timestamp/datetime
int columnSize = fieldSchema.getColumnSize().orElse(0);
int columnSize = fieldSchema.requiredColumnSize();
int scale = columnSize > 19 ? columnSize - 20 : 0;
if (scale > 6) {
scale = 6;
Expand All @@ -248,18 +248,18 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
case "DOUBLE":
return Type.DOUBLE;
case "DECIMAL": {
int precision = fieldSchema.getColumnSize().orElse(0);
int scale = fieldSchema.getDecimalDigits().orElse(0);
int precision = fieldSchema.requiredColumnSize();
int scale = fieldSchema.requiredDecimalDigits();
return createDecimalOrStringType(precision, scale);
}
case "CHAR":
ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
charType.setLength(fieldSchema.getColumnSize().orElse(0));
charType.setLength(fieldSchema.requiredColumnSize());
return charType;
case "VARCHAR":
return ScalarType.createVarcharType(fieldSchema.getColumnSize().orElse(0));
case "BIT":
if (fieldSchema.getColumnSize().orElse(0) == 1) {
if (fieldSchema.requiredColumnSize() == 1) {
return Type.BOOLEAN;
} else {
return ScalarType.createStringType();
Expand Down Expand Up @@ -360,8 +360,8 @@ private Type dorisTypeToDoris(JdbcFieldSchema fieldSchema) {
return Type.DOUBLE;
case "DECIMAL":
case "DECIMALV3": {
int precision = fieldSchema.getColumnSize().orElse(0);
int scale = fieldSchema.getDecimalDigits().orElse(0);
int precision = fieldSchema.requiredColumnSize();
int scale = fieldSchema.requiredDecimalDigits();
return createDecimalOrStringType(precision, scale);
}
case "DATE":
Expand All @@ -377,11 +377,12 @@ private Type dorisTypeToDoris(JdbcFieldSchema fieldSchema) {
return ScalarType.createDatetimeV2Type(scale);
}
case "CHAR":
case "CHARACTER":
ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
charType.setLength(fieldSchema.getColumnSize().orElse(0));
charType.setLength(fieldSchema.requiredColumnSize());
return charType;
case "VARCHAR":
return ScalarType.createVarcharType(fieldSchema.getColumnSize().orElse(0));
return ScalarType.createVarcharType(fieldSchema.requiredColumnSize());
case "STRING":
case "TEXT":
case "JSON":
Expand Down
Loading

0 comments on commit a150d16

Please sign in to comment.