diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 3f1126c4f39db2..add158dafa7cc6 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -434,10 +434,6 @@ class RuntimeState { : 0; } - bool enable_insert_strict() const { - return _query_options.__isset.enable_insert_strict && _query_options.enable_insert_strict; - } - void emplace_local_state(int id, std::shared_ptr state); std::shared_ptr get_local_state(int id); diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h index 4d9c6b03111654..b64d5ba9fdc441 100644 --- a/be/src/vec/functions/function_cast.h +++ b/be/src/vec/functions/function_cast.h @@ -87,7 +87,6 @@ #include "vec/functions/function_helpers.h" #include "vec/io/reader_buffer.h" #include "vec/runtime/vdatetime_value.h" -#include "vec/utils/template_helpers.hpp" class DateLUTImpl; @@ -1369,58 +1368,36 @@ struct ConvertThroughParsing { offsets = &col_from_string->get_offsets(); } - bool is_load = (context && context->state()->query_type() == TQueryType::type::LOAD); - bool is_strict_insert = (context && context->state()->enable_insert_strict()); size_t current_offset = 0; - auto status = std::visit( - [&](auto is_load_, auto is_strict_insert_) { - for (size_t i = 0; i < size; ++i) { - size_t next_offset = std::is_same_v - ? (*offsets)[i] - : (current_offset + fixed_string_size); - size_t string_size = std::is_same_v - ? next_offset - current_offset - : fixed_string_size; - - ReadBuffer read_buffer(&(*chars)[current_offset], string_size); - - bool parsed; - if constexpr (IsDataTypeDecimal) { - parsed = try_parse_impl( - vec_to[i], read_buffer, context->state()->timezone_obj(), - time_zone_cache, cache_lock, vec_to.get_scale()); - } else if constexpr (IsDataTypeDateTimeV2) { - auto type = check_and_get_data_type( - block.get_by_position(result).type.get()); - parsed = try_parse_impl( - vec_to[i], read_buffer, context->state()->timezone_obj(), - time_zone_cache, cache_lock, type->get_scale()); - } else { - parsed = try_parse_impl( - vec_to[i], read_buffer, context->state()->timezone_obj(), - time_zone_cache, cache_lock); - } - (*vec_null_map_to)[i] = !parsed || !is_all_read(read_buffer); - if constexpr (is_load_ && is_strict_insert_) { - if (string_size != 0 && (*vec_null_map_to)[i]) { - return Status::InternalError( - "Invalid value {} in strict mode for function {}, source " - "column {}, from " - "type " - "{} to type {}", - std::string((char*)&(*chars)[current_offset], string_size), - Name::name, col_from->get_name(), FromDataType().get_name(), - ToDataType().get_name()); - } - } + for (size_t i = 0; i < size; ++i) { + size_t next_offset = std::is_same_v + ? (*offsets)[i] + : (current_offset + fixed_string_size); + size_t string_size = std::is_same_v + ? next_offset - current_offset + : fixed_string_size; - current_offset = next_offset; - } - return Status::OK(); - }, - make_bool_variant(is_load), make_bool_variant(is_strict_insert)); + ReadBuffer read_buffer(&(*chars)[current_offset], string_size); - RETURN_IF_ERROR(status); + bool parsed; + if constexpr (IsDataTypeDecimal) { + parsed = try_parse_impl( + vec_to[i], read_buffer, context->state()->timezone_obj(), time_zone_cache, + cache_lock, vec_to.get_scale()); + } else if constexpr (IsDataTypeDateTimeV2) { + auto type = check_and_get_data_type( + block.get_by_position(result).type.get()); + parsed = try_parse_impl(vec_to[i], read_buffer, + context->state()->timezone_obj(), + time_zone_cache, cache_lock, type->get_scale()); + } else { + parsed = try_parse_impl( + vec_to[i], read_buffer, context->state()->timezone_obj(), time_zone_cache, + cache_lock); + } + (*vec_null_map_to)[i] = !parsed || !is_all_read(read_buffer); + current_offset = next_offset; + } block.get_by_position(result).column = ColumnNullable::create(std::move(col_to), std::move(col_null_map_to)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index b3662f3577feb3..a61b49781a8e57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2199,7 +2199,6 @@ public TQueryOptions toThrift() { tResult.setEnableParquetLazyMat(enableParquetLazyMat); tResult.setEnableOrcLazyMat(enableOrcLazyMat); - tResult.setEnableInsertStrict(enableInsertStrict); tResult.setTruncateCharOrVarcharColumns(truncateCharOrVarcharColumns); return tResult; diff --git a/regression-test/data/insert_p0/insert_invalid.out b/regression-test/data/insert_p0/insert_invalid.out deleted file mode 100644 index 7e268e6c7e7258..00000000000000 --- a/regression-test/data/insert_p0/insert_invalid.out +++ /dev/null @@ -1,11 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !select_inserted0 -- -12345678908876643 a -1234567890887664643 b - --- !select_inserted1 -- -12345678908876643 a -1234567890887664643 b - --- !select_inserted2 -- - diff --git a/regression-test/suites/insert_p0/insert_invalid.groovy b/regression-test/suites/insert_p0/insert_invalid.groovy deleted file mode 100644 index 6b3e99ab35c54d..00000000000000 --- a/regression-test/suites/insert_p0/insert_invalid.groovy +++ /dev/null @@ -1,215 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases -// and modified by Doris. -suite("insert_invalid") { - sql """ DROP TABLE IF EXISTS datatype_invalid; """ - sql """ - CREATE TABLE `datatype_invalid` (`timea` bigint NOT NULL, `creatr` varchar(30) NULL) - UNIQUE KEY(`timea`) - DISTRIBUTED BY HASH(`timea`) BUCKETS 1 - PROPERTIES ("replication_num" = "1"); - """ - - // strict insert - sql """ set enable_insert_strict=true; """ - - // test insert select: out of range value - sql """ DROP TABLE IF EXISTS datatype_invalid_base; """ - sql """ - CREATE TABLE `datatype_invalid_base` ( - `timea` varchar(30) NULL, - `creatr` varchar(30) NULL - ) UNIQUE KEY(`timea`) - DISTRIBUTED BY HASH(`timea`) BUCKETS 1 - PROPERTIES ("replication_num" = "1"); - """ - - sql """ - insert into - datatype_invalid_base - values - ("12345678908876643", "a"), - ("1234567890887664643", "b"), - ("123456789088766445456", "c"); - """ - - test { - sql """ insert into datatype_invalid select * from datatype_invalid_base;""" - exception "Invalid value" - } - - // test insert select: invalid value - sql """ DROP TABLE IF EXISTS datatype_invalid_base; """ - sql """ - CREATE TABLE `datatype_invalid_base` ( - `timea` varchar(30) NULL, - `creatr` varchar(30) NULL - ) UNIQUE KEY(`timea`) - DISTRIBUTED BY HASH(`timea`) BUCKETS 1 - PROPERTIES ("replication_num" = "1"); - """ - - sql """ - insert into - datatype_invalid_base - values - ("a", "a"); - """ - test { - sql """ insert into datatype_invalid select * from datatype_invalid_base;""" - exception "Invalid value" - } - - // test insert select: invalid value - sql """ DROP TABLE IF EXISTS datatype_invalid_base; """ - sql """ - CREATE TABLE `datatype_invalid_base` ( - `timea` varchar(30) NULL, - `creatr` varchar(30) NULL - ) UNIQUE KEY(`timea`) - DISTRIBUTED BY HASH(`timea`) BUCKETS 1 - PROPERTIES ("replication_num" = "1"); - """ - - sql """ - insert into - datatype_invalid_base - values - (" ", "a"); - """ - test { - sql """ insert into datatype_invalid select * from datatype_invalid_base;""" - exception "Invalid value" - } - - // test insert select: null into not nullable - sql """ DROP TABLE IF EXISTS datatype_invalid_base; """ - sql """ - CREATE TABLE `datatype_invalid_base` ( - `timea` varchar(30) NULL, - `creatr` varchar(30) NULL - ) UNIQUE KEY(`timea`) - DISTRIBUTED BY HASH(`timea`) BUCKETS 1 - PROPERTIES ("replication_num" = "1"); - """ - - sql """ - insert into - datatype_invalid_base - values - (null, "a"); - """ - test { - sql """ insert into datatype_invalid select * from datatype_invalid_base;""" - exception "Insert has filtered data in strict mode" - } - - // test insert - test { - sql """ insert into datatype_invalid values("a", "a");""" - exception "Invalid number format" - } - test { - sql """ insert into datatype_invalid values(" ", "a");""" - exception "Invalid number format" - } - test { - sql """ insert into datatype_invalid values(123456789088766445456, "a");""" - exception "Number out of range" - } - test { - sql """ insert into datatype_invalid values(null, "a");""" - exception "Insert has filtered data in strict mode" - } - - sql """ DROP TABLE IF EXISTS datatype_invalid; """ - sql """ - CREATE TABLE `datatype_invalid` (`timea` datetime NOT NULL, `creatr` varchar(30) NULL) - UNIQUE KEY(`timea`) - DISTRIBUTED BY HASH(`timea`) BUCKETS 1 - PROPERTIES ("replication_num" = "1"); - """ - test { - sql """ insert into datatype_invalid values ('2022-02-29', 'a'); """ - exception "Invalid value" - } - - sql """ set enable_insert_strict=false; """ - sql """ DROP TABLE IF EXISTS datatype_invalid; """ - sql """ - CREATE TABLE `datatype_invalid` (`timea` bigint NOT NULL, `creatr` varchar(30) NULL) - UNIQUE KEY(`timea`) - DISTRIBUTED BY HASH(`timea`) BUCKETS 1 - PROPERTIES ("replication_num" = "1"); - """ - - // non strict insert into select - sql """ DROP TABLE IF EXISTS datatype_invalid_base; """ - sql """ - CREATE TABLE `datatype_invalid_base` ( - `timea` varchar(30) NULL, - `creatr` varchar(30) NULL - ) UNIQUE KEY(`timea`) - DISTRIBUTED BY HASH(`timea`) BUCKETS 1 - PROPERTIES ("replication_num" = "1"); - """ - sql """ - insert into - datatype_invalid_base - values - ("a", "a"), - (" ", "a"), - ("12345678908876643", "a"), - ("1234567890887664643", "b"), - ("123456789088766445456", "c"); - """ - sql """ insert into datatype_invalid select * from datatype_invalid_base;""" - - qt_select_inserted0 """ select * from datatype_invalid order by timea """ - - sql """ DROP TABLE IF EXISTS datatype_invalid_base; """ - sql """ - CREATE TABLE `datatype_invalid_base` ( - `timea` varchar(30) NULL, - `creatr` varchar(30) NULL - ) UNIQUE KEY(`timea`) - DISTRIBUTED BY HASH(`timea`) BUCKETS 1 - PROPERTIES ("replication_num" = "1"); - """ - sql """ - insert into - datatype_invalid_base - values - (null, "a"); - """ - sql """ insert into datatype_invalid select * from datatype_invalid_base;""" - - qt_select_inserted1 """ select * from datatype_invalid order by timea """ - - sql """ DROP TABLE IF EXISTS datatype_invalid; """ - sql """ - CREATE TABLE `datatype_invalid` (`timea` datetime NOT NULL, `creatr` varchar(30) NULL) - UNIQUE KEY(`timea`) - DISTRIBUTED BY HASH(`timea`) BUCKETS 1 - PROPERTIES ("replication_num" = "1"); - """ - sql """ insert into datatype_invalid values ('2022-02-29', 'a'); """ - qt_select_inserted2 """ select * from datatype_invalid order by timea """ -} diff --git a/regression-test/suites/query_p0/join/test_full_join_batch_size.groovy b/regression-test/suites/query_p0/join/test_full_join_batch_size.groovy index 6ea3923fa28b45..7e7a70a8840ff9 100644 --- a/regression-test/suites/query_p0/join/test_full_join_batch_size.groovy +++ b/regression-test/suites/query_p0/join/test_full_join_batch_size.groovy @@ -16,25 +16,25 @@ // under the License. suite("test_full_join_batch_size", "query,p0") { - sql " drop table if exists test_left_join_batch_size_l; "; - sql " drop table if exists test_left_join_batch_size_r; "; + sql " drop table if exists test_full_join_batch_size_l; "; + sql " drop table if exists test_full_join_batch_size_r; "; sql """ - create table test_left_join_batch_size_l ( + create table test_full_join_batch_size_l ( k1 int, v1 int ) distributed by hash(k1) buckets 3 properties("replication_num" = "1"); """ sql """ - create table test_left_join_batch_size_r ( + create table test_full_join_batch_size_r ( k1 int, v1 int ) distributed by hash(k1) buckets 3 properties("replication_num" = "1"); """ - sql """ insert into test_left_join_batch_size_l values (1, 11), (1, 111), (1, 1111) """ - sql """ insert into test_left_join_batch_size_r values (1, null), (1, 211), (1, 311), (1, 411) """ + sql """ insert into test_full_join_batch_size_l values (1, 11), (1, 111), (1, 1111) """ + sql """ insert into test_full_join_batch_size_r values (1, null), (1, 211), (1, 311), (1, 411) """ qt_sql1 """ select /*+SET_VAR(batch_size=3)*/ @@ -43,8 +43,8 @@ suite("test_full_join_batch_size", "query,p0") { r.k1, r.v1 from - test_left_join_batch_size_l l - full join test_left_join_batch_size_r r on ( + test_full_join_batch_size_l l + full join test_full_join_batch_size_r r on ( r.v1 = 0 or r.v1 is null ) diff --git a/regression-test/suites/statistics/test_ddl.groovy b/regression-test/suites/statistics/test_ddl.groovy index f5a67d6d559074..06e1c72e16428c 100644 --- a/regression-test/suites/statistics/test_ddl.groovy +++ b/regression-test/suites/statistics/test_ddl.groovy @@ -44,7 +44,6 @@ suite("test_ddl") { INSERT INTO `agg_all_for_analyze_test` VALUES (-24673,0,-127,-1939606877,-105278987563368327,-257119.385,'bA5rPeM244SovDhOOQ02CmXeM69uhJ8GSHtU','2022-09-28','2022-08-23 01:34:09','wrHimKN3w24QvUiplB9HFWdeCCeX0bQbbFima85zhb1kQ0s6lP6ctie2oGuKF',-4060736.642127,NULL,'22bCQDgO6A0FJB22Q9bASB8cHnYqHeKKGsa1e','-6225805734985728798'),(-22254,0,28,702265972,-6301108547516189202,-667430.114,'M0sReWtDXk7zt7AiDCzuqciSo0JuZzNI3Kez','2022-11-24','2022-11-29 22:52:56','gxsUl9OwrHYuy8Ih0A6XShMYk6IDizgtma',-745714046.0527,-29918.941,'cRUx9','-4950602999644698671'),(-10955,0,-80,1424977723,-3088617296403778878,-633882.260,'tIcTUsIHsoBEhKZmrDwgcwq7ZzWE1yiYmIXG','2023-06-05','2023-04-17 08:52:02','sXrN5V',1713648501.801151,7117.6777,'QZ','-4345679880674984959'),(-9534,0,41,1613325683,696349825249038738,-77371.865,'FowxRcxFOokDiTzhrpLtFxA8gNwhSPEGvX8K','2022-06-24','2023-03-18 00:05:22','dB4XnskAyxxC9BwBKFE4',-57809310.3712,8150.8745,'motLpNp3kxvsrg0NMKnEQNq3ubnytf3tlzL67SDfVWsBoCEmityAFkvILMNwSBZqAEEchQeJOiMD6RuQBZaSye5eJ3QQj8sW','-1869908446789747184'),(-4948,0,3,849130726,-1619674584315212902,501148.400,'sN3JTd702RLsH2s73Da17KgGFh5nGJd7KmUY','2023-01-27','2023-03-09 02:15:02','kq9yv9307P',228054999.363208,1346.6277,'UVY8iiuAyj5FSvLleT6XFJhDufz6sWbgeDEr73CT767whwbQ5mhz0mJjWM5zseYuJhJNEeII4sCL6fde4b8rxzddskxve3iphZR5HH2vdeu6v8ObVgctRa5AciUS','-7308802494907481422'),(1162,1,115,897860329,-386961924924498181,585623.859,'mcnLF37GnEgKATDe0wqAiI0FhW0UHVEV5Kpz','2023-04-13','2022-11-25 21:11:16','7oxMuzsNhlXn9gBD0EmEIV0EIN3vQ1X1mD15p2wk1r',NULL,11619.977,NULL,'-5444423998543013178'),(3421,0,-66,928609877,3425896045284731687,808967.590,'RiNrQpZ2oyQ3PfmR8qDwh9xOHHPQRKyGrZGq','2022-12-29','2022-08-12 10:24:09','HplLUZyXTecpaooUFMZXu4',1859302923.9809761,NULL,'KxjmOGlRggjv9TxZam013MNNcNIxquAQC5oyeULxhU5h2pEee3AGRPAKcatVOOa34cC1kst0FxLHp4cekBHIFRTkNoahB7uKuff2MxLx2f9dqo','5073471243162286568'),(3440,0,-2,-1810502635,-1152153212814582370,354673.534,'ELI5AU4fBUGzKbmydHnXN5y9atfsV6YyzZ6N','2022-09-17','2023-06-04 03:19:23','',-1087026379.7795939,-9048.9424,'e9XpWT5zBbGxFSv5H49qwodgS0fDm3O1Q',NULL),(4195,0,-31,-1935556221,-1664199322117981477,256018.995,'iuBjWHEfvQIU2Dd8NXaRE6BZvBEb4Zf6zz6C','2023-02-08','2023-01-01 11:10:33',NULL,-1773292012.0985451,-7919.2183,'zZukUAgmReZ5DAsv05cKRW3h2S4Rpi4WelwJbndfen7PALAuFIW8FGKwpGExwrLZZ','-7204803994761149714'),(5187,0,13,1356487954,-6110284427694045409,644829.776,'GvdFXhr1YmiUAZgnTsSnwZayJ1Ps6IEkmSSq',NULL,'2022-07-15 16:18:27','JLhxmYkg2hcTcFa308inIoweUXVmy1jm0JvYq8rKi01EiJXqrrUXC3c5l',1726350878.7230589,31705.006,'jB9HvvMURh8lqSj4ELI8UKzUqTcBTQlnf06o7zL9UXxT6k5cHsfKqbH7p5UfYw5buogu3Ikq398rtoWLidqCA6DjhVD1AS','7212038235544903867'),(8299,1,-72,368891628,1695757292890124771,959687.899,'GdAN6PZbZPFOwuCWAyteDm1BXzBjGRBniKrO','2023-01-28','2022-09-26 01:15:17','YpVBcqinEvUsUgN0pHtFjazbskHEvuQkNoSxS',7273321.938684,-20596.896,'l','8023400916908780684'),(10070,0,-86,-56997455,-7220854438302716090,435174.259,NULL,'2023-02-14','2022-10-09 15:21:22','3W8Tmdr0elTQ98yhb3dvgKxo6Ybme6qgBbXGuez',1428473753.898315,6188.2759,'Yt6P4qcpoWiEA29mJxe2XEZjyzIJZHa7RC9qYBcqYbPBEKMqfSlMbxDhdubtUB9Du94OY7ixFpL46nhT4sIw9U5Z67taJAJHVMs23hgFoExHMKRGrg1w7','4076448588744994031'),(13937,0,-61,1764726920,-1858610362502972801,-878642.130,'6jra9ar8wnccLLDYxPXx4RgDMTQqX7knbgkV','2022-10-23','2023-01-13 09:58:22','f4wy0hdYsHwYnUrWsB6ynxgWxEroWJkAeK0Lixo7JxKaBttkFqxbDnV6vBf',2072443906.6214759,-4308.2153,'J',NULL),(18207,1,NULL,328023897,8508197801117683670,-736954.593,'11lyC6NUWqI4qpvHNP9L9MYRfJ3FgAlZnXMR','2022-09-07','2023-03-02 19:43:47','EGQBiS5H1ac4a4X8aIgtH4WLBVYZiQhoajALPOM4eXKtjJroQhvs0PD',-71095753.073478,-8648.1748,'y8abEvZq38Wf','-7039513382696657226'),(30365,1,32,1819863984,-4478313006953134506,983950.742,'tMkZzSPr5N9jl86TzIgglP4TAfdje6PDmVrk','2022-07-05','2023-01-31 16:17:53','8Shy8x7dXSd9blHjLIbdXzPMZUZFNWMnoMZOTRyFMhEekaW5',549029749.592098,-28284.465,'7QeFUV0KsN87U64iREOEYg4CkOj5qSMDott58plWWtxKByghk6VkKh0HthS9OrUmR4a1LiZT8fne1fbgSVcHNnX2jab0vgpY7ZV','6058055427499915015') """ - sql "set enable_insert_strict=false" sql """ INSERT INTO __internal_schema.column_statistics SELECT id, catalog_id, db_id, tbl_id, idx_id, col_id, part_id, row_count, ndv, null_count, min, max, data_size, update_time