Skip to content

Commit

Permalink
Revert "[fix](testcase) fix test case failure of insert null value in…
Browse files Browse the repository at this point in the history 
…to not null column (#20963)" (#23462)

* Revert "[fix](testcase) fix test case failure of insert null value into not null column (#20963)"

This reverts commit 55a6649.

Mannual Revert "fix in strict mode, return error for insert if datatype convert fails (#20378)"

This mannual reverts commit 1b94b63

* fix case failure
  • Loading branch information
@jacktengg
jacktengg authored Aug 25, 2023
1 parent 6d4f066 commit 1312c12
Show file tree
Hide file tree
Showing 7 changed files with 35 additions and 290 deletions.
4 changes: 0 additions & 4 deletions be/src/runtime/runtime_state.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -434,10 +434,6 @@ class RuntimeState {
: 0;
}

bool enable_insert_strict() const {
return _query_options.__isset.enable_insert_strict && _query_options.enable_insert_strict;
}

void emplace_local_state(int id, std::shared_ptr<doris::pipeline::PipelineXLocalState> state);

std::shared_ptr<doris::pipeline::PipelineXLocalState> get_local_state(int id);
Expand Down
77 changes: 27 additions & 50 deletions be/src/vec/functions/function_cast.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@
#include "vec/functions/function_helpers.h"
#include "vec/io/reader_buffer.h"
#include "vec/runtime/vdatetime_value.h"
#include "vec/utils/template_helpers.hpp"

class DateLUTImpl;

Expand Down Expand Up @@ -1369,58 +1368,36 @@ struct ConvertThroughParsing {
offsets = &col_from_string->get_offsets();
}

bool is_load = (context && context->state()->query_type() == TQueryType::type::LOAD);
bool is_strict_insert = (context && context->state()->enable_insert_strict());
size_t current_offset = 0;
auto status = std::visit(
[&](auto is_load_, auto is_strict_insert_) {
for (size_t i = 0; i < size; ++i) {
size_t next_offset = std::is_same_v<FromDataType, DataTypeString>
? (*offsets)[i]
: (current_offset + fixed_string_size);
size_t string_size = std::is_same_v<FromDataType, DataTypeString>
? next_offset - current_offset
: fixed_string_size;

ReadBuffer read_buffer(&(*chars)[current_offset], string_size);

bool parsed;
if constexpr (IsDataTypeDecimal<ToDataType>) {
parsed = try_parse_impl<ToDataType>(
vec_to[i], read_buffer, context->state()->timezone_obj(),
time_zone_cache, cache_lock, vec_to.get_scale());
} else if constexpr (IsDataTypeDateTimeV2<ToDataType>) {
auto type = check_and_get_data_type<DataTypeDateTimeV2>(
block.get_by_position(result).type.get());
parsed = try_parse_impl<ToDataType>(
vec_to[i], read_buffer, context->state()->timezone_obj(),
time_zone_cache, cache_lock, type->get_scale());
} else {
parsed = try_parse_impl<ToDataType, void*, FromDataType>(
vec_to[i], read_buffer, context->state()->timezone_obj(),
time_zone_cache, cache_lock);
}
(*vec_null_map_to)[i] = !parsed || !is_all_read(read_buffer);
if constexpr (is_load_ && is_strict_insert_) {
if (string_size != 0 && (*vec_null_map_to)[i]) {
return Status::InternalError(
"Invalid value {} in strict mode for function {}, source "
"column {}, from "
"type "
"{} to type {}",
std::string((char*)&(*chars)[current_offset], string_size),
Name::name, col_from->get_name(), FromDataType().get_name(),
ToDataType().get_name());
}
}
for (size_t i = 0; i < size; ++i) {
size_t next_offset = std::is_same_v<FromDataType, DataTypeString>
? (*offsets)[i]
: (current_offset + fixed_string_size);
size_t string_size = std::is_same_v<FromDataType, DataTypeString>
? next_offset - current_offset
: fixed_string_size;

current_offset = next_offset;
}
return Status::OK();
},
make_bool_variant(is_load), make_bool_variant(is_strict_insert));
ReadBuffer read_buffer(&(*chars)[current_offset], string_size);

RETURN_IF_ERROR(status);
bool parsed;
if constexpr (IsDataTypeDecimal<ToDataType>) {
parsed = try_parse_impl<ToDataType>(
vec_to[i], read_buffer, context->state()->timezone_obj(), time_zone_cache,
cache_lock, vec_to.get_scale());
} else if constexpr (IsDataTypeDateTimeV2<ToDataType>) {
auto type = check_and_get_data_type<DataTypeDateTimeV2>(
block.get_by_position(result).type.get());
parsed = try_parse_impl<ToDataType>(vec_to[i], read_buffer,
context->state()->timezone_obj(),
time_zone_cache, cache_lock, type->get_scale());
} else {
parsed = try_parse_impl<ToDataType, void*, FromDataType>(
vec_to[i], read_buffer, context->state()->timezone_obj(), time_zone_cache,
cache_lock);
}
(*vec_null_map_to)[i] = !parsed || !is_all_read(read_buffer);
current_offset = next_offset;
}

block.get_by_position(result).column =
ColumnNullable::create(std::move(col_to), std::move(col_null_map_to));
Expand Down
1 change: 0 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2199,7 +2199,6 @@ public TQueryOptions toThrift() {
tResult.setEnableParquetLazyMat(enableParquetLazyMat);
tResult.setEnableOrcLazyMat(enableOrcLazyMat);

tResult.setEnableInsertStrict(enableInsertStrict);
tResult.setTruncateCharOrVarcharColumns(truncateCharOrVarcharColumns);

return tResult;
Expand Down
11 changes: 0 additions & 11 deletions regression-test/data/insert_p0/insert_invalid.out
View file

This file was deleted.

215 changes: 0 additions & 215 deletions regression-test/suites/insert_p0/insert_invalid.groovy
View file

This file was deleted.

16 changes: 8 additions & 8 deletions regression-test/suites/query_p0/join/test_full_join_batch_size.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,25 @@
// under the License.

suite("test_full_join_batch_size", "query,p0") {
sql " drop table if exists test_left_join_batch_size_l; ";
sql " drop table if exists test_left_join_batch_size_r; ";
sql " drop table if exists test_full_join_batch_size_l; ";
sql " drop table if exists test_full_join_batch_size_r; ";
sql """
create table test_left_join_batch_size_l (
create table test_full_join_batch_size_l (
k1 int,
v1 int
) distributed by hash(k1) buckets 3
properties("replication_num" = "1");
"""
sql """
create table test_left_join_batch_size_r (
create table test_full_join_batch_size_r (
k1 int,
v1 int
) distributed by hash(k1) buckets 3
properties("replication_num" = "1");
"""

sql """ insert into test_left_join_batch_size_l values (1, 11), (1, 111), (1, 1111) """
sql """ insert into test_left_join_batch_size_r values (1, null), (1, 211), (1, 311), (1, 411) """
sql """ insert into test_full_join_batch_size_l values (1, 11), (1, 111), (1, 1111) """
sql """ insert into test_full_join_batch_size_r values (1, null), (1, 211), (1, 311), (1, 411) """

qt_sql1 """
select /*+SET_VAR(batch_size=3)*/
Expand All @@ -43,8 +43,8 @@ suite("test_full_join_batch_size", "query,p0") {
r.k1,
r.v1
from
test_left_join_batch_size_l l
full join test_left_join_batch_size_r r on (
test_full_join_batch_size_l l
full join test_full_join_batch_size_r r on (
r.v1 = 0
or r.v1 is null
)
Expand Down
1 change: 0 additions & 1 deletion regression-test/suites/statistics/test_ddl.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ suite("test_ddl") {
INSERT INTO `agg_all_for_analyze_test` VALUES (-24673,0,-127,-1939606877,-105278987563368327,-257119.385,'bA5rPeM244SovDhOOQ02CmXeM69uhJ8GSHtU','2022-09-28','2022-08-23 01:34:09','wrHimKN3w24QvUiplB9HFWdeCCeX0bQbbFima85zhb1kQ0s6lP6ctie2oGuKF',-4060736.642127,NULL,'22bCQDgO6A0FJB22Q9bASB8cHnYqHeKKGsa1e','-6225805734985728798'),(-22254,0,28,702265972,-6301108547516189202,-667430.114,'M0sReWtDXk7zt7AiDCzuqciSo0JuZzNI3Kez','2022-11-24','2022-11-29 22:52:56','gxsUl9OwrHYuy8Ih0A6XShMYk6IDizgtma',-745714046.0527,-29918.941,'cRUx9','-4950602999644698671'),(-10955,0,-80,1424977723,-3088617296403778878,-633882.260,'tIcTUsIHsoBEhKZmrDwgcwq7ZzWE1yiYmIXG','2023-06-05','2023-04-17 08:52:02','sXrN5V',1713648501.801151,7117.6777,'QZ','-4345679880674984959'),(-9534,0,41,1613325683,696349825249038738,-77371.865,'FowxRcxFOokDiTzhrpLtFxA8gNwhSPEGvX8K','2022-06-24','2023-03-18 00:05:22','dB4XnskAyxxC9BwBKFE4',-57809310.3712,8150.8745,'motLpNp3kxvsrg0NMKnEQNq3ubnytf3tlzL67SDfVWsBoCEmityAFkvILMNwSBZqAEEchQeJOiMD6RuQBZaSye5eJ3QQj8sW','-1869908446789747184'),(-4948,0,3,849130726,-1619674584315212902,501148.400,'sN3JTd702RLsH2s73Da17KgGFh5nGJd7KmUY','2023-01-27','2023-03-09 02:15:02','kq9yv9307P',228054999.363208,1346.6277,'UVY8iiuAyj5FSvLleT6XFJhDufz6sWbgeDEr73CT767whwbQ5mhz0mJjWM5zseYuJhJNEeII4sCL6fde4b8rxzddskxve3iphZR5HH2vdeu6v8ObVgctRa5AciUS','-7308802494907481422'),(1162,1,115,897860329,-386961924924498181,585623.859,'mcnLF37GnEgKATDe0wqAiI0FhW0UHVEV5Kpz','2023-04-13','2022-11-25 21:11:16','7oxMuzsNhlXn9gBD0EmEIV0EIN3vQ1X1mD15p2wk1r',NULL,11619.977,NULL,'-5444423998543013178'),(3421,0,-66,928609877,3425896045284731687,808967.590,'RiNrQpZ2oyQ3PfmR8qDwh9xOHHPQRKyGrZGq','2022-12-29','2022-08-12 10:24:09','HplLUZyXTecpaooUFMZXu4',1859302923.9809761,NULL,'KxjmOGlRggjv9TxZam013MNNcNIxquAQC5oyeULxhU5h2pEee3AGRPAKcatVOOa34cC1kst0FxLHp4cekBHIFRTkNoahB7uKuff2MxLx2f9dqo','5073471243162286568'),(3440,0,-2,-1810502635,-1152153212814582370,354673.534,'ELI5AU4fBUGzKbmydHnXN5y9atfsV6YyzZ6N','2022-09-17','2023-06-04 03:19:23','',-1087026379.7795939,-9048.9424,'e9XpWT5zBbGxFSv5H49qwodgS0fDm3O1Q',NULL),(4195,0,-31,-1935556221,-1664199322117981477,256018.995,'iuBjWHEfvQIU2Dd8NXaRE6BZvBEb4Zf6zz6C','2023-02-08','2023-01-01 11:10:33',NULL,-1773292012.0985451,-7919.2183,'zZukUAgmReZ5DAsv05cKRW3h2S4Rpi4WelwJbndfen7PALAuFIW8FGKwpGExwrLZZ','-7204803994761149714'),(5187,0,13,1356487954,-6110284427694045409,644829.776,'GvdFXhr1YmiUAZgnTsSnwZayJ1Ps6IEkmSSq',NULL,'2022-07-15 16:18:27','JLhxmYkg2hcTcFa308inIoweUXVmy1jm0JvYq8rKi01EiJXqrrUXC3c5l',1726350878.7230589,31705.006,'jB9HvvMURh8lqSj4ELI8UKzUqTcBTQlnf06o7zL9UXxT6k5cHsfKqbH7p5UfYw5buogu3Ikq398rtoWLidqCA6DjhVD1AS','7212038235544903867'),(8299,1,-72,368891628,1695757292890124771,959687.899,'GdAN6PZbZPFOwuCWAyteDm1BXzBjGRBniKrO','2023-01-28','2022-09-26 01:15:17','YpVBcqinEvUsUgN0pHtFjazbskHEvuQkNoSxS',7273321.938684,-20596.896,'l','8023400916908780684'),(10070,0,-86,-56997455,-7220854438302716090,435174.259,NULL,'2023-02-14','2022-10-09 15:21:22','3W8Tmdr0elTQ98yhb3dvgKxo6Ybme6qgBbXGuez',1428473753.898315,6188.2759,'Yt6P4qcpoWiEA29mJxe2XEZjyzIJZHa7RC9qYBcqYbPBEKMqfSlMbxDhdubtUB9Du94OY7ixFpL46nhT4sIw9U5Z67taJAJHVMs23hgFoExHMKRGrg1w7','4076448588744994031'),(13937,0,-61,1764726920,-1858610362502972801,-878642.130,'6jra9ar8wnccLLDYxPXx4RgDMTQqX7knbgkV','2022-10-23','2023-01-13 09:58:22','f4wy0hdYsHwYnUrWsB6ynxgWxEroWJkAeK0Lixo7JxKaBttkFqxbDnV6vBf',2072443906.6214759,-4308.2153,'J',NULL),(18207,1,NULL,328023897,8508197801117683670,-736954.593,'11lyC6NUWqI4qpvHNP9L9MYRfJ3FgAlZnXMR','2022-09-07','2023-03-02 19:43:47','EGQBiS5H1ac4a4X8aIgtH4WLBVYZiQhoajALPOM4eXKtjJroQhvs0PD',-71095753.073478,-8648.1748,'y8abEvZq38Wf','-7039513382696657226'),(30365,1,32,1819863984,-4478313006953134506,983950.742,'tMkZzSPr5N9jl86TzIgglP4TAfdje6PDmVrk','2022-07-05','2023-01-31 16:17:53','8Shy8x7dXSd9blHjLIbdXzPMZUZFNWMnoMZOTRyFMhEekaW5',549029749.592098,-28284.465,'7QeFUV0KsN87U64iREOEYg4CkOj5qSMDott58plWWtxKByghk6VkKh0HthS9OrUmR4a1LiZT8fne1fbgSVcHNnX2jab0vgpY7ZV','6058055427499915015')
"""

sql "set enable_insert_strict=false"
sql """
INSERT INTO __internal_schema.column_statistics SELECT id, catalog_id, db_id, tbl_id, idx_id, col_id,
part_id, row_count, ndv, null_count, min, max, data_size, update_time
Expand Down

0 comments on commit 1312c12

Please sign in to comment.