From 72f1e5771f2ab070257982e81551fda8b0f53cd1 Mon Sep 17 00:00:00 2001 From: xzhangxian1008 Date: Thu, 17 Nov 2022 16:09:57 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20error=20data=20input=20for=20date(CAST(v?= =?UTF-8?q?alue=20AS=20DATETIME))=20causing=20high=20=E2=80=A6=20(#6321)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit close pingcap/tiflash#5097 --- dbms/src/Common/MyTime.cpp | 60 ++++--- dbms/src/Common/MyTime.h | 11 +- dbms/src/Common/tests/gtest_mytime.cpp | 2 +- dbms/src/Functions/FunctionsTiDBConversion.h | 151 +++++++++--------- .../Functions/tests/gtest_tidb_conversion.cpp | 33 ++-- tests/fullstack-test/expr/return_warning.test | 15 ++ 6 files changed, 151 insertions(+), 121 deletions(-) diff --git a/dbms/src/Common/MyTime.cpp b/dbms/src/Common/MyTime.cpp index 279462c9875..c59d93c5f6c 100644 --- a/dbms/src/Common/MyTime.cpp +++ b/dbms/src/Common/MyTime.cpp @@ -30,14 +30,23 @@ int32_t adjustYear(int32_t year) return year; } -void scanTimeArgs(const std::vector & seps, std::initializer_list && list) +bool scanTimeArgs(const std::vector & seps, std::initializer_list && list) { int i = 0; - for (auto * ptr : list) + try { - *ptr = std::stoi(seps[i]); - i++; + for (auto * ptr : list) + { + *ptr = std::stoi(seps[i]); + i++; + } } + catch (std::exception & e) + { + return false; + } + + return true; } // find index of fractional point. @@ -631,7 +640,7 @@ std::pair parseMyDateTimeAndJudgeIsDate(const String & str, int8_t } default: { - throw TiFlashException("Wrong datetime format: " + str, Errors::Types::WrongValue); + return {Field(), is_date}; } } if (l == 5 || l == 6 || l == 8) @@ -680,40 +689,44 @@ std::pair parseMyDateTimeAndJudgeIsDate(const String & str, int8_t } if (truncated_or_incorrect) { - throw TiFlashException("Datetime truncated: " + str, Errors::Types::Truncated); + return {Field(), is_date}; } break; } case 3: { // YYYY-MM-DD - scanTimeArgs(seps, {&year, &month, &day}); + if (!scanTimeArgs(seps, {&year, &month, &day})) + return {Field(), is_date}; is_date = true; break; } case 4: { // YYYY-MM-DD HH - scanTimeArgs(seps, {&year, &month, &day, &hour}); + if (!scanTimeArgs(seps, {&year, &month, &day, &hour})) + return {Field(), is_date}; break; } case 5: { // YYYY-MM-DD HH-MM - scanTimeArgs(seps, {&year, &month, &day, &hour, &minute}); + if (!scanTimeArgs(seps, {&year, &month, &day, &hour, &minute})) + return {Field(), is_date}; break; } case 6: { // We don't have fractional seconds part. // YYYY-MM-DD HH-MM-SS - scanTimeArgs(seps, {&year, &month, &day, &hour, &minute, &second}); + if (!scanTimeArgs(seps, {&year, &month, &day, &hour, &minute, &second})) + return {Field(), is_date}; hhmmss = true; break; } default: { - throw Exception("Wrong datetime format"); + return {Field(), is_date}; } } @@ -770,7 +783,7 @@ std::pair parseMyDateTimeAndJudgeIsDate(const String & str, int8_t if (needCheckTimeValid && !checkTimeValid(year, month, day, hour, minute, second)) { - throw Exception("Wrong datetime format"); + return {Field(), is_date}; } MyDateTime result(year, month, day, hour, minute, second, micro_second); @@ -779,7 +792,7 @@ std::pair parseMyDateTimeAndJudgeIsDate(const String & str, int8_t { if (!hhmmss) { - throw TiFlashException("Invalid datetime value: " + str, Errors::Types::WrongValue); + return {Field(), is_date}; } if (!tz_hour.empty()) { @@ -793,7 +806,7 @@ std::pair parseMyDateTimeAndJudgeIsDate(const String & str, int8_t if (delta_hour > 14 || delta_minute > 59 || (delta_hour == 14 && delta_minute != 0) || (tz_sign == "-" && delta_hour == 0 && delta_minute == 0)) { - throw TiFlashException("Invalid datetime value: " + str, Errors::Types::WrongValue); + return {Field(), is_date}; } // by default, if the temporal string literal does not contain timezone information, it will be in the timezone // specified by the time_zone system variable. However, if the timezone is specified in the string literal, we @@ -1025,21 +1038,19 @@ size_t maxFormattedDateTimeStringLength(const String & format) return std::max(result, 1); } -void MyTimeBase::check(bool allow_zero_in_date, bool allow_invalid_date) const +bool MyTimeBase::isValid(bool allow_zero_in_date, bool allow_invalid_date) const { if (!(year == 0 && month == 0 && day == 0)) { if (!allow_zero_in_date && (month == 0 || day == 0)) { - throw TiFlashException( - fmt::format("Incorrect datetime value: {0:04d}-{1:02d}-{2:02d}", year, month, day), - Errors::Types::WrongValue); + return false; } } if (year >= 9999 || month > 12) { - throw TiFlashException("Incorrect time value", Errors::Types::WrongValue); + return false; } UInt8 max_day = 31; @@ -1057,23 +1068,22 @@ void MyTimeBase::check(bool allow_zero_in_date, bool allow_invalid_date) const } if (day > max_day) { - throw TiFlashException( - fmt::format("Incorrect datetime value: {0:04d}-{1:02d}-{2:02d}", year, month, day), - Errors::Types::WrongValue); + return false; } if (hour < 0 || hour >= 24) { - throw TiFlashException("Incorrect datetime value", Errors::Types::WrongValue); + return false; } if (minute >= 60) { - throw TiFlashException("Incorrect datetime value", Errors::Types::WrongValue); + return false; } if (second >= 60) { - throw TiFlashException("Incorrect datetime value", Errors::Types::WrongValue); + return false; } + return true; } bool toCoreTimeChecked(const UInt64 & year, const UInt64 & month, const UInt64 & day, const UInt64 & hour, const UInt64 & minute, const UInt64 & second, const UInt64 & microsecond, MyDateTime & result) diff --git a/dbms/src/Common/MyTime.h b/dbms/src/Common/MyTime.h index 85aa4f20a5a..0204a9837d3 100644 --- a/dbms/src/Common/MyTime.h +++ b/dbms/src/Common/MyTime.h @@ -91,8 +91,8 @@ struct MyTimeBase std::tuple calcWeek(UInt32 mode) const; // Check validity of time under specified SQL_MODE. - // May throw exception. - void check(bool allow_zero_in_date, bool allow_invalid_date) const; + // return false if time is invalid + bool isValid(bool allow_zero_in_date, bool allow_invalid_date) const; }; struct MyDateTime : public MyTimeBase @@ -159,8 +159,11 @@ struct MyDateTimeParser std::vector parsers; }; -Field parseMyDateTime(const String & str, int8_t fsp = 6, bool needCheckTimeValid = false); -std::pair parseMyDateTimeAndJudgeIsDate(const String & str, int8_t fsp = 6, bool needCheckTimeValid = false); +static int8_t default_fsp = 6; +static bool default_need_check_time_valid = false; + +Field parseMyDateTime(const String & str, int8_t fsp = default_fsp, bool need_check_time_valid = default_need_check_time_valid); +std::pair parseMyDateTimeAndJudgeIsDate(const String & str, int8_t fsp = default_fsp, bool need_check_time_valid = default_need_check_time_valid); void convertTimeZone(UInt64 from_time, UInt64 & to_time, const DateLUTImpl & time_zone_from, const DateLUTImpl & time_zone_to, bool throw_exception = false); diff --git a/dbms/src/Common/tests/gtest_mytime.cpp b/dbms/src/Common/tests/gtest_mytime.cpp index 679a2752125..15fb3c198d5 100644 --- a/dbms/src/Common/tests/gtest_mytime.cpp +++ b/dbms/src/Common/tests/gtest_mytime.cpp @@ -62,7 +62,7 @@ class TestMyTime : public testing::Test if (expect_error) { MyDateTime datetime(0, 0, 0, 0, 0, 0, 0); - EXPECT_THROW({ numberToDateTime(input, datetime, ctx); }, TiFlashException) << "Original time number: " << input; + EXPECT_TRUE(numberToDateTime(input, datetime, ctx)); return; } diff --git a/dbms/src/Functions/FunctionsTiDBConversion.h b/dbms/src/Functions/FunctionsTiDBConversion.h index fde9cbca1d4..96ceecaa8a5 100644 --- a/dbms/src/Functions/FunctionsTiDBConversion.h +++ b/dbms/src/Functions/FunctionsTiDBConversion.h @@ -1280,27 +1280,30 @@ struct TiDBConvertToTime size_t string_size = next_offset - current_offset - 1; StringRef string_ref(&(*chars)[current_offset], string_size); String string_value = string_ref.toString(); - try - { - Field packed_uint_value = parseMyDateTime(string_value, to_fsp); - UInt64 packed_uint = packed_uint_value.template safeGet(); - MyDateTime datetime(packed_uint); - if constexpr (std::is_same_v) - { - MyDate date(datetime.year, datetime.month, datetime.day); - vec_to[i] = date.toPackedUInt(); - } - else - { - vec_to[i] = packed_uint; - } - } - catch (const Exception &) + + Field packed_uint_value = parseMyDateTime(string_value, to_fsp); + + if (packed_uint_value.isNull()) { // Fill NULL if cannot parse (*vec_null_map_to)[i] = 1; - context.getDAGContext()->handleInvalidTime("Invalid time value: '" + string_value + "'", Errors::Types::WrongValue); + vec_to[i] = 0; + current_offset = next_offset; + continue; + } + + UInt64 packed_uint = packed_uint_value.template safeGet(); + MyDateTime datetime(packed_uint); + if constexpr (std::is_same_v) + { + MyDate date(datetime.year, datetime.month, datetime.day); + vec_to[i] = date.toPackedUInt(); + } + else + { + vec_to[i] = packed_uint; } + current_offset = next_offset; } } @@ -1361,29 +1364,26 @@ struct TiDBConvertToTime for (size_t i = 0; i < size; i++) { - try + MyDateTime datetime(0, 0, 0, 0, 0, 0, 0); + bool is_null = numberToDateTime(vec_from[i], datetime, context.getDAGContext()); + + if (is_null) { - MyDateTime datetime(0, 0, 0, 0, 0, 0, 0); - bool is_null = numberToDateTime(vec_from[i], datetime, context.getDAGContext()); - if constexpr (std::is_same_v) - { - MyDate date(datetime.year, datetime.month, datetime.day); - vec_to[i] = date.toPackedUInt(); - } - else - { - vec_to[i] = datetime.toPackedUInt(); - } - (*vec_null_map_to)[i] = is_null; + (*vec_null_map_to)[i] = 1; + vec_to[i] = 0; + continue; } - catch (const TiFlashException & e) + + if constexpr (std::is_same_v) { - // Cannot cast, fill with NULL - (*vec_null_map_to)[i] = 1; - context.getDAGContext()->handleInvalidTime( - "Invalid time value: '" + toString(vec_from[i]) + "'", - Errors::Types::WrongValue); + MyDate date(datetime.year, datetime.month, datetime.day); + vec_to[i] = date.toPackedUInt(); + } + else + { + vec_to[i] = datetime.toPackedUInt(); } + (*vec_null_map_to)[i] = is_null; } } else if constexpr (std::is_floating_point_v) @@ -1408,26 +1408,26 @@ struct TiDBConvertToTime } else { - try - { - Field packed_uint_value = parseMyDateTime(value_str, to_fsp); - UInt64 packed_uint = packed_uint_value.template safeGet(); - MyDateTime datetime(packed_uint); - if constexpr (std::is_same_v) - { - MyDate date(datetime.year, datetime.month, datetime.day); - vec_to[i] = date.toPackedUInt(); - } - else - { - vec_to[i] = packed_uint; - } - } - catch (const Exception &) + Field packed_uint_value = parseMyDateTime(value_str, to_fsp); + + if (packed_uint_value.isNull()) { // Fill NULL if cannot parse (*vec_null_map_to)[i] = 1; - context.getDAGContext()->handleInvalidTime("Invalid time value: '" + value_str + "'", Errors::Types::WrongValue); + vec_to[i] = 0; + continue; + } + + UInt64 packed_uint = packed_uint_value.template safeGet(); + MyDateTime datetime(packed_uint); + if constexpr (std::is_same_v) + { + MyDate date(datetime.year, datetime.month, datetime.day); + vec_to[i] = date.toPackedUInt(); + } + else + { + vec_to[i] = packed_uint; } } } @@ -1441,24 +1441,24 @@ struct TiDBConvertToTime for (size_t i = 0; i < size; i++) { String value_str = vec_from[i].toString(type.getScale()); - try + Field value = parseMyDateTime(value_str, to_fsp); + + if (value.getType() == Field::Types::Null) { - Field value = parseMyDateTime(value_str, to_fsp); - MyDateTime datetime(value.template safeGet()); - if constexpr (std::is_same_v) - { - MyDate date(datetime.year, datetime.month, datetime.day); - vec_to[i] = date.toPackedUInt(); - } - else - { - vec_to[i] = datetime.toPackedUInt(); - } + (*vec_null_map_to)[i] = 1; + vec_to[i] = 0; + continue; } - catch (const Exception &) + + MyDateTime datetime(value.template safeGet()); + if constexpr (std::is_same_v) { - (*vec_null_map_to)[i] = 1; - context.getDAGContext()->handleInvalidTime("Invalid time value: '" + value_str + "'", Errors::Types::WrongValue); + MyDate date(datetime.year, datetime.month, datetime.day); + vec_to[i] = date.toPackedUInt(); + } + else + { + vec_to[i] = datetime.toPackedUInt(); } } } @@ -1553,6 +1553,7 @@ struct TiDBConvertToDuration } }; +// Return true if the time is invalid. inline bool getDatetime(const Int64 & num, MyDateTime & result, DAGContext * ctx) { UInt64 ymd = num / 1000000; @@ -1570,15 +1571,15 @@ inline bool getDatetime(const Int64 & num, MyDateTime & result, DAGContext * ctx if (toCoreTimeChecked(year, month, day, hour, minute, second, 0, result)) { - throw TiFlashException("Incorrect time value", Errors::Types::WrongValue); + return true; } if (ctx) { - result.check(ctx->allowZeroInDate(), ctx->allowInvalidDate()); + return !result.isValid(ctx->allowZeroInDate(), ctx->allowInvalidDate()); } else { - result.check(false, false); + return !result.isValid(false, false); } return false; } @@ -1604,7 +1605,7 @@ inline bool numberToDateTime(Int64 number, MyDateTime & result, DAGContext * ctx // check MMDD if (number < 101) { - throw TiFlashException("Incorrect time value", Errors::Types::WrongValue); + return true; } // check YYMMDD: 2000-2069 @@ -1616,7 +1617,7 @@ inline bool numberToDateTime(Int64 number, MyDateTime & result, DAGContext * ctx if (number < 70 * 10000 + 101) { - throw TiFlashException("Incorrect time value", Errors::Types::WrongValue); + return true; } // check YYMMDD @@ -1636,7 +1637,7 @@ inline bool numberToDateTime(Int64 number, MyDateTime & result, DAGContext * ctx // check MMDDHHMMSS if (number < 101000000) { - throw TiFlashException("Incorrect time value", Errors::Types::WrongValue); + return true; } // check YYMMDDhhmmss: 2000-2069 @@ -1649,7 +1650,7 @@ inline bool numberToDateTime(Int64 number, MyDateTime & result, DAGContext * ctx // check YYYYMMDDhhmmss if (number < 70 * 10000000000 + 101000000) { - throw TiFlashException("Incorrect time value", Errors::Types::WrongValue); + return true; } // check YYMMDDHHMMSS diff --git a/dbms/src/Functions/tests/gtest_tidb_conversion.cpp b/dbms/src/Functions/tests/gtest_tidb_conversion.cpp index 887e896168f..cf8dab39b57 100644 --- a/dbms/src/Functions/tests/gtest_tidb_conversion.cpp +++ b/dbms/src/Functions/tests/gtest_tidb_conversion.cpp @@ -702,37 +702,38 @@ try createDateTimeColumnNullable({{}, {{2021, 10, 26, 16, 8, 59, 0}}}, 6), executeFunction(func_name, {createColumn>({{}, 20211026160859}), - createCastTypeConstColumn("Nullable(MyDateTime(6))")})); + createCastTypeConstColumn("Nullable(MyDateTime(6))")})); ASSERT_COLUMN_EQ( createDateTimeColumnNullable({{}, {{2021, 10, 26, 16, 8, 59, 0}}}, 6), executeFunction(func_name, {createColumn>({{}, 20211026160859}), - createCastTypeConstColumn("Nullable(MyDateTime(6))")})); - ASSERT_THROW( + createCastTypeConstColumn("Nullable(MyDateTime(6))")})); + + ASSERT_COLUMN_EQ( + createDateTimeColumnNullable({{}}, 6), executeFunction(func_name, {createColumn>({MAX_UINT8}), - createCastTypeConstColumn("Nullable(MyDateTime(6))")}), - TiFlashException); - ASSERT_THROW( + createCastTypeConstColumn("Nullable(MyDateTime(6))")})); + ASSERT_COLUMN_EQ( + createDateTimeColumnNullable({{}}, 6), executeFunction(func_name, {createColumn>({MAX_UINT16}), - createCastTypeConstColumn("Nullable(MyDateTime(6))")}), - TiFlashException); - ASSERT_THROW( + createCastTypeConstColumn("Nullable(MyDateTime(6))")})); + ASSERT_COLUMN_EQ( + createDateTimeColumnNullable({{}}, 6), executeFunction(func_name, {createColumn>({MAX_UINT32}), - createCastTypeConstColumn("Nullable(MyDateTime(6))")}), - TiFlashException); + createCastTypeConstColumn("Nullable(MyDateTime(6))")})); ASSERT_COLUMN_EQ( createDateTimeColumnNullable({{}}, 6), executeFunction(func_name, {createColumn>({0}), - createCastTypeConstColumn("Nullable(MyDateTime(6))")})); - ASSERT_THROW( + createCastTypeConstColumn("Nullable(MyDateTime(6))")})); + ASSERT_COLUMN_EQ( + createDateTimeColumnNullable({{}, {}}, 6), executeFunction(func_name, {createColumn>({{}, -20211026160859}), - createCastTypeConstColumn("Nullable(MyDateTime(6))")}), - TiFlashException); + createCastTypeConstColumn("Nullable(MyDateTime(6))")})); } CATCH @@ -994,4 +995,4 @@ try CATCH } // namespace -} // namespace DB::tests +} // namespace DB::tests \ No newline at end of file diff --git a/tests/fullstack-test/expr/return_warning.test b/tests/fullstack-test/expr/return_warning.test index 66d3a554097..8eecf603bbe 100644 --- a/tests/fullstack-test/expr/return_warning.test +++ b/tests/fullstack-test/expr/return_warning.test @@ -1,3 +1,18 @@ +# Copyright 2022 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#RETURN + mysql> drop table if exists test.t mysql> create table test.t(a int) mysql> insert into test.t values(1), (20201212)