diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index 1ee87cd0ed1..16d64843a8f 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -945,7 +945,7 @@ std::unordered_map scalar_func_map({ //{tipb::ScalarFuncSig::BitLength, "cast"}, //{tipb::ScalarFuncSig::Bin, "cast"}, - //{tipb::ScalarFuncSig::ASCII, "cast"}, + {tipb::ScalarFuncSig::ASCII, "ascii"}, //{tipb::ScalarFuncSig::Char, "cast"}, {tipb::ScalarFuncSig::CharLengthUTF8, "lengthUTF8"}, {tipb::ScalarFuncSig::Concat, "tidbConcat"}, {tipb::ScalarFuncSig::ConcatWS, "tidbConcatWS"}, @@ -972,9 +972,9 @@ std::unordered_map scalar_func_map({ {tipb::ScalarFuncSig::LTrim, "ltrim"}, {tipb::ScalarFuncSig::LeftUTF8, "leftUTF8"}, //{tipb::ScalarFuncSig::Left, "cast"}, {tipb::ScalarFuncSig::Length, "length"}, - //{tipb::ScalarFuncSig::Locate2ArgsUTF8, "cast"}, + {tipb::ScalarFuncSig::Locate2ArgsUTF8, "position"}, //{tipb::ScalarFuncSig::Locate3ArgsUTF8, "cast"}, - //{tipb::ScalarFuncSig::Locate2Args, "cast"}, + {tipb::ScalarFuncSig::Locate2Args, "position"}, //{tipb::ScalarFuncSig::Locate3Args, "cast"}, {tipb::ScalarFuncSig::Lower, "lower"}, @@ -1002,7 +1002,7 @@ std::unordered_map scalar_func_map({ //{tipb::ScalarFuncSig::SubstringIndex, "cast"}, //{tipb::ScalarFuncSig::ToBase64, "cast"}, - //{tipb::ScalarFuncSig::Trim1Arg, "cast"}, + {tipb::ScalarFuncSig::Trim1Arg, "trim"}, //{tipb::ScalarFuncSig::Trim2Args, "cast"}, //{tipb::ScalarFuncSig::Trim3Args, "cast"}, //{tipb::ScalarFuncSig::UnHex, "cast"}, diff --git a/dbms/src/Functions/FunctionsString.cpp b/dbms/src/Functions/FunctionsString.cpp index 3f68cf645c0..1666796f0e7 100644 --- a/dbms/src/Functions/FunctionsString.cpp +++ b/dbms/src/Functions/FunctionsString.cpp @@ -2792,6 +2792,189 @@ class PadUTF8Impl : public IFunction } }; +class FunctionASCII : public IFunction +{ +public: + static constexpr auto name = "ascii"; + FunctionASCII(const Context & context) : context(context) {} + + static FunctionPtr create(const Context & context) + { + return std::make_shared(context); + } + + std::string getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.size() != 1) + throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + + ", should be 1.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) override + { + const IColumn * c0_col = block.getByPosition(arguments[0]).column.get(); + const ColumnConst * c0_const = checkAndGetColumn(c0_col); + const ColumnString * c0_string = checkAndGetColumn(c0_col); + const ColumnFixedString * c0_fixed_string = checkAndGetColumn(c0_col); + + Field res_field; + int val_num = c0_col->size(); + auto col_res = ColumnInt64::create(); + col_res->reserve(val_num); + if (c0_const == nullptr && c0_string == nullptr && c0_fixed_string == nullptr) + throw Exception("Illegal argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + for (int i = 0; i < val_num; i++) + { + c0_col->get(i, res_field); + String handled_str = res_field.get(); + Int64 res = handled_str.size() == 0 ? 0 : static_cast(handled_str[0]); + col_res->insert(res); + } + + block.getByPosition(result).column = std::move(col_res); + } + +private: + const Context & context; +}; + +class FunctionLength : public IFunction +{ +public: + static constexpr auto name = "length"; + FunctionLength(const Context & context) : context(context) {} + + static FunctionPtr create(const Context & context) + { + return std::make_shared(context); + } + + std::string getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.size() != 1) + throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + + ", should be 1.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) override + { + const IColumn * c0_col = block.getByPosition(arguments[0]).column.get(); + const ColumnConst * c0_const = checkAndGetColumn(c0_col); + const ColumnString * c0_string = checkAndGetColumn(c0_col); + const ColumnFixedString * c0_fixed_string = checkAndGetColumn(c0_col); + + Field res_field; + int val_num = c0_col->size(); + auto col_res = ColumnInt64::create(); + col_res->reserve(val_num); + if (c0_const == nullptr && c0_string == nullptr && c0_fixed_string == nullptr) + throw Exception("Illegal argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + for (int i = 0; i < val_num; i++) + { + c0_col->get(i, res_field); + String handled_str = res_field.get(); + col_res->insert(static_cast(handled_str.size())); + } + + block.getByPosition(result).column = std::move(col_res); + } + +private: + const Context & context; +}; + +class FunctionPosition : public IFunction +{ +public: + static constexpr auto name = "position"; + FunctionPosition(const Context & context) : context(context) {} + + static FunctionPtr create(const Context & context) + { + return std::make_shared(context); + } + + std::string getName() const override { return name; } + size_t getNumberOfArguments() const override { return 2; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.size() != 2) + throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + + ", should be 2.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) override + { + const IColumn * c0_col = block.getByPosition(arguments[0]).column.get(); + const ColumnConst * c0_const = checkAndGetColumn(c0_col); + const ColumnString * c0_string = checkAndGetColumn(c0_col); + const ColumnFixedString * c0_fixed_string = checkAndGetColumn(c0_col); + Field c0_field; + + const IColumn * c1_col = block.getByPosition(arguments[1]).column.get(); + const ColumnConst * c1_const = checkAndGetColumn(c1_col); + const ColumnString * c1_string = checkAndGetColumn(c1_col); + const ColumnFixedString * c1_fixed_string = checkAndGetColumn(c1_col); + Field c1_field; + + if ((c0_const == nullptr && c0_string == nullptr && c0_fixed_string== nullptr) || + (c1_const == nullptr && c1_string == nullptr && c1_fixed_string== nullptr)) + throw Exception("Illegal argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (c0_col->size() != c1_col->size()) + throw Exception("Function " + getName() + " column number is inconformity", ErrorCodes::LOGICAL_ERROR); + + auto col_res = ColumnInt64::create(); + int val_num = c0_col->size(); + col_res->reserve(val_num); + + for (int i = 0; i < val_num; i++) + { + c0_col->get(i, c0_field); + c1_col->get(i, c1_field); + + String c0_str = c0_field.get(); + String c1_str = c1_field.get(); + + // return -1 when c1_str not contains the c0_str + Int64 idx = c1_str.find(c0_str); + col_res->insert(getPositionUTF8(c1_str, idx)); + } + + block.getByPosition(result).column = std::move(col_res); + } + +private: + Int64 getPositionUTF8(const String &c1_str, Int64 idx) + { + if (idx == -1) + return 0; + + auto data = reinterpret_cast(c1_str.data()); + return static_cast(UTF8::countCodePoints(data, idx) + 1); + } + + const Context & context; +}; + struct NameEmpty { @@ -2872,7 +3055,7 @@ struct NameConcatAssumeInjective using FunctionEmpty = FunctionStringOrArrayToT, NameEmpty, UInt8>; using FunctionNotEmpty = FunctionStringOrArrayToT, NameNotEmpty, UInt8>; -using FunctionLength = FunctionStringOrArrayToT; +// using FunctionLength = FunctionStringOrArrayToT; using FunctionLengthUTF8 = FunctionStringOrArrayToT; using FunctionLower = FunctionStringToString, NameLower>; using FunctionUpper = FunctionStringToString, NameUpper>; @@ -2912,5 +3095,7 @@ void registerFunctionsString(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); } } diff --git a/dbms/src/Functions/FunctionsStringSearch.cpp b/dbms/src/Functions/FunctionsStringSearch.cpp index aacf7cb9dbd..25fae5f5ed5 100644 --- a/dbms/src/Functions/FunctionsStringSearch.cpp +++ b/dbms/src/Functions/FunctionsStringSearch.cpp @@ -1798,7 +1798,7 @@ struct NameReplaceRegexpAll static constexpr auto name = "replaceRegexpAll"; }; -using FunctionPosition = FunctionsStringSearch, NamePosition>; +// using FunctionPosition = FunctionsStringSearch, NamePosition>; using FunctionPositionUTF8 = FunctionsStringSearch, NamePositionUTF8>; using FunctionPositionCaseInsensitive = FunctionsStringSearch, NamePositionCaseInsensitive>; using FunctionPositionCaseInsensitiveUTF8 @@ -1821,7 +1821,7 @@ void registerFunctionsStringSearch(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); - factory.registerFunction(); + // factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); diff --git a/dbms/src/Functions/tests/CMakeLists.txt b/dbms/src/Functions/tests/CMakeLists.txt index b40d2a80acb..5667759b0f4 100644 --- a/dbms/src/Functions/tests/CMakeLists.txt +++ b/dbms/src/Functions/tests/CMakeLists.txt @@ -18,3 +18,11 @@ target_link_libraries (arithmetic_functions dbms gtest_main clickhouse_functions add_executable (strings_replace gtest_strings_replace.cpp) target_link_libraries (strings_replace dbms gtest_main clickhouse_functions) +add_executable (strings_ascii gtest_strings_ascii.cpp) +target_link_libraries (strings_ascii dbms gtest_main clickhouse_functions) + +add_executable (strings_length gtest_strings_length.cpp) +target_link_libraries (strings_length dbms gtest_main clickhouse_functions) + +add_executable (strings_position gtest_strings_position.cpp) +target_link_libraries (strings_position dbms gtest_main clickhouse_functions) diff --git a/dbms/src/Functions/tests/gtest_strings_ascii.cpp b/dbms/src/Functions/tests/gtest_strings_ascii.cpp new file mode 100644 index 00000000000..c60c19b5411 --- /dev/null +++ b/dbms/src/Functions/tests/gtest_strings_ascii.cpp @@ -0,0 +1,153 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsign-compare" +#include + +#pragma GCC diagnostic pop + +namespace DB +{ +namespace tests +{ + +class StringASCII : public ::testing::Test +{ +protected: + static void SetUpTestCase() + { + try + { + registerFunctions(); + } + catch (DB::Exception &) + { + // Maybe another test has already registed, ignore exception here. + } + } +}; + +// test string and fixed string +TEST_F(StringASCII, str_and_fixed_str_Test) +{ + const Context context = TiFlashTestEnv::getContext(); + + auto & factory = FunctionFactory::instance(); + + std::vector strs{"hello", "HELLO", "23333", "#%@#^", ""}; + + for (int i = 0; i < 2; i++) + { + MutableColumnPtr csp; + if (i == 0) + csp = ColumnString::create(); + else + csp = ColumnFixedString::create(5); + + for (const auto & str : strs) + { + csp->insert(Field(str.c_str(), str.size())); + } + + Block testBlock; + ColumnWithTypeAndName ctn = ColumnWithTypeAndName(std::move(csp), std::make_shared(), "test_ascii"); + ColumnsWithTypeAndName ctns{ctn}; + testBlock.insert(ctn); + ColumnNumbers cns{0}; + + // test ascii + auto bp = factory.tryGet("ascii", context); + ASSERT_TRUE(bp != nullptr); + ASSERT_FALSE(bp->isVariadic()); + + auto func = bp->build(ctns); + testBlock.insert({nullptr, func->getReturnType(), "res"}); + func->execute(testBlock, cns, 1); + const IColumn * res = testBlock.getByPosition(1).column.get(); + const ColumnInt64 * res_string = checkAndGetColumn(res); + + Field resField; + std::vector results{104, 72, 50, 35, 0}; + for (size_t t = 0; t < results.size(); t++) + { + res_string->get(t, resField); + Int64 res_val = resField.get(); + EXPECT_EQ(results[t], res_val); + } + } + +} + +// test NULL +TEST_F(StringASCII, null_Test) +{ + const Context context = TiFlashTestEnv::getContext(); + + auto & factory = FunctionFactory::instance(); + + std::vector strs{"a", "b", "c", "d", "e", "f"}; + std::vector results{0, 98, 0, 100, 101, 0}; + std::vector null_map{1, 0, 1, 0, 0, 1}; + auto input_str_col = ColumnString::create(); + for (auto str : strs) { + Field field(str.c_str(), str.size()); + input_str_col->insert(field); + } + + auto input_null_map = ColumnUInt8::create(strs.size(), 0); + ColumnUInt8::Container &input_vec_null_map = input_null_map->getData(); + for (size_t i = 0; i < strs.size(); i++) { + input_vec_null_map[i] = null_map[i]; + } + + auto input_null_col = ColumnNullable::create(std::move(input_str_col), std::move(input_null_map)); + DataTypePtr string_type = std::make_shared(); + DataTypePtr nullable_string_type = makeNullable(string_type); + + auto col1 = ColumnWithTypeAndName(std::move(input_null_col), nullable_string_type, "ascii"); + ColumnsWithTypeAndName ctns{col1}; + + Block testBlock; + testBlock.insert(col1); + ColumnNumbers cns{0}; + + auto bp = factory.tryGet("ascii", context); + ASSERT_TRUE(bp != nullptr); + ASSERT_FALSE(bp->isVariadic()); + auto func = bp->build(ctns); + testBlock.insert({nullptr, func->getReturnType(), "res"}); + func->execute(testBlock, cns, 1); + auto res_col = testBlock.getByPosition(1).column; + + ColumnPtr result_null_map_column = static_cast(*res_col).getNullMapColumnPtr(); + MutableColumnPtr mutable_result_null_map_column = (*std::move(result_null_map_column)).mutate(); + NullMap & result_null_map = static_cast(*mutable_result_null_map_column).getData(); + const IColumn * res = testBlock.getByPosition(1).column.get(); + const ColumnNullable * res_nullable_string = checkAndGetColumn(res); + const IColumn & res_string = res_nullable_string->getNestedColumn(); + + Field resField; + + for (size_t i = 0; i < null_map.size(); i++) { + EXPECT_EQ(result_null_map[i], null_map[i]); + if (result_null_map[i] == 0) { + res_string.get(i, resField); + Int64 res_val = resField.get(); + EXPECT_EQ(results[i], res_val); + } + } +} + +} // namespace tests +} // namespace DB \ No newline at end of file diff --git a/dbms/src/Functions/tests/gtest_strings_length.cpp b/dbms/src/Functions/tests/gtest_strings_length.cpp new file mode 100644 index 00000000000..c7b471c0d9f --- /dev/null +++ b/dbms/src/Functions/tests/gtest_strings_length.cpp @@ -0,0 +1,160 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsign-compare" +#include + +#pragma GCC diagnostic pop + +namespace DB +{ +namespace tests +{ + +class StringLength : public ::testing::Test +{ +protected: + static void SetUpTestCase() + { + try + { + registerFunctions(); + } + catch (DB::Exception &) + { + // Maybe another test has already registed, ignore exception here. + } + } +}; + +// test string and fixed string +TEST_F(StringLength, str_and_fixed_str_Test) +{ + const Context context = TiFlashTestEnv::getContext(); + + auto & factory = FunctionFactory::instance(); + + std::vector fixed_strs{"hello", "HELLO", "23333", "#%@#^", "ni好"}; + std::vector fixed_strs_results{5, 5, 5, 5, 5}; + + std::vector strs{"hi~", "23333", "pingcap", "你好", "233哈哈", ""}; + std::vector results{3, 5, 7, 6, 9, 0}; + + for (int i = 0; i < 2; i++) + { + MutableColumnPtr csp; + if (i == 0) + csp = ColumnString::create(); + else + { + strs = fixed_strs; + results = fixed_strs_results; + csp = ColumnFixedString::create(5); + } + + for (const auto & str : strs) + { + csp->insert(Field(str.c_str(), str.size())); + } + + Block testBlock; + ColumnWithTypeAndName ctn = ColumnWithTypeAndName(std::move(csp), std::make_shared(), "test_ascii"); + ColumnsWithTypeAndName ctns{ctn}; + testBlock.insert(ctn); + ColumnNumbers cns{0}; + + // test length + auto bp = factory.tryGet("length", context); + ASSERT_TRUE(bp != nullptr); + ASSERT_FALSE(bp->isVariadic()); + + auto func = bp->build(ctns); + testBlock.insert({nullptr, func->getReturnType(), "res"}); + func->execute(testBlock, cns, 1); + const IColumn * res = testBlock.getByPosition(1).column.get(); + const ColumnInt64 * res_string = checkAndGetColumn(res); + + Field resField; + + for (size_t t = 0; t < results.size(); t++) + { + res_string->get(t, resField); + Int64 res_val = resField.get(); + EXPECT_EQ(results[t], res_val); + } + } +} + +// test NULL +TEST_F(StringLength, null_Test) +{ + const Context context = TiFlashTestEnv::getContext(); + + auto & factory = FunctionFactory::instance(); + + std::vector strs{"a", "abcd", "嗯", "饼干", "馒头", "?？?"}; + std::vector results{0, 4, 0, 6, 6, 0}; + std::vector null_map{1, 0, 1, 0, 0, 1}; + auto input_str_col = ColumnString::create(); + for (auto str : strs) { + Field field(str.c_str(), str.size()); + input_str_col->insert(field); + } + + auto input_null_map = ColumnUInt8::create(strs.size(), 0); + ColumnUInt8::Container &input_vec_null_map = input_null_map->getData(); + for (size_t i = 0; i < strs.size(); i++) { + input_vec_null_map[i] = null_map[i]; + } + + auto input_null_col = ColumnNullable::create(std::move(input_str_col), std::move(input_null_map)); + DataTypePtr string_type = std::make_shared(); + DataTypePtr nullable_string_type = makeNullable(string_type); + + auto col1 = ColumnWithTypeAndName(std::move(input_null_col), nullable_string_type, "length"); + ColumnsWithTypeAndName ctns{col1}; + + Block testBlock; + testBlock.insert(col1); + ColumnNumbers cns{0}; + + auto bp = factory.tryGet("length", context); + ASSERT_TRUE(bp != nullptr); + ASSERT_FALSE(bp->isVariadic()); + auto func = bp->build(ctns); + testBlock.insert({nullptr, func->getReturnType(), "res"}); + func->execute(testBlock, cns, 1); + auto res_col = testBlock.getByPosition(1).column; + + ColumnPtr result_null_map_column = static_cast(*res_col).getNullMapColumnPtr(); + MutableColumnPtr mutable_result_null_map_column = (*std::move(result_null_map_column)).mutate(); + NullMap & result_null_map = static_cast(*mutable_result_null_map_column).getData(); + const IColumn * res = testBlock.getByPosition(1).column.get(); + const ColumnNullable * res_nullable_string = checkAndGetColumn(res); + const IColumn & res_string = res_nullable_string->getNestedColumn(); + + Field resField; + + for (size_t i = 0; i < null_map.size(); i++) { + EXPECT_EQ(result_null_map[i], null_map[i]); + if (result_null_map[i] == 0) { + res_string.get(i, resField); + Int64 res_val = resField.get(); + EXPECT_EQ(results[i], res_val); + } + } +} + +} // namespace tests +} // namespace DB diff --git a/dbms/src/Functions/tests/gtest_strings_position.cpp b/dbms/src/Functions/tests/gtest_strings_position.cpp new file mode 100644 index 00000000000..7d6ae5afddf --- /dev/null +++ b/dbms/src/Functions/tests/gtest_strings_position.cpp @@ -0,0 +1,334 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsign-compare" +#include + +#pragma GCC diagnostic pop + +namespace DB +{ +namespace tests +{ + +class StringPosition : public ::testing::Test +{ +protected: + static void SetUpTestCase() + { + try + { + registerFunctions(); + } + catch (DB::Exception &) + { + // Maybe another test has already registed, ignore exception here. + } + } +}; + +// test string and fixed string +TEST_F(StringPosition, str_and_fixed_str_Test) +{ + const Context context = TiFlashTestEnv::getContext(); + + auto & factory = FunctionFactory::instance(); + + // case insensitive + std::vector c0_var_strs{ "ell", "LL", "3", "ElL", "ye", "aaaa", "world", "", "", "biu" }; + std::vector c1_var_strs{ "hello", "HELLO", "23333", "HeLlO", "hey", "a", "WoRlD", "", "ping", "" }; + + std::vector c0_fixed_strs{ "ell", "LLo", "333", "ElL", "yye", "aaa", "orl", "321", "xzx", "biu" }; + std::vector c1_fixed_strs{ "hello", "HELLO", "23333", "HeLlO", "heyyy", "aaaaa", "WoRlD", "12345", "apple", "b_i_u"}; + + // var-var + std::vector result0{2, 3, 2, 0, 0, 0, 0, 1, 1, 0}; + // fixed-fixed + std::vector result1{2, 0, 2, 0, 0, 1, 0, 0, 0, 0}; + // var-fixed + std::vector result2{2, 3, 2, 0, 0, 1, 0, 1, 1, 0}; + // fixed-var + std::vector result3{2, 0, 2, 0, 0, 0, 0, 0, 0, 0}; + + std::vector c0_strs; + std::vector c1_strs; + std::vector results; + for (int i = 0; i < 4; i++) + { + MutableColumnPtr csp0; + MutableColumnPtr csp1; + if (i == 0) + { + // var-var + c0_strs = c0_var_strs; + c1_strs = c1_var_strs; + results = result0; + + csp0 = ColumnString::create(); + csp1 = ColumnString::create(); + } + else if (i == 1) + { + // fixed-fixed + c0_strs = c0_fixed_strs; + c1_strs = c1_fixed_strs; + results = result1; + + csp0 = ColumnFixedString::create(3); + csp1 = ColumnFixedString::create(5); + } + else if (i == 2) + { + // var-fixed + c0_strs = c0_var_strs; + c1_strs = c1_fixed_strs; + results = result2; + + csp0 = ColumnString::create(); + csp1 = ColumnFixedString::create(5); + } + else + { + // fixed-var + c0_strs = c0_fixed_strs; + c1_strs = c1_var_strs; + results = result3; + + csp0 = ColumnFixedString::create(3); + csp1 = ColumnString::create(); + } + + for (size_t i = 0; i < c0_strs.size(); i++) + { + csp0->insert(Field(c0_strs[i].c_str(), c0_strs[i].size())); + csp1->insert(Field(c1_strs[i].c_str(), c1_strs[i].size())); + } + + Block testBlock; + ColumnWithTypeAndName ctn0 = ColumnWithTypeAndName(std::move(csp0), std::make_shared(), "test_position_0"); + ColumnWithTypeAndName ctn1 = ColumnWithTypeAndName(std::move(csp1), std::make_shared(), "test_position_1"); + ColumnsWithTypeAndName ctns{ctn0, ctn1}; + testBlock.insert(ctn0); + testBlock.insert(ctn1); + // for result from position + testBlock.insert({}); + ColumnNumbers cns{0, 1}; + + // test position + auto bp = factory.tryGet("position", context); + ASSERT_TRUE(bp != nullptr); + ASSERT_FALSE(bp->isVariadic()); + + bp->build(ctns)->execute(testBlock, cns, 2); + const IColumn * res = testBlock.getByPosition(2).column.get(); + const ColumnInt64 * res_string = checkAndGetColumn(res); + + Field resField; + + for (size_t t = 0; t < results.size(); t++) + { + res_string->get(t, resField); + Int64 res_val = resField.get(); + EXPECT_EQ(results[t], res_val); + } + } +} + +// test string and fixed string in utf8 +TEST_F(StringPosition, utf8_str_and_fixed_str_Test) +{ + const Context context = TiFlashTestEnv::getContext(); + + auto & factory = FunctionFactory::instance(); + + // case insensitive + std::vector c0_var_strs{ "好", "平凯", "aa哈", "？！", "呵呵呵", "233", "嗯？？" }; + std::vector c1_var_strs{ "ni好", "平凯星辰", "啊啊aaa哈哈", "？？！！", "呵呵呵", "哈哈2333", "嗯？" }; + + std::vector c0_fixed_strs{ "好", "凯", "哈", "！", "嗯", "二", "？" }; + std::vector c1_fixed_strs{ "好de_", "平凯", "aaa哈", "！？", "嗯嗯", "2二33", "？？" }; + + // var-var + std::vector result0{3, 1, 4, 2, 1, 3, 0}; + // fixed-fixed + std::vector result1{1, 2, 4, 1, 1, 2, 1}; + // var-fixed + std::vector result2{1, 1, 2, 0, 0, 0, 0}; + // fixed-var + std::vector result3{3, 2, 6, 3, 0, 0, 2}; + + std::vector c0_strs; + std::vector c1_strs; + std::vector results; + for (int i = 0; i < 4; i++) + { + MutableColumnPtr csp0; + MutableColumnPtr csp1; + if (i == 0) + { + // var-var + c0_strs = c0_var_strs; + c1_strs = c1_var_strs; + results = result0; + + csp0 = ColumnString::create(); + csp1 = ColumnString::create(); + } + else if (i == 1) + { + // fixed-fixed + c0_strs = c0_fixed_strs; + c1_strs = c1_fixed_strs; + results = result1; + + csp0 = ColumnFixedString::create(3); + csp1 = ColumnFixedString::create(6); + } + else if (i == 2) + { + // var-fixed + c0_strs = c0_var_strs; + c1_strs = c1_fixed_strs; + results = result2; + + csp0 = ColumnString::create(); + csp1 = ColumnFixedString::create(6); + } + else + { + // fixed-var + c0_strs = c0_fixed_strs; + c1_strs = c1_var_strs; + results = result3; + + csp0 = ColumnFixedString::create(3); + csp1 = ColumnString::create(); + } + + for (size_t i = 0; i < c0_strs.size(); i++) + { + csp0->insert(Field(c0_strs[i].c_str(), c0_strs[i].size())); + csp1->insert(Field(c1_strs[i].c_str(), c1_strs[i].size())); + } + + Block testBlock; + ColumnWithTypeAndName ctn0 = ColumnWithTypeAndName(std::move(csp0), std::make_shared(), "test_position_0"); + ColumnWithTypeAndName ctn1 = ColumnWithTypeAndName(std::move(csp1), std::make_shared(), "test_position_1"); + ColumnsWithTypeAndName ctns{ctn0, ctn1}; + testBlock.insert(ctn0); + testBlock.insert(ctn1); + // for result from position + testBlock.insert({}); + ColumnNumbers cns{0, 1}; + + // test position + auto bp = factory.tryGet("position", context); + ASSERT_TRUE(bp != nullptr); + ASSERT_FALSE(bp->isVariadic()); + + bp->build(ctns)->execute(testBlock, cns, 2); + const IColumn * res = testBlock.getByPosition(2).column.get(); + const ColumnInt64 * res_string = checkAndGetColumn(res); + + Field resField; + + for (size_t t = 0; t < results.size(); t++) + { + res_string->get(t, resField); + Int64 res_val = resField.get(); + EXPECT_EQ(results[t], res_val); + } + } +} + +// test NULL +TEST_F(StringPosition, null_Test) +{ + const Context context = TiFlashTestEnv::getContext(); + + auto & factory = FunctionFactory::instance(); + + std::vector c0_strs{"aa", "c", "香锅", "cap", "f"}; + std::vector c1_strs{"a", "cc", "麻辣v香锅", "pingcap", "f"}; + std::vector results{0, 0, 4, 5, 0}; + + std::vector c0_null_map{0, 1, 0, 0, 1}; + std::vector c1_null_map{1, 0, 0, 0, 1}; + std::vector c2_null_map{1, 1, 0, 0, 1}; // for result + + auto input_str_col0 = ColumnString::create(); + auto input_str_col1 = ColumnString::create(); + for (size_t i = 0; i < c0_strs.size(); i++) + { + Field field0(c0_strs[i].c_str(), c0_strs[i].size()); + Field field1(c1_strs[i].c_str(), c1_strs[i].size()); + input_str_col0->insert(field0); + input_str_col1->insert(field1); + } + + auto input_null_map0 = ColumnUInt8::create(c0_strs.size(), 0); + auto input_null_map1 = ColumnUInt8::create(c1_strs.size(), 0); + ColumnUInt8::Container &input_vec_null_map0 = input_null_map0->getData(); + ColumnUInt8::Container &input_vec_null_map1 = input_null_map1->getData(); + for (size_t i = 0; i < c0_null_map.size(); i++) + { + input_vec_null_map0[i] = c0_null_map[i]; + input_vec_null_map1[i] = c1_null_map[i]; + } + + DataTypePtr string_type = std::make_shared(); + DataTypePtr nullable_string_type = makeNullable(string_type); + + auto input_null_col0 = ColumnNullable::create(std::move(input_str_col0), std::move(input_null_map0)); + auto input_null_col1 = ColumnNullable::create(std::move(input_str_col1), std::move(input_null_map1)); + auto col0 = ColumnWithTypeAndName(std::move(input_null_col0), nullable_string_type, "position"); + auto col1 = ColumnWithTypeAndName(std::move(input_null_col1), nullable_string_type, "position"); + ColumnsWithTypeAndName ctns{col0, col1}; + + Block testBlock; + testBlock.insert(col0); + testBlock.insert(col1); + ColumnNumbers cns{0, 1}; + + auto bp = factory.tryGet("position", context); + ASSERT_TRUE(bp != nullptr); + ASSERT_FALSE(bp->isVariadic()); + auto func = bp->build(ctns); + testBlock.insert({nullptr, func->getReturnType(), "res"}); + func->execute(testBlock, cns, 2); + auto res_col = testBlock.getByPosition(2).column; + + ColumnPtr result_null_map_column = static_cast(*res_col).getNullMapColumnPtr(); + MutableColumnPtr mutable_result_null_map_column = (*std::move(result_null_map_column)).mutate(); + NullMap & result_null_map = static_cast(*mutable_result_null_map_column).getData(); + const IColumn * res = testBlock.getByPosition(2).column.get(); + const ColumnNullable * res_nullable_string = checkAndGetColumn(res); + const IColumn & res_string = res_nullable_string->getNestedColumn(); + + Field resField; + + for (size_t i = 0; i < c2_null_map.size(); i++) + { + EXPECT_EQ(result_null_map[i], c2_null_map[i]); + if (c2_null_map[i] == 0) + { + res_string.get(i, resField); + Int64 res_val = resField.get(); + EXPECT_EQ(results[i], res_val); + } + } +} + +} // namespace tests +} // namespace DB diff --git a/tests/fullstack-test/expr/ascii_pushdown.test b/tests/fullstack-test/expr/ascii_pushdown.test new file mode 100644 index 00000000000..e6764998af2 --- /dev/null +++ b/tests/fullstack-test/expr/ascii_pushdown.test @@ -0,0 +1,210 @@ +mysql> drop table if exists test.test_tb +mysql> create table if not exists test.test_tb(id int NOT NULL AUTO_INCREMENT,char_ CHAR(4) NULL,enum_ ENUM('enum1', 'enum2', 'enum3') NULL,longtext_ LONGTEXT NULL,mediumtext_ MEDIUMTEXT NULL,set_ SET('set1', 'set2', 'set3') NULL,text_ TEXT NULL,tinytext_ TINYTEXT NULL,varchar_ VARCHAR(10) NULL,bit_ BIT NULL,bigint_ BIGINT NULL,boolean_ BOOLEAN NULL,decimal_ DECIMAL NULL,double_ DOUBLE NULL,float_ FLOAT NULL,int_ INT NULL,mediumint_ MEDIUMINT NULL,real_ REAL NULL,smallint_ SMALLINT NULL,tinyint_ TINYINT NULL,date_ DATE NULL,datetime_ DATETIME NULL,timestamp_ TIMESTAMP NULL,time_ TIME NULL,year_ YEAR NULL,blob_ BLOB NULL,mediumblob_ MEDIUMBLOB NULL,longblob_ LONGBLOB NULL,tinyblob_ TINYBLOB NULL,json_ JSON NULL,PRIMARY KEY(id)) + +mysql> INSERT INTO test.test_tb(id,char_,enum_,longtext_,mediumtext_,set_,text_,tinytext_,varchar_,bit_,bigint_,boolean_,decimal_,double_,float_,int_,mediumint_,real_,smallint_,tinyint_,date_,datetime_,timestamp_,time_,year_,blob_,mediumblob_,longblob_,tinyblob_)VALUES(1,'char','enum1','longtext','mediumtext','set1','text','tinytext','varchar',1,123,true,1.1,1.2,1.3,2,3,1.4,4,5,'2021-7-15','2021-7-15 17:04:52','2021-7-15 17:05:01','17:05:01',2021,'blob','mediumblob','longblob','tinyblob') + +mysql> analyze table test.test_tb +mysql> alter table test.test_tb set tiflash replica 1 +func> wait_table test test_tb + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(char_) = 99; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(longtext_) = 108; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(mediumtext_) = 109; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(text_) = 116; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(tinytext_) = 116; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(varchar_) = 118; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(bigint_) = 49; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(boolean_) = 49; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(decimal_) = 49; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(double_) = 49; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(float_) = 49; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(int_) = 50; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(mediumint_) = 51; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(real_) = 49; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(smallint_) = 52; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(tinyint_) = 53; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(date_) = 50; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(datetime_) = 50; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(timestamp_) = 50; ++----+ +| id | ++----+ +| 1 | ++----+ + + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(year_) = 50; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(blob_) = 98; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(mediumblob_) = 109; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(longblob_) = 108; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ascii(tinyblob_) = 116; ++----+ +| id | ++----+ +| 1 | ++----+ + +# test NULL +mysql> INSERT INTO test.test_tb(id) VALUES (2); +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ISNULL(ascii(char_)); ++----+ +| id | ++----+ +| 2 | ++----+ \ No newline at end of file diff --git a/tests/fullstack-test/expr/coalesce_pushdown.test b/tests/fullstack-test/expr/coalesce_pushdown.test new file mode 100644 index 00000000000..9b89f5e76d8 --- /dev/null +++ b/tests/fullstack-test/expr/coalesce_pushdown.test @@ -0,0 +1,337 @@ +mysql> drop table if exists test.test_tb +mysql> create table if not exists test.test_tb(id int NOT NULL AUTO_INCREMENT,char_ CHAR(4) NULL,enum_ ENUM('enum1', 'enum2', 'enum3') NULL,longtext_ LONGTEXT NULL,mediumtext_ MEDIUMTEXT NULL,set_ SET('set1', 'set2', 'set3') NULL,text_ TEXT NULL,tinytext_ TINYTEXT NULL,varchar_ VARCHAR(10) NULL,bit_ BIT NULL,bigint_ BIGINT NULL,boolean_ BOOLEAN NULL,decimal_ DECIMAL NULL,double_ DOUBLE NULL,float_ FLOAT NULL,int_ INT NULL,mediumint_ MEDIUMINT NULL,real_ REAL NULL,smallint_ SMALLINT NULL,tinyint_ TINYINT NULL,date_ DATE NULL,datetime_ DATETIME NULL,timestamp_ TIMESTAMP NULL,time_ TIME NULL,year_ YEAR NULL,blob_ BLOB NULL,mediumblob_ MEDIUMBLOB NULL,longblob_ LONGBLOB NULL,tinyblob_ TINYBLOB NULL,json_ JSON NULL,PRIMARY KEY(id)) + +mysql> INSERT INTO test.test_tb(id,char_,enum_,longtext_,mediumtext_,set_,text_,tinytext_,varchar_,bit_,bigint_,boolean_,decimal_,double_,float_,int_,mediumint_,real_,smallint_,tinyint_,date_,datetime_,timestamp_,time_,year_,blob_,mediumblob_,longblob_,tinyblob_)VALUES(1,'char','enum1','longtext','mediumtext','set1','text','tinytext','varchar',1,123,true,1.1,1.2,1.3,2,3,1.4,4,5,'2021-7-15','2021-7-15 17:04:52','2021-7-15 17:05:01','17:05:01',2021,'blob','mediumblob','longblob','tinyblob') + +mysql> analyze table test.test_tb +mysql> alter table test.test_tb set tiflash replica 1 +func> wait_table test test_tb + + + +# start checking +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where char_ = coalesce(null, char_); ++----+ +| id | ++----+ +| 1 | ++----+ + +#mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where enum_ = coalesce(null, enum_); +#+----+ +#| id | +#+----+ +#| 1 | +#+----+ + +#mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where set_ = coalesce(null, set_); +#+----+ +#| id | +#+----+ +#| 1 | +#+----+ + +#mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where bit_ = coalesce(null, bit_); +#+----+ +#| id | +#+----+ +#| 1 | +#+----+ + +#mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where time_ = coalesce(null, time_); +#+----+ +#| id | +#+----+ +#| 1 | +#+----+ + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where longtext_ = coalesce(null, longtext_); ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where mediumtext_ = coalesce(null, mediumtext_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where text_ = coalesce(null, text_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where tinytext_ = coalesce(null, tinytext_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where varchar_ = coalesce(null, varchar_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where bigint_ = coalesce(null, bigint_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where boolean_ = coalesce(null, boolean_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where decimal_ = coalesce(null, decimal_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where double_ = coalesce(null, double_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where float_ = coalesce(null, float_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where int_ = coalesce(null, int_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where mediumint_ = coalesce(null, mediumint_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where real_ = coalesce(null, real_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where smallint_ = coalesce(null, smallint_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where tinyint_ = coalesce(null, tinyint_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where date_ = coalesce(null, date_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where datetime_ = coalesce(null, datetime_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where timestamp_ = coalesce(null, timestamp_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where year_ = coalesce(null, year_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where blob_ = coalesce(null, blob_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where mediumblob_ = coalesce(null, mediumblob_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where longblob_ = coalesce(null, longblob_); ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where tinyblob_ = coalesce(null, tinyblob_); ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, char_, decimal_) = char_; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, decimal_, char_) = decimal_; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, id, char_) = id; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, char_, id) = char_; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, id, decimal_) = id; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, decimal_, id) = decimal_; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, datetime_, id) = datetime_; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, id, datetime_) = id; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, datetime_, decimal_) = datetime_; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, decimal_, datetime_) = decimal_; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, datetime_, char_) = datetime_; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, char_, datetime_) = char_; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, datetime_, decimal_) = datetime_; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[books]) */ id from test.test_tb where coalesce(null, decimal_, datetime_) = decimal_; ++----+ +| id | ++----+ +| 1 | ++----+ + + +# test NULL +mysql> INSERT INTO test.test_tb(id)VALUES(2); +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where ISNULL(coalesce(char_)); ++----+ +| id | ++----+ +| 2 | ++----+ + diff --git a/tests/fullstack-test/expr/length_pushdown.test b/tests/fullstack-test/expr/length_pushdown.test new file mode 100644 index 00000000000..dfaf37ae573 --- /dev/null +++ b/tests/fullstack-test/expr/length_pushdown.test @@ -0,0 +1,213 @@ +mysql> drop table if exists test.test_tb +mysql> create table if not exists test.test_tb(id int NOT NULL AUTO_INCREMENT,char_ CHAR(4) NULL,enum_ ENUM('enum1', 'enum2', 'enum3') NULL,longtext_ LONGTEXT NULL,mediumtext_ MEDIUMTEXT NULL,set_ SET('set1', 'set2', 'set3') NULL,text_ TEXT NULL,tinytext_ TINYTEXT NULL,varchar_ VARCHAR(10) NULL,bit_ BIT NULL,bigint_ BIGINT NULL,boolean_ BOOLEAN NULL,decimal_ DECIMAL NULL,double_ DOUBLE NULL,float_ FLOAT NULL,int_ INT NULL,mediumint_ MEDIUMINT NULL,real_ REAL NULL,smallint_ SMALLINT NULL,tinyint_ TINYINT NULL,date_ DATE NULL,datetime_ DATETIME NULL,timestamp_ TIMESTAMP NULL,time_ TIME NULL,year_ YEAR NULL,blob_ BLOB NULL,mediumblob_ MEDIUMBLOB NULL,longblob_ LONGBLOB NULL,tinyblob_ TINYBLOB NULL,json_ JSON NULL,PRIMARY KEY(id)) + +mysql> INSERT INTO test.test_tb(id,char_,enum_,longtext_,mediumtext_,set_,text_,tinytext_,varchar_,bit_,bigint_,boolean_,decimal_,double_,float_,int_,mediumint_,real_,smallint_,tinyint_,date_,datetime_,timestamp_,time_,year_,blob_,mediumblob_,longblob_,tinyblob_)VALUES(1,'char','enum1','longtext','mediumtext','set1','text','tinytext','varchar',1,123,true,1.1,1.2,1.3,2,3,1.4,4,5,'2021-7-15','2021-7-15 17:04:52','2021-7-15 17:05:01','17:05:01',2021,'blob','mediumblob','longblob','tinyblob') +mysql> INSERT INTO test.test_tb(id, varchar_) VALUES (2, '平CAP'); +mysql> INSERT INTO test.test_tb(id, varchar_) VALUES (3, '#￥aaa'); +mysql> INSERT INTO test.test_tb(id, varchar_) VALUES (4, '哈哈哈'); + +mysql> analyze table test.test_tb +mysql> alter table test.test_tb set tiflash replica 1 +func> wait_table test test_tb + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(char_) = 4; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(longtext_) = 8; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(mediumtext_) = 10; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(text_) = 4; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(tinytext_) = 8; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(varchar_) = 7; ++----+ +| id | ++----+ +| 1 | +| 3 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(bigint_) = 3; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(boolean_) = 1; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(decimal_) = 1; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(double_) = 3; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(float_) = 3; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(int_) = 1; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(mediumint_) = 1; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(real_) = 3; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(smallint_) = 1; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(tinyint_) = 1; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(date_) = 10; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(datetime_) = 19; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(timestamp_) = 19; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(year_) = 4; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(blob_) = 4; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(mediumblob_) = 10; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(longblob_) = 8; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where length(tinyblob_) = 8; ++----+ +| id | ++----+ +| 1 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where isnull(length(char_)); ++----+ +| id | ++----+ +| 2 | +| 3 | +| 4 | ++----+ \ No newline at end of file diff --git a/tests/fullstack-test/expr/position_pushdown.test b/tests/fullstack-test/expr/position_pushdown.test new file mode 100644 index 00000000000..ce473019f09 --- /dev/null +++ b/tests/fullstack-test/expr/position_pushdown.test @@ -0,0 +1,93 @@ +mysql> drop table if exists test.test_tb +mysql> create table if not exists test.test_tb(id int NOT NULL AUTO_INCREMENT,char_ CHAR(4) NULL,enum_ ENUM('enum1', 'enum2', 'enum3') NULL,longtext_ LONGTEXT NULL,mediumtext_ MEDIUMTEXT NULL,set_ SET('set1', 'set2', 'set3') NULL,text_ TEXT NULL,tinytext_ TINYTEXT NULL,varchar_ VARCHAR(50) NULL,bit_ BIT NULL,bigint_ BIGINT NULL,boolean_ BOOLEAN NULL,decimal_ DECIMAL NULL,double_ DOUBLE NULL,float_ FLOAT NULL,int_ INT NULL,mediumint_ MEDIUMINT NULL,real_ REAL NULL,smallint_ SMALLINT NULL,tinyint_ TINYINT NULL,date_ DATE NULL,datetime_ DATETIME NULL,timestamp_ TIMESTAMP NULL,time_ TIME NULL,year_ YEAR NULL,blob_ BLOB NULL,mediumblob_ MEDIUMBLOB NULL,longblob_ LONGBLOB NULL,tinyblob_ TINYBLOB NULL,json_ JSON NULL,PRIMARY KEY(id)) + +mysql> INSERT INTO test.test_tb(id, varchar_) VALUES (1, 'hello world '); +mysql> INSERT INTO test.test_tb(id, varchar_) VALUES (2, 'HEllo WoRld '); +mysql> INSERT INTO test.test_tb(id, varchar_) VALUES (3, '平凯星辰'); +mysql> INSERT INTO test.test_tb(id, varchar_) VALUES (4, '平凯&星辰 '); +mysql> INSERT INTO test.test_tb(id, char_, varchar_) VALUES (5, 'haha', '233haha2'); + +mysql> analyze table test.test_tb +mysql> alter table test.test_tb set tiflash replica 1 +func> wait_table test test_tb + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where position('hello' in varchar_) = 1; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where position('o WoR' in varchar_) = 5; ++----+ +| id | ++----+ +| 2 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where position('平凯' in varchar_) = 1; ++----+ +| id | ++----+ +| 3 | +| 4 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where position('星辰' in varchar_) = 4; ++----+ +| id | ++----+ +| 4 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where position(char_ in varchar_) = 4; ++----+ +| id | ++----+ +| 5 | ++----+ + +# test NULL +mysql> INSERT INTO test.test_tb(id, char_) VALUES (6, 'haha'); +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where isnull(position(NULL in varchar_)); ++----+ +| id | ++----+ +| 1 | +| 2 | +| 3 | +| 4 | +| 5 | +| 6 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where isnull(position(NULL in char_)); ++----+ +| id | ++----+ +| 1 | +| 2 | +| 3 | +| 4 | +| 5 | +| 6 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where isnull(position('ppp' in varchar_)); ++----+ +| id | ++----+ +| 6 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where isnull(position('ppp' in char_)); ++----+ +| id | ++----+ +| 1 | +| 2 | +| 3 | +| 4 | ++----+ \ No newline at end of file diff --git a/tests/fullstack-test/expr/trim_pushdown.test b/tests/fullstack-test/expr/trim_pushdown.test new file mode 100644 index 00000000000..12c00d25afc --- /dev/null +++ b/tests/fullstack-test/expr/trim_pushdown.test @@ -0,0 +1,55 @@ +mysql> drop table if exists test.test_tb +mysql> create table if not exists test.test_tb(id int NOT NULL AUTO_INCREMENT,char_ CHAR(4) NULL,enum_ ENUM('enum1', 'enum2', 'enum3') NULL,longtext_ LONGTEXT NULL,mediumtext_ MEDIUMTEXT NULL,set_ SET('set1', 'set2', 'set3') NULL,text_ TEXT NULL,tinytext_ TINYTEXT NULL,varchar_ VARCHAR(30) NULL,bit_ BIT NULL,bigint_ BIGINT NULL,boolean_ BOOLEAN NULL,decimal_ DECIMAL NULL,double_ DOUBLE NULL,float_ FLOAT NULL,int_ INT NULL,mediumint_ MEDIUMINT NULL,real_ REAL NULL,smallint_ SMALLINT NULL,tinyint_ TINYINT NULL,date_ DATE NULL,datetime_ DATETIME NULL,timestamp_ TIMESTAMP NULL,time_ TIME NULL,year_ YEAR NULL,blob_ BLOB NULL,mediumblob_ MEDIUMBLOB NULL,longblob_ LONGBLOB NULL,tinyblob_ TINYBLOB NULL,json_ JSON NULL,PRIMARY KEY(id)) + +mysql> INSERT INTO test.test_tb(id, varchar_) VALUES (1, 'hello world'); +mysql> INSERT INTO test.test_tb(id, varchar_) VALUES (2, ' aaa'); +mysql> INSERT INTO test.test_tb(id, varchar_) VALUES (3, 'aaa '); +mysql> INSERT INTO test.test_tb(id, varchar_) VALUES (4, ' aaa aaa '); +mysql> INSERT INTO test.test_tb(id, varchar_) VALUES (5, ' 平凯 星辰 '); + +mysql> analyze table test.test_tb +mysql> alter table test.test_tb set tiflash replica 1 +func> wait_table test test_tb + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where trim(varchar_) = 'hello world'; ++----+ +| id | ++----+ +| 1 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where trim(varchar_) = 'aaa'; ++----+ +| id | ++----+ +| 2 | +| 3 | ++----+ + + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where trim(varchar_) = 'aaa aaa'; ++----+ +| id | ++----+ +| 4 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where trim(varchar_) = '平凯 星辰'; ++----+ +| id | ++----+ +| 5 | ++----+ + +mysql> select /*+ read_from_storage(tiflash[test.test_tb]) */ id from test.test_tb where isnull(trim(char_)); ++----+ +| id | ++----+ +| 1 | +| 2 | +| 3 | +| 4 | +| 5 | ++----+ \ No newline at end of file