Skip to content

Commit

Permalink
Merge branch 'master' into ut/daymonthyear
Browse files Browse the repository at this point in the history
  • Loading branch information
@bestwoody
bestwoody authored Dec 24, 2021
2 parents e029526 + a73ceff commit cf71742
Show file tree
Hide file tree
Showing 5 changed files with 205 additions and 11 deletions.
3 changes: 2 additions & 1 deletion dbms/src/Flash/Coprocessor/DAGContext.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,8 @@ class DAGContext
}

const tipb::DAGRequest * dag_request;
Int64 compile_time_ns;
Int64 compile_time_ns = 0;
size_t final_concurrency = 1;
bool has_read_wait_index = false;
Clock::time_point read_wait_index_start_timestamp{Clock::duration::zero()};
Clock::time_point read_wait_index_end_timestamp{Clock::duration::zero()};
Expand Down
3 changes: 2 additions & 1 deletion dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1099,6 +1099,7 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline)
"execution stream size for query block(before aggregation) {} is {}",
query_block.qb_column_prefix,
pipeline.streams.size());
dagContext().final_concurrency = std::max(dagContext().final_concurrency, pipeline.streams.size());

if (res.before_aggregation)
{
Expand Down Expand Up @@ -1220,7 +1221,7 @@ void DAGQueryBlockInterpreter::executeExchangeSender(DAGPipeline & pipeline)

void DAGQueryBlockInterpreter::restorePipelineConcurrency(DAGPipeline & pipeline)
{
restoreConcurrency(pipeline, max_streams, taskLogger());
restoreConcurrency(pipeline, dagContext().final_concurrency, taskLogger());
}

BlockInputStreams DAGQueryBlockInterpreter::execute()
Expand Down
119 changes: 111 additions & 8 deletions dbms/src/Functions/tests/gtest_string_lrtrim.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,18 +113,121 @@ try

// ltrim(column)
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"xx aa", "xxaa xx ", "\t aa \t", "", {}}),
executeFunction("tidbLTrim", createColumn<Nullable<String>>({" xx aa", " xxaa xx ", "\t aa \t", "", {}})));
createColumn<Nullable<String>>({"xx aa", "xxaa xx ", "\t aa \t", "", {}, "", "\n\t"}),
executeFunction("tidbLTrim", createColumn<Nullable<String>>({" xx aa", " xxaa xx ", "\t aa \t", "", {}, " ", "\n\t"})));
ASSERT_COLUMN_EQ(
createColumn<String>({"xx aa", "xxaa xx ", "\t aa \t", "", {}}),
executeFunction("tidbLTrim", createColumn<String>({" xx aa", " xxaa xx ", "\t aa \t", "", {}})));
createColumn<String>({"xx aa", "xxaa xx ", "\t aa \t", "", {}, "", "\n\t"}),
executeFunction("tidbLTrim", createColumn<String>({" xx aa", " xxaa xx ", "\t aa \t", "", {}, " ", "\n\t"})));
// rtrim(column)
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({" xx aa", " xxaa xx", "\t aa \t", "", {}}),
executeFunction("tidbRTrim", createColumn<Nullable<String>>({" xx aa", " xxaa xx ", "\t aa \t", "", {}})));
createColumn<Nullable<String>>({" xx aa", " xxaa xx", "\t aa \t", "", {}, "", "\n\t"}),
executeFunction("tidbRTrim", createColumn<Nullable<String>>({" xx aa", " xxaa xx ", "\t aa \t", "", {}, " ", "\n\t"})));
ASSERT_COLUMN_EQ(
createColumn<String>({" xx aa", " xxaa xx", "\t aa \t", "", {}}),
executeFunction("tidbRTrim", createColumn<String>({" xx aa", " xxaa xx ", "\t aa \t", "", {}})));
createColumn<String>({" xx aa", " xxaa xx", "\t aa \t", "", {}, "", "\n\t"}),
executeFunction("tidbRTrim", createColumn<String>({" xx aa", " xxaa xx ", "\t aa \t", "", {}, " ", "\n\t"})));

// NULL cases
ASSERT_COLUMN_EQ(
createOnlyNullColumnConst(5),
executeFunction("tidbLTrim", createOnlyNullColumnConst(5)));
ASSERT_COLUMN_EQ(
createOnlyNullColumnConst(5),
executeFunction("tidbRTrim", createOnlyNullColumnConst(5)));
ASSERT_COLUMN_EQ(
createOnlyNullColumnConst(5),
executeFunction("tidbLTrim", createOnlyNullColumn(5)));
ASSERT_COLUMN_EQ(
createOnlyNullColumnConst(5),
executeFunction("tidbRTrim", createOnlyNullColumn(5)));

ASSERT_COLUMN_EQ(
createConstColumn<Nullable<String>>(5, {}),
executeFunction("tidbLTrim", createConstColumn<Nullable<String>>(5, {})));
ASSERT_COLUMN_EQ(
createConstColumn<Nullable<String>>(5, {}),
executeFunction("tidbRTrim", createConstColumn<Nullable<String>>(5, {})));
ASSERT_COLUMN_EQ(
createConstColumn<String>(5, {}),
executeFunction("tidbLTrim", createConstColumn<String>(5, {})));
ASSERT_COLUMN_EQ(
createConstColumn<String>(5, {}),
executeFunction("tidbRTrim", createConstColumn<String>(5, {})));

// ltrim(column) Nullable<String> ASCII group and non-ASCII group
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"a", "b", "c ", "d ", "e f", "g h", "i j ", "k l "}),
executeFunction("tidbLTrim", createColumn<Nullable<String>>({"a", " b", "c ", " d ", "e f", " g h", "i j ", " k l "})));
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"你", "好", "平 ", "凯 ", "星 辰", "啊 波", "次 得 ", "额 佛 "}),
executeFunction("tidbLTrim", createColumn<Nullable<String>>({"你", " 好", "平 ", " 凯 ", "星 辰", " 啊 波", "次 得 ", " 额 佛 "})));
// ltrim(column) String ASCII group and non-ASCII group
ASSERT_COLUMN_EQ(
createColumn<String>({"a", "b", "c ", "d ", "e f", "g h", "i j ", "k l "}),
executeFunction("tidbLTrim", createColumn<String>({"a", " b", "c ", " d ", "e f", " g h", "i j ", " k l "})));
ASSERT_COLUMN_EQ(
createColumn<String>({"你", "好", "平 ", "凯 ", "星 辰", "啊 波", "次 得 ", "额 佛 "}),
executeFunction("tidbLTrim", createColumn<String>({"你", " 好", "平 ", " 凯 ", "星 辰", " 啊 波", "次 得 ", " 额 佛 "})));
// rtrim(column) Nullable<String> ASCII group and non-ASCII group
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"a", " b", "c", " d", "e f", " g h", "i j", " k l"}),
executeFunction("tidbRTrim", createColumn<Nullable<String>>({"a", " b", "c ", " d ", "e f", " g h", "i j ", " k l "})));
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"你", " 好", "平", " 凯", "星 辰", " 啊 波", "次 得", " 额 佛"}),
executeFunction("tidbRTrim", createColumn<Nullable<String>>({"你", " 好", "平 ", " 凯 ", "星 辰", " 啊 波", "次 得 ", " 额 佛 "})));
// rtrim(column) String ASCII group and non-ASCII group
ASSERT_COLUMN_EQ(
createColumn<String>({"a", " b", "c", " d", "e f", " g h", "i j", " k l"}),
executeFunction("tidbRTrim", createColumn<String>({"a", " b", "c ", " d ", "e f", " g h", "i j ", " k l "})));
ASSERT_COLUMN_EQ(
createColumn<String>({"你", " 好", "平", " 凯", "星 辰", " 啊 波", "次 得", " 额 佛"}),
executeFunction("tidbRTrim", createColumn<String>({"你", " 好", "平 ", " 凯 ", "星 辰", " 啊 波", "次 得 ", " 额 佛 "})));

// const cases
InferredDataInitializerList<String> inputs[] = {
{"", "/n/t"}, // corner cases
{"a", " b", "c ", " d ", "e f", " g h", "i j ", " k l "}, // ASCII
{"你", " 好", "平 ", " 凯 ", "星 辰", " 啊 波", "次 得 ", " 额 佛 "}}; // non-ASCII

InferredDataInitializerList<String> results_ltrim[] = {
{"", "/n/t"}, // corner cases
{"a", "b", "c ", "d ", "e f", "g h", "i j ", "k l "}, // ASCII
{"你", "好", "平 ", "凯 ", "星 辰", "啊 波", "次 得 ", "额 佛 "}}; // non-ASCII

InferredDataInitializerList<String> results_rtrim[] = {
{"", "/n/t"}, // corner cases
{"a", " b", "c", " d", "e f", " g h", "i j", " k l"}, // ASCII
{"你", " 好", "平", " 凯", "星 辰", " 啊 波", "次 得", " 额 佛"}}; // non-ASCII


//const
for (int i = 0; i < 3; i++)
{
auto & input = inputs[i];
auto & result_ltrim = results_ltrim[i];
auto & result_rtrim = results_rtrim[i];
int cnt = 0;
for (auto input_iter = input.begin(), lres_iter = result_ltrim.begin(), rres_iter = result_rtrim.begin();
input_iter != input.end() && lres_iter != result_ltrim.end() && rres_iter != result_rtrim.end();
input_iter++, lres_iter++, rres_iter++)
{
ASSERT_COLUMN_EQ(
createConstColumn<Nullable<String>>(5, *lres_iter),
executeFunction("tidbLTrim", createConstColumn<Nullable<String>>(5, *input_iter)));
ASSERT_COLUMN_EQ(
createConstColumn<Nullable<String>>(5, *rres_iter),
executeFunction("tidbRTrim", createConstColumn<Nullable<String>>(5, *input_iter)));
ASSERT_COLUMN_EQ(
createConstColumn<String>(5, *lres_iter),
executeFunction("tidbLTrim", createConstColumn<String>(5, *input_iter)));
ASSERT_COLUMN_EQ(
createConstColumn<String>(5, *rres_iter),
executeFunction("tidbRTrim", createConstColumn<String>(5, *input_iter)));
cnt++;
}
ASSERT_EQ(cnt, input.size());
ASSERT_EQ(cnt, result_ltrim.size());
ASSERT_EQ(cnt, result_rtrim.size());
}
}
CATCH

Expand Down
77 changes: 77 additions & 0 deletions dbms/src/Functions/tests/gtest_strings_trim.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -875,6 +875,83 @@ try
createConstColumn<Nullable<String>>(5, "xxax x"),
createColumn<Nullable<String>>({"x", "xx", "xxa", " x", {}}),
createConstColumn<Nullable<Int8>>(5, 3)));

//different trim policy
for (int i = 0; i < 3; i++)
{
//test NULL and "" case
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"", "", "", {}, "", "", ""}),
executeFunction("tidbTrim",
createColumn<Nullable<String>>({"", "", "", "", "", "", ""}),
createColumn<Nullable<String>>({"", "x", "xx", {}, "啊", "\t", " "}),
createConstColumn<Nullable<Int8>>(7, i)));
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({{}, {}, {}, {}, {}, {}, {}}),
executeFunction("tidbTrim",
createColumn<Nullable<String>>({{}, {}, {}, {}, {}, {}, {}}),
createColumn<Nullable<String>>({"", "x", "xx", {}, "啊", "\t", " "}),
createConstColumn<Nullable<Int8>>(7, i)));

//test repeated pattern: ASCII & non-ASCII
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"", "x", "", "x", "", "x", {}}),
executeFunction("tidbTrim",
createColumn<Nullable<String>>({"", "x", "xx", "xxx", "xxxx", "xxxxx", {}}),
createColumn<Nullable<String>>({"xx", "xx", "xx", "xx", "xx", "xx", "xx"}),
createConstColumn<Nullable<Int8>>(7, i)));


ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"", "啊", "", "啊", "", "啊"}),
executeFunction("tidbTrim",
createColumn<Nullable<String>>({"", "啊", "啊啊", "啊啊啊", "啊啊啊啊", "啊啊啊啊啊"}),
createColumn<Nullable<String>>({"啊啊", "啊啊", "啊啊", "啊啊", "啊啊", "啊啊"}),
createConstColumn<Nullable<Int8>>(6, i)));
}

//test non-ASCII cases
InferredDataInitializerList<Nullable<String>> results_columns_ws_with_core_text[] = {
{" 波 波 ", "啊 波 波 啊", " 波 波 ", "啊 波 波 啊", " 波 波 "}, //default(both)
{" 波 波 ", "啊 波 波 啊", " 波 波 ", "啊 波 波 啊", " 波 波 "}, //both
{" 波 波 ", "啊 波 波 啊", " 波 波 啊啊", "啊 波 波 啊啊啊", " 波 波 啊啊啊啊"}, //left
{" 波 波 ", "啊 波 波 啊", "啊啊 波 波 ", "啊啊啊 波 波 啊", "啊啊啊啊 波 波 "} //right
};

InferredDataInitializerList<Nullable<String>> input_columns_ws_with_core_text = {" 波 波 ", "啊 波 波 啊", "啊啊 波 波 啊啊", "啊啊啊 波 波 啊啊啊", "啊啊啊啊 波 波 啊啊啊啊"};

//different trim policy
for (int i = 0; i < 3; i++)
{
//non-const
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>(results_columns_ws_with_core_text[i]),
executeFunction("tidbTrim",
createColumn<Nullable<String>>(input_columns_ws_with_core_text),
createColumn<Nullable<String>>({"啊啊", "啊啊", "啊啊", "啊啊", "啊啊", "啊啊"}),
createConstColumn<Nullable<Int8>>(6, i)));

//const
for (size_t j = 0; j < results_columns_ws_with_core_text[i].size(); j++)
{
auto input_itr = input_columns_ws_with_core_text.begin();
size_t cnt = 0;
for (auto res_itr = results_columns_ws_with_core_text[i].begin();
res_itr != results_columns_ws_with_core_text[i].end() && input_itr != input_columns_ws_with_core_text.end();
res_itr++, input_itr++)
{
ASSERT_COLUMN_EQ(
createConstColumn<Nullable<String>>(5, *res_itr),
executeFunction("tidbTrim",
createConstColumn<Nullable<String>>(5, *input_itr),
createConstColumn<Nullable<String>>(5, "啊啊"),
createConstColumn<Nullable<Int8>>(5, i)));
cnt++;
}
ASSERT_EQ(cnt, input_columns_ws_with_core_text.size());
ASSERT_EQ(cnt, results_columns_ws_with_core_text[i].size());
}
}
}
CATCH

Expand Down
14 changes: 13 additions & 1 deletion libs/libcommon/include/common/logger_useful.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,15 @@ inline constexpr size_t getFileNameOffset(T (&/*str*/)[1])
{
return 0;
}

template <typename S, typename Ignored, typename... Args>
std::string toCheckedFmtStr(const S & format, const Ignored &, Args &&... args)
{
// The second arg is the same as `format`, just ignore
// Apply `make_args_checked` for checks `format` validity at compile time.
// https://fmt.dev/latest/api.html#argument-lists
return fmt::vformat(format, fmt::make_args_checked<Args...>(format, args...));
}
} // namespace LogFmtDetails


Expand All @@ -102,13 +111,16 @@ inline constexpr size_t getFileNameOffset(T (&/*str*/)[1])
/// and the latter arguments treat as values to substitute.
/// If only one argument is provided, it is threat as message without substitutions.

#define LOG_GET_FIRST_ARG(arg, ...) arg
#define LOG_FMT_IMPL(logger, PRIORITY, ...) \
do \
{ \
if ((logger)->is((PRIORITY))) \
{ \
std::string formatted_message = LogFmtDetails::numArgs(__VA_ARGS__) > 1 \
? fmt::format(__VA_ARGS__) \
? LogFmtDetails::toCheckedFmtStr( \
FMT_STRING(LOG_GET_FIRST_ARG(__VA_ARGS__)), \
__VA_ARGS__) \
: LogFmtDetails::firstArg(__VA_ARGS__); \
Poco::Message poco_message( \
/*source*/ (logger)->name(), \
Expand Down

0 comments on commit cf71742

Please sign in to comment.