Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: format throw data truncated error (#5272) #5340

Merged
merged 20 commits into from
Jul 14, 2022
Merged

fix: format throw data truncated error (#5272) #5340

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
16e31e5
fix
xzhangxian1008 Jul 1, 2022
d974bf0
update
xzhangxian1008 Jul 1, 2022
1e3769f
update
xzhangxian1008 Jul 5, 2022
b602589
format
xzhangxian1008 Jul 5, 2022
d91dcca
update
xzhangxian1008 Jul 5, 2022
6ee5986
format
xzhangxian1008 Jul 5, 2022
af0711a
enable ut
xzhangxian1008 Jul 8, 2022
8613092
update
xzhangxian1008 Jul 9, 2022
832e2bc
update
xzhangxian1008 Jul 9, 2022
08e004c
udpate
xzhangxian1008 Jul 11, 2022
a343c2a
add fullstack test
xzhangxian1008 Jul 11, 2022
4dadfca
clang-tidy
xzhangxian1008 Jul 14, 2022
93707d3
fix
xzhangxian1008 Jul 14, 2022
554f6e2
fix
xzhangxian1008 Jul 14, 2022
372d1be
update
xzhangxian1008 Jul 14, 2022
5e1b152
fix
xzhangxian1008 Jul 14, 2022
b2db960
fix
xzhangxian1008 Jul 14, 2022
f08c0bc
fix
xzhangxian1008 Jul 14, 2022
adcfec8
fix
xzhangxian1008 Jul 14, 2022
27362a2
Merge branch 'release-5.4' of https://github.com/pingcap/tiflash into…
xzhangxian1008 Jul 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
221 changes: 112 additions & 109 deletions dbms/src/Functions/FunctionsString.cpp
View file
Open in desktop

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions dbms/src/Functions/FunctionsString.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ struct LowerUpperUTF8Impl
ColumnString::Chars_t & res_data,
ColumnString::Offsets & res_offsets);

static void vector_fixed(const ColumnString::Chars_t & data, size_t n, ColumnString::Chars_t & res_data);
static void vectorFixed(const ColumnString::Chars_t & data, size_t n, ColumnString::Chars_t & res_data);

static void constant(const std::string & data, std::string & res_data);

Expand Down Expand Up @@ -183,7 +183,7 @@ class FunctionStringToString : public IFunction
else if (const ColumnFixedString * col = checkAndGetColumn<ColumnFixedString>(column.get()))
{
auto col_res = ColumnFixedString::create(col->getN());
Impl::vector_fixed(col->getChars(), col->getN(), col_res->getChars());
Impl::vectorFixed(col->getChars(), col->getN(), col_res->getChars());
block.getByPosition(result).column = std::move(col_res);
}
else
Expand All @@ -203,7 +203,7 @@ struct TiDBLowerUpperUTF8Impl
ColumnString::Chars_t & res_data,
ColumnString::Offsets & res_offsets);

static void vector_fixed(const ColumnString::Chars_t & data, size_t n, ColumnString::Chars_t & res_data);
static void vectorFixed(const ColumnString::Chars_t & data, size_t n, ColumnString::Chars_t & res_data);

static void constant(const std::string & data, std::string & res_data);

Expand All @@ -228,7 +228,7 @@ struct TiDBLowerUpperBinaryImpl
throw Exception("the TiDB function of lower or upper for binary should do noting.");
}

static void vector_fixed(const ColumnString::Chars_t &, size_t, ColumnString::Chars_t &)
static void vectorFixed(const ColumnString::Chars_t &, size_t, ColumnString::Chars_t &)
{
throw Exception("the TiDB function of lower or upper for binary should do noting.");
}
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Functions/FunctionsURL.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ void DecodeURLComponentImpl::vector(const ColumnString::Chars_t & data, const Co
}


void DecodeURLComponentImpl::vector_fixed(const ColumnString::Chars_t &, size_t, ColumnString::Chars_t &)
void DecodeURLComponentImpl::vectorFixed(const ColumnString::Chars_t &, size_t, ColumnString::Chars_t &)
{
throw Exception("Column of type FixedString is not supported by URL functions", ErrorCodes::ILLEGAL_COLUMN);
}
Expand Down
6 changes: 3 additions & 3 deletions dbms/src/Functions/FunctionsURL.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -960,7 +960,7 @@ struct ExtractSubstringImpl
res_data.assign(start, length);
}

static void vector_fixed(const ColumnString::Chars_t &, size_t, ColumnString::Chars_t &)
static void vectorFixed(const ColumnString::Chars_t &, size_t, ColumnString::Chars_t &)
{
throw Exception("Column of type FixedString is not supported by URL functions", ErrorCodes::ILLEGAL_COLUMN);
}
Expand Down Expand Up @@ -1018,7 +1018,7 @@ struct CutSubstringImpl
res_data.append(start + length, data.data() + data.size());
}

static void vector_fixed(const ColumnString::Chars_t &, size_t, ColumnString::Chars_t &)
static void vectorFixed(const ColumnString::Chars_t &, size_t, ColumnString::Chars_t &)
{
throw Exception("Column of type FixedString is not supported by URL functions", ErrorCodes::ILLEGAL_COLUMN);
}
Expand All @@ -1033,7 +1033,7 @@ struct DecodeURLComponentImpl
static void constant(const std::string & data,
std::string & res_data);

static void vector_fixed(const ColumnString::Chars_t & data, size_t n, ColumnString::Chars_t & res_data);
static void vectorFixed(const ColumnString::Chars_t & data, size_t n, ColumnString::Chars_t & res_data);
};

} // namespace DB
9 changes: 6 additions & 3 deletions dbms/src/Functions/tests/gtest_strings_format.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class StringFormat : public DB::tests::FunctionTest
using FieldType = DecimalField<Decimal>;
using NullableDecimal = Nullable<Decimal>;
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"0.0000", "-0.0120", "0.0120", "12,332.1000", "12,332", "12,332", "12,332.300000000000000000000000000000", "-12,332.30000", "-1,000.0", "-333.33", {}}),
createColumn<Nullable<String>>({"0.0000", "-0.0120", "0.0120", "12,332.1000", "12,332", "12,332", "12,332.300000000000000000000000000000", "-12,332.30000", "-1,000.0", "-333.33", {}, "99,999.9999000000", "100,000.000", "100,000"}),
executeFunction(
func_name,
createColumn<NullableDecimal>(
Expand All @@ -35,8 +35,11 @@ class StringFormat : public DB::tests::FunctionTest
FieldType(static_cast<Native>(-123323000), 4),
FieldType(static_cast<Native>(-9999999), 4),
FieldType(static_cast<Native>(-3333330), 4),
FieldType(static_cast<Native>(0), 0)}),
createColumn<Nullable<Int64>>({4, 4, 4, 4, 0, -1, 31, 5, 1, 2, {}})));
FieldType(static_cast<Native>(0), 0),
FieldType(static_cast<Native>(999999999), 4),
FieldType(static_cast<Native>(999999999), 4),
FieldType(static_cast<Native>(999999999), 4)}),
createColumn<Nullable<Int64>>({4, 4, 4, 4, 0, -1, 31, 5, 1, 2, {}, 10, 3, -5})));
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"12,332.100", "-12,332.300", "-1,000.000", "-333.333"}),
executeFunction(
Expand Down
49 changes: 49 additions & 0 deletions tests/fullstack-test/expr/format.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,52 @@ int_val
1,234.000

mysql> drop table if exists test.t

mysql> create table test.t(id int, value decimal(65,4))
mysql> alter table test.t set tiflash replica 1
mysql> insert into test.t values(1,9999999999999999999999999999999999999999999999999999999999999.9999)

func> wait_table test t

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,-3) as result from test.t
result
10,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,0) as result from test.t
result
10,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,3) as result from test.t
result
10,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000.000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,10) as result from test.t
result
9,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999.9999000000


mysql> drop table if exists test.t

mysql> create table test.t(id int, value decimal(7,4))
mysql> alter table test.t set tiflash replica 1
mysql> insert into test.t values(1,999.9999)

func> wait_table test t

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,-2) as result from test.t
result
1,000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,0) as result from test.t
result
1,000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,2) as result from test.t
result
1,000.00

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,10) as result from test.t
result
999.9999000000

mysql> drop table if exists test.t