Skip to content

Commit

Permalink
Support cast string as double push down (#2038) (#2068)
Browse files Browse the repository at this point in the history 
* cherry pick #2038 to release-5.0

Signed-off-by: ti-srebot <[email protected]>

* fix conflict

Co-authored-by: Yu Lei <[email protected]>
  • Loading branch information
@ti-srebot @leiysky
ti-srebot and leiysky authored Jun 17, 2021
1 parent 31f3a23 commit 31d63d6
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 22 deletions.
10 changes: 4 additions & 6 deletions dbms/src/Flash/Coprocessor/DAGUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -515,8 +515,7 @@ std::unordered_map<tipb::ScalarFuncSig, String> scalar_func_map({
//{tipb::ScalarFuncSig::CastIntAsDuration, "cast"},
//{tipb::ScalarFuncSig::CastIntAsJson, "cast"},

{tipb::ScalarFuncSig::CastRealAsInt, "tidb_cast"},
{tipb::ScalarFuncSig::CastRealAsReal, "tidb_cast"},
{tipb::ScalarFuncSig::CastRealAsInt, "tidb_cast"}, {tipb::ScalarFuncSig::CastRealAsReal, "tidb_cast"},
{tipb::ScalarFuncSig::CastRealAsString, "tidb_cast"}, {tipb::ScalarFuncSig::CastRealAsDecimal, "tidb_cast"},
{tipb::ScalarFuncSig::CastRealAsTime, "tidb_cast"},
//{tipb::ScalarFuncSig::CastRealAsDuration, "cast"},
Expand All @@ -529,17 +528,16 @@ std::unordered_map<tipb::ScalarFuncSig, String> scalar_func_map({
//{tipb::ScalarFuncSig::CastDecimalAsDuration, "cast"},
//{tipb::ScalarFuncSig::CastDecimalAsJson, "cast"},

{tipb::ScalarFuncSig::CastStringAsInt, "tidb_cast"},
//{tipb::ScalarFuncSig::CastStringAsReal, "cast"},
{tipb::ScalarFuncSig::CastStringAsInt, "tidb_cast"}, {tipb::ScalarFuncSig::CastStringAsReal, "tidb_cast"},
{tipb::ScalarFuncSig::CastStringAsString, "tidb_cast"}, {tipb::ScalarFuncSig::CastStringAsDecimal, "tidb_cast"},
{tipb::ScalarFuncSig::CastStringAsTime, "tidb_cast"},
//{tipb::ScalarFuncSig::CastStringAsDuration, "cast"},
//{tipb::ScalarFuncSig::CastStringAsJson, "cast"},

{tipb::ScalarFuncSig::CastTimeAsInt, "tidb_cast"},
//{tipb::ScalarFuncSig::CastTimeAsReal, "tidb_cast"},
//{tipb::ScalarFuncSig::CastTimeAsString, "tidb_cast"},
{tipb::ScalarFuncSig::CastTimeAsDecimal, "tidb_cast"}, {tipb::ScalarFuncSig::CastTimeAsTime, "tidb_cast"},
{tipb::ScalarFuncSig::CastTimeAsString, "tidb_cast"}, {tipb::ScalarFuncSig::CastTimeAsDecimal, "tidb_cast"},
{tipb::ScalarFuncSig::CastTimeAsTime, "tidb_cast"},
//{tipb::ScalarFuncSig::CastTimeAsDuration, "cast"},
//{tipb::ScalarFuncSig::CastTimeAsJson, "cast"},

Expand Down
6 changes: 3 additions & 3 deletions dbms/src/Functions/FunctionsTiDBConversion.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
namespace DB
{

StringRef trim(const StringRef & value)
String trim(const StringRef & value)
{
StringRef ret;
ret.size = 0;
Expand All @@ -22,10 +22,10 @@ StringRef trim(const StringRef & value)
break;
}
if (start >= end)
return ret;
return ret.toString();
ret.data = value.data + start;
ret.size = end - start;
return ret;
return ret.toString();
}

void registerFunctionsTiDBConversion(FunctionFactory & factory) { factory.registerFunction<FunctionBuilderTiDBCast>(); }
Expand Down
34 changes: 21 additions & 13 deletions dbms/src/Functions/FunctionsTiDBConversion.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
namespace DB
{

StringRef trim(const StringRef & value);
String trim(const StringRef & value);

enum CastError
{
Expand Down Expand Up @@ -394,14 +394,14 @@ struct TiDBConvertToInteger
static T strToInt(const StringRef & value, const Context & context)
{
// trim space
StringRef trim_string = trim(value);
if (trim_string.size == 0)
String trim_string = trim(value);
if (trim_string.size() == 0)
{
if (value.size != 0)
context.getDAGContext()->handleTruncateError("cast str as int");
return static_cast<T>(0);
}
StringRef int_string = getValidIntPrefix(trim_string);
StringRef int_string = getValidIntPrefix(StringRef(trim_string));
if (int_string.size == 0)
{
if (value.size != 0)
Expand Down Expand Up @@ -590,8 +590,7 @@ struct TiDBConvertToFloat
}

template <typename T>
static std::enable_if_t<std::is_floating_point_v<T> || std::is_integral_v<T>, Float64> toFloat(
const T & value)
static std::enable_if_t<std::is_floating_point_v<T> || std::is_integral_v<T>, Float64> toFloat(const T & value)
{
return static_cast<Float64>(value);
}
Expand Down Expand Up @@ -652,14 +651,24 @@ struct TiDBConvertToFloat

static Float64 strToFloat(const StringRef & value, bool need_truncate, Float64 shift, Float64 max_f, const Context & context)
{
StringRef trim_string = trim(value);
StringRef float_string = getValidFloatPrefix(trim_string);
if (trim_string.size == 0 && value.size != 0)
String trim_string = trim(value);
StringRef float_string = getValidFloatPrefix(StringRef(trim_string));
if (trim_string.size() == 0 && value.size != 0)
{
context.getDAGContext()->handleTruncateError("cast str as real");
context.getDAGContext()->handleTruncateError("Truncated incorrect DOUBLE value");
return 0.0;
}
Float64 f = strtod(float_string.data, nullptr);
if (f == std::numeric_limits<Float64>::infinity())
{
context.getDAGContext()->handleOverflowError("Truncated incorrect DOUBLE value");
return std::numeric_limits<Float64>::max();
}
if (f == -std::numeric_limits<double>::infinity())
{
context.getDAGContext()->handleOverflowError("Truncated incorrect DOUBLE value");
return -std::numeric_limits<Float64>::max();
}
return produceTargetFloat64(f, need_truncate, shift, max_f, context);
}

Expand Down Expand Up @@ -713,17 +722,16 @@ struct TiDBConvertToFloat
MyDateTime date_time(vec_from[i]);
if (type.getFraction() > 0)
vec_to[i] = toFloat(date_time.year * 10000000000ULL + date_time.month * 100000000ULL + date_time.day * 100000
+ date_time.hour * 1000 + date_time.minute * 100 + date_time.second + date_time.micro_second / 1000000.0);
+ date_time.hour * 1000 + date_time.minute * 100 + date_time.second + date_time.micro_second / 1000000.0);
else
vec_to[i] = toFloat(date_time.year * 10000000000ULL + date_time.month * 100000000ULL + date_time.day * 100000
+ date_time.hour * 1000 + date_time.minute * 100 + date_time.second);
+ date_time.hour * 1000 + date_time.minute * 100 + date_time.second);
}
}
}
else if constexpr (std::is_same_v<FromDataType, DataTypeString>)
{
/// cast string as real
/// the implementation is quite different from TiDB/TiKV, so cast string as float will not be pushed to TiFlash
const IColumn * col_from = block.getByPosition(arguments[0]).column.get();
const ColumnString * col_from_string = checkAndGetColumn<ColumnString>(col_from);
const ColumnString::Chars_t * chars = &col_from_string->getChars();
Expand Down
29 changes: 29 additions & 0 deletions tests/fullstack-test/expr/cast_string_as_real.test
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
mysql> drop table if exists test.t
mysql> create table test.t(a char(30))
mysql> alter table test.t set tiflash replica 1
mysql> insert into test.t values ('1.23'),('123'),('-123.99'),('+123.123-'),(0),(0.0),(NULL),('1.11.00'),('11xx'),('11.xx'),('xx.11'),('1e649'),('-1e649'),('9.9999999999999999'),('9.999999999999999')

func> wait_table test t

mysql> set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; select a, b from (select a, cast(a as double) as b from test.t) t group by a, b order by a
+--------------------+-------------------------+
| a | b |
+--------------------+-------------------------+
| NULL | NULL |
| +123.123- | 123.123 |
| -123.99 | -123.99 |
| -1e649 | -1.7976931348623157e308 |
| 0 | 0 |
| 0.0 | 0 |
| 1.11.00 | 1.11 |
| 1.23 | 1.23 |
| 11.xx | 11 |
| 11xx | 11 |
| 123 | 123 |
| 1e649 | 1.7976931348623157e308 |
| 9.999999999999999 | 9.999999999999998 |
| 9.9999999999999999 | 10 |
| xx.11 | 0 |
+--------------------+-------------------------+

mysql> drop table if exists test.t

0 comments on commit 31d63d6

Please sign in to comment.