Fix cast decimal as string

dbms/src/DataTypes/DataTypeMyDateTime.h

@@ -8,6 +8,7 @@ namespace DB
 
 class DataTypeMyDateTime final : public DataTypeMyTimeBase
 {
+public:
     int fraction;
 
 public:

dbms/src/Debug/dbgFuncCoprocessor.cpp

@@ -274,6 +274,22 @@ std::unordered_map<String, tipb::ScalarFuncSig> func_name_to_sig({
     {"cast_decimal_decimal", tipb::ScalarFuncSig::CastDecimalAsDecimal},
     {"cast_time_decimal", tipb::ScalarFuncSig::CastTimeAsDecimal},
     {"cast_string_decimal", tipb::ScalarFuncSig::CastStringAsDecimal},
+    {"cast_int_string", tipb::ScalarFuncSig::CastIntAsString},
+    {"cast_real_string", tipb::ScalarFuncSig::CastRealAsString},
+    {"cast_decimal_string", tipb::ScalarFuncSig::CastDecimalAsString},
+    {"cast_time_string", tipb::ScalarFuncSig::CastTimeAsString},
+    {"cast_string_string", tipb::ScalarFuncSig::CastStringAsString},
+    {"cast_int_date", tipb::ScalarFuncSig::CastIntAsTime},
+    {"cast_real_date", tipb::ScalarFuncSig::CastRealAsTime},
+    {"cast_decimal_date", tipb::ScalarFuncSig::CastDecimalAsTime},
+    {"cast_time_date", tipb::ScalarFuncSig::CastTimeAsTime},
+    {"cast_string_date", tipb::ScalarFuncSig::CastStringAsTime},
+    {"cast_int_datetime", tipb::ScalarFuncSig::CastIntAsTime},
+    {"cast_real_datetime", tipb::ScalarFuncSig::CastRealAsTime},
+    {"cast_decimal_datetime", tipb::ScalarFuncSig::CastDecimalAsTime},
+    {"cast_time_datetime", tipb::ScalarFuncSig::CastTimeAsTime},
+    {"cast_string_datetime", tipb::ScalarFuncSig::CastStringAsTime},
+
 });
 
 void compileExpr(const DAGSchema & input, ASTPtr ast, tipb::Expr * expr, std::unordered_set<String> & referred_columns,
@@ -399,6 +415,24 @@ void compileExpr(const DAGSchema & input, ASTPtr ast, tipb::Expr * expr, std::un
                 expr->set_sig(tipb::ScalarFuncSig::DateFormatSig);
                 expr->mutable_field_type()->set_tp(TiDB::TypeString);
                 break;
+            case tipb::ScalarFuncSig::CastIntAsTime:
+            case tipb::ScalarFuncSig::CastRealAsTime:
+            case tipb::ScalarFuncSig::CastTimeAsTime:
+            case tipb::ScalarFuncSig::CastDecimalAsTime:
+            case tipb::ScalarFuncSig::CastStringAsTime:
+            {
+                expr->set_sig(it_sig->second);
+                auto * ft = expr->mutable_field_type();
+                if (it_sig->first.find("datetime"))
+                {
+                    ft->set_tp(TiDB::TypeDatetime);
+                }
+                else
+                {
+                    ft->set_tp(TiDB::TypeDate);
+                }
+                break;
+            }
             default:
             {
                 expr->set_sig(it_sig->second);

dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp

@@ -431,6 +431,7 @@ String DAGExpressionAnalyzer::convertToUInt8(ExpressionActionsPtr & actions, con
     {
         /// use tidb_cast to make it compatible with TiDB
         tipb::FieldType field_type;
+        // TODO: Use TypeDouble as return type, to be compatible with TiDB
         field_type.set_tp(TiDB::TypeLongLong);
         tipb::Expr type_expr;
         constructStringLiteralTiExpr(type_expr, "Nullable(Int64)");

dbms/src/Flash/Coprocessor/DAGUtils.cpp

@@ -356,6 +356,17 @@ void constructInt64LiteralTiExpr(tipb::Expr & expr, Int64 value)
     field_type->set_flag(TiDB::ColumnFlagNotNull);
 }
 
+void constructFloat64LiteralTiExpr(tipb::Expr & expr, Float64 value)
+{
+    expr.set_tp(tipb::ExprType::Float64);
+    std::stringstream ss;
+    encodeDAGFloat64(value, ss);
+    expr.set_val(ss.str());
+    auto * field_type = expr.mutable_field_type();
+    field_type->set_tp(TiDB::TypeDouble);
+    field_type->set_flag(TiDB::ColumnFlagNotNull);
+}
+
 void constructDateTimeLiteralTiExpr(tipb::Expr & expr, UInt64 packed_value)
 {
     expr.set_tp(tipb::ExprType::MysqlTime);

dbms/src/Flash/Coprocessor/DAGUtils.h

@@ -30,6 +30,7 @@ String exprToString(const tipb::Expr & expr, const std::vector<NameAndTypePair>
 bool exprHasValidFieldType(const tipb::Expr & expr);
 void constructStringLiteralTiExpr(tipb::Expr & expr, const String & value);
 void constructInt64LiteralTiExpr(tipb::Expr & expr, Int64 value);
+void constructFloat64LiteralTiExpr(tipb::Expr & expr, Float64 value);
 void constructDateTimeLiteralTiExpr(tipb::Expr & expr, UInt64 packed_value);
 void constructNULLLiteralTiExpr(tipb::Expr & expr);
 extern std::unordered_map<tipb::ExprType, String> agg_func_map;

dbms/src/Functions/FunctionsConversion.h

@@ -438,6 +438,15 @@ struct FormatImpl<DataTypeMyDateTime>
     }
 };
 
+template <typename DecimalType>
+struct FormatImpl<DataTypeDecimal<DecimalType>>
+{
+    static void execute(const typename DataTypeDecimal<DecimalType>::FieldType v, WriteBuffer & wb, const DataTypeDecimal<DecimalType> * tp, const DateLUTImpl *)
+    {
+        writeText(v, tp->getScale(), wb);
+    }
+};
+
 template <typename FieldType>
 struct FormatImpl<DataTypeEnum<FieldType>>
 {

dbms/src/Functions/FunctionsTiDBConversion.h

@@ -101,7 +101,6 @@ struct TiDBConvertToString
         auto col_to = ColumnString::create();
         ColumnString::Chars_t & data_to = col_to->getChars();
         ColumnString::Offsets & offsets_to = col_to->getOffsets();
-        WriteBufferFromVector<ColumnString::Chars_t> write_buffer(data_to);
 
         if constexpr (std::is_same_v<FromDataType, DataTypeString>)
         {
@@ -113,6 +112,8 @@ struct TiDBConvertToString
 
             offsets_to.resize(size);
 
+            WriteBufferFromVector<ColumnString::Chars_t> write_buffer(data_to);
+
             size_t current_offset = 0;
             for (size_t i = 0; i < size; i++)
             {
@@ -148,10 +149,13 @@ struct TiDBConvertToString
             container_per_element.resize(decimal_max_prec);
             offsets_to.resize(size);
 
+            WriteBufferFromVector<ColumnString::Chars_t> write_buffer(data_to);
+
             for (size_t i = 0; i < size; ++i)
             {
                 WriteBufferFromVector<ColumnString::Chars_t> element_write_buffer(container_per_element);
-                FormatImpl<FromDataType>::execute(vec_from[i], element_write_buffer, &type, nullptr);
+                FormatImpl<FromDataType>::execute(
+                    vec_from[i], element_write_buffer, dynamic_cast<const DataTypeDecimal<FromFieldType> *>(&type), nullptr);
                 size_t byte_length = element_write_buffer.count();
                 if (tp.flen() > 0)
                     byte_length = std::min(byte_length, tp.flen());
@@ -189,6 +193,8 @@ struct TiDBConvertToString
             }
             offsets_to.resize(size);
 
+            WriteBufferFromVector<ColumnString::Chars_t> write_buffer(data_to);
+
             for (size_t i = 0; i < size; ++i)
             {
                 WriteBufferFromVector<ColumnString::Chars_t> element_write_buffer(container_per_element);
@@ -1172,6 +1178,20 @@ struct TiDBConvertToTime
         const auto & col_with_type_and_name = block.getByPosition(arguments[0]);
         const auto & type = static_cast<const FromDataType &>(*col_with_type_and_name.type);
 
+        // Use to truncate fraction part(microseconds) of DATE
+        // Result microseconds should be (micro - micro % fraction_trunc)
+        int fraction_trunc = 1;
+        if constexpr (std::is_same_v<ToDataType, DataTypeMyDateTime>)
+        {
+            auto * to_type = block.getByPosition(result).type.get();
+            if constexpr (return_nullable)
+            {
+                to_type = dynamic_cast<const DataTypeNullable *>(to_type)->getNestedType().get();
+            }
+            const auto * tp = dynamic_cast<const DataTypeMyDateTime *>(to_type);
+            fraction_trunc = std::pow(10, 6 - tp->fraction);
+        }
+
         if constexpr (return_nullable)
         {
             col_null_map_to = ColumnUInt8::create(size, 0);
@@ -1198,6 +1218,7 @@ struct TiDBConvertToTime
                     Field packed_uint_value = parseMyDateTime(string_value);
                     UInt64 packed_uint = packed_uint_value.template safeGet<UInt64>();
                     MyDateTime datetime(packed_uint);
+                    datetime.micro_second = datetime.micro_second - datetime.micro_second % fraction_trunc;
                     if constexpr (std::is_same_v<ToDataType, DataTypeMyDate>)
                     {
                         MyDate date(datetime.year, datetime.month, datetime.day);
@@ -1219,6 +1240,7 @@ struct TiDBConvertToTime
         }
         else if constexpr (std::is_same_v<FromDataType, DataTypeMyDate> || std::is_same_v<FromDataType, DataTypeMyDateTime>)
         {
+            // cast time as time
             const auto * col_from = checkAndGetColumn<ColumnUInt64>(block.getByPosition(arguments[0]).column.get());
             const ColumnUInt64::Container & vec_from = col_from->getData();
 
@@ -1233,6 +1255,7 @@ struct TiDBConvertToTime
                 }
                 else
                 {
+                    datetime.micro_second = datetime.micro_second - datetime.micro_second % fraction_trunc;
                     vec_to[i] = datetime.toPackedUInt();
                 }
             }
@@ -1257,6 +1280,7 @@ struct TiDBConvertToTime
                     }
                     else
                     {
+                        datetime.micro_second = datetime.micro_second - datetime.micro_second % fraction_trunc;
                         vec_to[i] = datetime.toPackedUInt();
                     }
                 }
@@ -1302,6 +1326,7 @@ struct TiDBConvertToTime
                         }
                         else
                         {
+                            datetime.micro_second = datetime.micro_second - datetime.micro_second % fraction_trunc;
                             vec_to[i] = packed_uint;
                         }
                     }
@@ -1334,6 +1359,7 @@ struct TiDBConvertToTime
                     }
                     else
                     {
+                        datetime.micro_second = datetime.micro_second - datetime.micro_second % fraction_trunc;
                         vec_to[i] = datetime.toPackedUInt();
                     }
                 }

tests/mutable-test/expr/cast_as_decimal.test

@@ -12,7 +12,6 @@
 => DBGInvoke __put_region(4, 0, 100, default, test)
 => DBGInvoke __raft_insert_row(default, test, 4, 50, 1, 1.234, 1.234, '123', '2020-09-15 01:00:00')
 
-# test date_format
 => DBGInvoke dag('select count(1) from default.test group by a, cast_int_decimal(a), cast_real_decimal(b), cast_decimal_decimal(c), cast_string_decimal(d), cast_time_decimal(e)', 4,'encode_type:chunk')
 ┌─count(1)─┬─a─┬─cast_int_decimal(a)─┬─cast_real_decimal(b)─┬─cast_decimal_decimal(c)─┬─cast_string_decimal(d)─┬─cast_time_decimal(e)─┐
 │        1 │ 1 │                   1 │                    1 │                       1 │                    123 │       20200901501000 │

tests/mutable-test/expr/cast_as_int.test

@@ -12,7 +12,6 @@
 => DBGInvoke __put_region(4, 0, 100, default, test)
 => DBGInvoke __raft_insert_row(default, test, 4, 50, 1, 1.234, 1.234, '123', '2020-09-15 01:00:00')
 
-# test date_format
 => DBGInvoke dag('select count(1) from default.test group by a, cast_int_int(a), cast_real_int(b), cast_decimal_int(c), cast_string_int(d), cast_time_int(e)', 4,'encode_type:chunk')
 ┌─count(1)─┬─a─┬─cast_int_int(a)─┬─cast_real_int(b)─┬─cast_decimal_int(c)─┬─cast_string_int(d)─┬─cast_time_int(e)─┐
 │        1 │ 1 │               1 │                1 │                   1 │                123 │   20200901501000 │

tests/mutable-test/expr/cast_as_string.test

@@ -0,0 +1,22 @@
+# Preparation.
+=> DBGInvoke __enable_schema_sync_service('true')
+
+=> DBGInvoke __drop_tidb_table(default, test)
+=> drop table if exists default.test
+
+=> DBGInvoke __set_flush_threshold(1000000, 1000000)
+
+# Data.
+=> DBGInvoke __mock_tidb_table(default, test, 'a int, b float, c decimal(4, 3), d char, e MyDatetime')
+=> DBGInvoke __refresh_schemas()
+=> DBGInvoke __put_region(4, 0, 100, default, test)
+=> DBGInvoke __raft_insert_row(default, test, 4, 50, 1, 1.234, 1.234, '123', '2020-09-15 01:00:00')
+
+=> DBGInvoke dag('select count(1) from default.test group by a, cast_int_string(a), cast_real_string(b), cast_decimal_string(c), cast_string_string(d)', 4,'encode_type:chunk')
+┌─count(1)─┬─a─┬─cast_int_string(a)─┬─cast_real_string(b)─┬─cast_decimal_string(c)─┬─cast_string_string(d)─┐
+│        1 │ 1 │                  1 │                   1 │                      1 │                   123 │
+└──────────┴───┴────────────────────┴─────────────────────┴────────────────────────┴───────────────────────┘
+
+# Clean up.
+=> DBGInvoke __drop_tidb_table(default, test)
+=> drop table if exists default.test

tests/mutable-test/expr/cast_as_time.test

@@ -0,0 +1,25 @@
+# Preparation.
+=> DBGInvoke __enable_schema_sync_service('true')
+
+=> DBGInvoke __drop_tidb_table(default, test)
+=> drop table if exists default.test
+
+=> DBGInvoke __set_flush_threshold(1000000, 1000000)
+
+# Data.
+=> DBGInvoke __mock_tidb_table(default, test, 'a int, b double, c decimal(20, 6), d char, e MyDatetime')
+=> DBGInvoke __refresh_schemas()
+=> DBGInvoke __put_region(4, 0, 100, default, test)
+=> DBGInvoke __raft_insert_row(default, test, 4, 50, 20201203, 20201203010000, 20201203010000, '2020-12-03 01:00:00', '2020-09-15 01:00:00')
+
+=> DBGInvoke dag('select count(1) from default.test group by a, cast_int_datetime(a), cast_real_datetime(b), cast_decimal_datetime(c), cast_string_datetime(d), cast_time_datetime(e)', 4,'encode_type:chunk')
+┌─count(1)─┬────────a─┬─cast_int_datetime(a)─┬─cast_real_datetime(b)─┬─cast_decimal_datetime(c)─┬─cast_string_datetime(d)─┬─cast_time_datetime(e)─┐
+│        1 │ 20201203 │  2020-12-03 00:00:00 │   2020-12-03 01:00:00 │      2020-12-03 01:00:00 │     2020-12-03 01:00:00 │   2020-09-15 01:00:00 │
+└──────────┴──────────┴──────────────────────┴───────────────────────┴──────────────────────────┴─────────────────────────┴───────────────────────┘
+
+# TODO:
+# => DBGInvoke dag('select count(1) from default.test group by a, cast_int_date(a), cast_real_date(b), cast_decimal_date(c), cast_string_date(d), cast_time_date(e)', 4,'encode_type:chunk')
+
+# Clean up.
+=> DBGInvoke __drop_tidb_table(default, test)
+=> drop table if exists default.test