Fix cast decimal as string (#1256)

dbms/src/Common/MyTime.h

@@ -114,7 +114,7 @@ struct MyDate : public MyTimeBase
     String toString() const { return dateFormat("%Y-%m-%d"); }
 };
 
-Field parseMyDateTime(const String & str);
+Field parseMyDateTime(const String & str, int8_t fsp = 6);
 
 void convertTimeZone(UInt64 from_time, UInt64 & to_time, const DateLUTImpl & time_zone_from, const DateLUTImpl & time_zone_to);
 

dbms/src/Common/TiFlashException.h

@@ -113,7 +113,8 @@ namespace DB
         E(Internal, "MPP internal error.",                                                                                  \
             "Please contact with developer, \n"                                                                             \
             "better providing information about your cluster(log, topology information etc.).",                             \
-            "");)
+            "");)                                                                                                           \
+    C(Types, E(Truncated, "Data is truncated during conversion.", "", ""); E(WrongValue, "Input value is in wrong format", "", "");)
 
 /// TiFlashError is core struct of standard error,
 /// which contains all information about an error except message.

dbms/src/Common/tests/CMakeLists.txt

@@ -77,3 +77,6 @@ target_link_libraries (persisted_container clickhouse_common_io)
 
 add_executable(decimal_test_decimal_type gtest_decimal_type.cpp)
 target_link_libraries(decimal_test_decimal_type clickhouse_common_io gtest_main)
+
+add_executable(mytime_test gtest_mytime.cpp)
+target_link_libraries(mytime_test clickhouse_common_io clickhouse_functions gtest_main)

dbms/src/Common/tests/gtest_mytime.cpp

@@ -0,0 +1,157 @@
+#include <Common/Exception.h>
+#include <Common/MyTime.h>
+#include <DataTypes/DataTypeMyDateTime.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <string>
+#include <tuple>
+#include <vector>
+
+namespace DB
+{
+namespace tests
+{
+
+class TestMyTime : public testing::Test
+{
+protected:
+    virtual void SetUp() override {}
+    virtual void TearDown() override {}
+
+public:
+    static void checkParseMyDateTime(const std::string & str, const std::string & expected, const DataTypeMyDateTime & type)
+    {
+        try
+        {
+            UInt64 res = parseMyDateTime(str, type.getFraction()).template safeGet<UInt64>();
+            MyDateTime datetime(res);
+            std::string actual = datetime.toString(type.getFraction());
+            EXPECT_EQ(actual, expected) << "Original datetime string: " << str;
+        }
+        catch (...)
+        {
+            std::cerr << "Error occurs when parsing: \"" << str << "\"" << std::endl;
+            throw;
+        }
+    }
+
+    static void checkParseMyDateTime(const std::string & str, MyDateTime & expected, const DataTypeMyDateTime & type)
+    {
+        try
+        {
+            UInt64 res = parseMyDateTime(str, type.getFraction()).template safeGet<UInt64>();
+            MyDateTime source(res);
+            EXPECT_EQ(source.year, expected.year) << "Original datetime string: " << str;
+            EXPECT_EQ(source.month, expected.month) << "Original datetime string: " << str;
+            EXPECT_EQ(source.day, expected.day) << "Original datetime string: " << str;
+            EXPECT_EQ(source.hour, expected.hour) << "Original datetime string: " << str;
+            EXPECT_EQ(source.minute, expected.minute) << "Original datetime string: " << str;
+            EXPECT_EQ(source.second, expected.second) << "Original datetime string: " << str;
+            EXPECT_EQ(source.micro_second, expected.micro_second) << "Original datetime string: " << str;
+        }
+        catch (...)
+        {
+            std::cerr << "Error occurs when parsing: \"" << str << "\"" << std::endl;
+            throw;
+        }
+    }
+};
+
+TEST_F(TestMyTime, ParseMyDateTimeWithFraction)
+try
+{
+    std::vector<std::tuple<std::string, std::string>> cases_with_fsp{
+        {"2020-12-10 11:11:11.123456", "2020-12-10 11:11:11.123456"}, // YYYY-MM-DD HH:MM:SS.mmmmmm
+        {"00-00-00 00:00:00.123", "2000-00-00 00:00:00.123000"},
+        {"1701020304.1", "2017-01-02 03:04:01.000000"},
+        {"1701020302.11", "2017-01-02 03:02:11.000000"},
+        {"170102037.11", "2017-01-02 03:07:11.000000"},
+        {"2018.01.01", "2018-01-01 00:00:00.000000"},
+        {"2020.10.10 10.10.10", "2020-10-10 10:10:10.000000"},
+        {"2020-10-10 10-10.10", "2020-10-10 10:10:10.000000"},
+        {"2020-10-10 10.10", "2020-10-10 10:10:00.000000"},
+        {"2018.01.01", "2018-01-01 00:00:00.000000"},
+    };
+    DataTypeMyDateTime type_with_fraction(6);
+    for (auto & [str, expected] : cases_with_fsp)
+    {
+        checkParseMyDateTime(str, expected, type_with_fraction);
+    }
+}
+catch (Exception & e)
+{
+    std::cerr << e.displayText() << std::endl;
+    GTEST_FAIL();
+}
+
+TEST_F(TestMyTime, ParseMyDateTimeWithoutFraction)
+try
+{
+    std::vector<std::tuple<std::string, std::string>> cases_without_fsp{
+        {"2012-12-31 11:30:45", "2012-12-31 11:30:45"},
+        {"0000-00-00 00:00:00", "0000-00-00 00:00:00"},
+        {"0001-01-01 00:00:00", "0001-01-01 00:00:00"},
+        {"00-12-31 11:30:45", "2000-12-31 11:30:45"},
+        {"12-12-31 11:30:45", "2012-12-31 11:30:45"},
+        {"2012-12-31", "2012-12-31 00:00:00"},
+        {"20121231", "2012-12-31 00:00:00"},
+        {"121231", "2012-12-31 00:00:00"},
+        {"2012^12^31 11+30+45", "2012-12-31 11:30:45"},
+        {"2012^12^31T11+30+45", "2012-12-31 11:30:45"},
+        {"2012-2-1 11:30:45", "2012-02-01 11:30:45"},
+        {"12-2-1 11:30:45", "2012-02-01 11:30:45"},
+        {"20121231113045", "2012-12-31 11:30:45"},
+        {"121231113045", "2012-12-31 11:30:45"},
+        {"2012-02-29", "2012-02-29 00:00:00"},
+        {"00-00-00", "0000-00-00 00:00:00"},
+        {"11111111111", "2011-11-11 11:11:01"},
+        {"1701020301.", "2017-01-02 03:01:00"},
+        {"170102036", "2017-01-02 03:06:00"},
+        {"170102039.", "2017-01-02 03:09:00"},
+        {"2018-01-01 18", "2018-01-01 18:00:00"},
+        {"18-01-01 18", "2018-01-01 18:00:00"},
+        {"2018.01.01 00:00:00", "2018-01-01 00:00:00"},
+        {"2018/01/01-00:00:00", "2018-01-01 00:00:00"},
+        {"4710072", "2047-10-07 02:00:00"},
+    };
+    DataTypeMyDateTime type_without_fraction(0);
+    for (auto & [str, expected] : cases_without_fsp)
+    {
+        checkParseMyDateTime(str, expected, type_without_fraction);
+    }
+}
+catch (Exception & e)
+{
+    std::cerr << e.displayText() << std::endl;
+    GTEST_FAIL();
+}
+
+TEST_F(TestMyTime, ParseMyDateTimeWithTimezone)
+try
+{
+    std::vector<std::tuple<std::string, MyDateTime>> cases{
+        {"2006-01-02T15:04:05Z", MyDateTime(2006, 1, 2, 15, 4, 5, 0)},
+        {"2020-10-21T16:05:10Z", MyDateTime(2020, 10, 21, 16, 5, 10, 0)},
+        {"2020-10-21T16:05:10.50+08", MyDateTime(2020, 10, 21, 8, 5, 10, 500 * 1000)},
+        {"2020-10-21T16:05:10.50-0700", MyDateTime(2020, 10, 21, 23, 5, 10, 500 * 1000)},
+        {"2020-10-21T16:05:10.50+09:00", MyDateTime(2020, 10, 21, 7, 5, 10, 500 * 1000)},
+        {"2006-01-02T15:04:05+09:00", MyDateTime(2006, 1, 2, 6, 4, 5, 0)},
+        {"2006-01-02T15:04:05-02:00", MyDateTime(2006, 1, 2, 17, 4, 5, 0)},
+        {"2006-01-02T15:04:05-14:00", MyDateTime(2006, 1, 3, 5, 4, 5, 0)},
+    };
+    DataTypeMyDateTime type(6);
+    for (auto & [str, expected] : cases)
+    {
+        checkParseMyDateTime(str, expected, type);
+    }
+}
+catch (Exception & e)
+{
+    std::cerr << e.displayText() << std::endl;
+    GTEST_FAIL();
+}
+
+} // namespace tests
+
+} // namespace DB

dbms/src/DataTypes/DataTypeMyDateTime.cpp

@@ -1,16 +1,13 @@
-#include <IO/ReadHelpers.h>
-#include <IO/WriteHelpers.h>
-
 #include <Columns/ColumnsNumber.h>
 #include <Common/typeid_cast.h>
 #include <DataTypes/DataTypeFactory.h>
 #include <DataTypes/DataTypeMyDateTime.h>
-#include <common/DateLUT.h>
-
 #include <IO/Operators.h>
+#include <IO/ReadHelpers.h>
 #include <IO/WriteBufferFromString.h>
-
+#include <IO/WriteHelpers.h>
 #include <Parsers/ASTLiteral.h>
+#include <common/DateLUT.h>
 
 
 namespace DB
@@ -102,7 +99,7 @@ bool DataTypeMyDateTime::equals(const IDataType & rhs) const
 {
     /// DateTime with different timezones are equal, because:
     /// "all types with different time zones are equivalent and may be used interchangingly."
-    return typeid(rhs) == typeid(*this);
+    return typeid(rhs) == typeid(*this) && getFraction() == dynamic_cast<const DataTypeMyDateTime *>(&rhs)->getFraction();
 }
 
 

dbms/src/Debug/dbgFuncCoprocessor.cpp

@@ -274,6 +274,22 @@ std::unordered_map<String, tipb::ScalarFuncSig> func_name_to_sig({
     {"cast_decimal_decimal", tipb::ScalarFuncSig::CastDecimalAsDecimal},
     {"cast_time_decimal", tipb::ScalarFuncSig::CastTimeAsDecimal},
     {"cast_string_decimal", tipb::ScalarFuncSig::CastStringAsDecimal},
+    {"cast_int_string", tipb::ScalarFuncSig::CastIntAsString},
+    {"cast_real_string", tipb::ScalarFuncSig::CastRealAsString},
+    {"cast_decimal_string", tipb::ScalarFuncSig::CastDecimalAsString},
+    {"cast_time_string", tipb::ScalarFuncSig::CastTimeAsString},
+    {"cast_string_string", tipb::ScalarFuncSig::CastStringAsString},
+    {"cast_int_date", tipb::ScalarFuncSig::CastIntAsTime},
+    {"cast_real_date", tipb::ScalarFuncSig::CastRealAsTime},
+    {"cast_decimal_date", tipb::ScalarFuncSig::CastDecimalAsTime},
+    {"cast_time_date", tipb::ScalarFuncSig::CastTimeAsTime},
+    {"cast_string_date", tipb::ScalarFuncSig::CastStringAsTime},
+    {"cast_int_datetime", tipb::ScalarFuncSig::CastIntAsTime},
+    {"cast_real_datetime", tipb::ScalarFuncSig::CastRealAsTime},
+    {"cast_decimal_datetime", tipb::ScalarFuncSig::CastDecimalAsTime},
+    {"cast_time_datetime", tipb::ScalarFuncSig::CastTimeAsTime},
+    {"cast_string_datetime", tipb::ScalarFuncSig::CastStringAsTime},
+
 });
 
 void compileExpr(const DAGSchema & input, ASTPtr ast, tipb::Expr * expr, std::unordered_set<String> & referred_columns,
@@ -399,6 +415,24 @@ void compileExpr(const DAGSchema & input, ASTPtr ast, tipb::Expr * expr, std::un
                 expr->set_sig(tipb::ScalarFuncSig::DateFormatSig);
                 expr->mutable_field_type()->set_tp(TiDB::TypeString);
                 break;
+            case tipb::ScalarFuncSig::CastIntAsTime:
+            case tipb::ScalarFuncSig::CastRealAsTime:
+            case tipb::ScalarFuncSig::CastTimeAsTime:
+            case tipb::ScalarFuncSig::CastDecimalAsTime:
+            case tipb::ScalarFuncSig::CastStringAsTime:
+            {
+                expr->set_sig(it_sig->second);
+                auto * ft = expr->mutable_field_type();
+                if (it_sig->first.find("datetime"))
+                {
+                    ft->set_tp(TiDB::TypeDatetime);
+                }
+                else
+                {
+                    ft->set_tp(TiDB::TypeDate);
+                }
+                break;
+            }
             default:
             {
                 expr->set_sig(it_sig->second);

dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp

@@ -431,6 +431,7 @@ String DAGExpressionAnalyzer::convertToUInt8(ExpressionActionsPtr & actions, con
     {
         /// use tidb_cast to make it compatible with TiDB
         tipb::FieldType field_type;
+        // TODO: Use TypeDouble as return type, to be compatible with TiDB
         field_type.set_tp(TiDB::TypeLongLong);
         tipb::Expr type_expr;
         constructStringLiteralTiExpr(type_expr, "Nullable(Int64)");

dbms/src/Functions/FunctionsConversion.h

@@ -438,6 +438,15 @@ struct FormatImpl<DataTypeMyDateTime>
     }
 };
 
+template <typename DecimalType>
+struct FormatImpl<DataTypeDecimal<DecimalType>>
+{
+    static void execute(const typename DataTypeDecimal<DecimalType>::FieldType v, WriteBuffer & wb, const DataTypeDecimal<DecimalType> * tp, const DateLUTImpl *)
+    {
+        writeText(v, tp->getScale(), wb);
+    }
+};
+
 template <typename FieldType>
 struct FormatImpl<DataTypeEnum<FieldType>>
 {