Skip to content

Commit

Permalink
Fix cast decimal as string (#1256) (#1281)
Browse files Browse the repository at this point in the history
  • Loading branch information
@leiysky
leiysky authored Dec 13, 2020
1 parent 4bb5397 commit de0a49b
Show file tree
Hide file tree
Showing 16 changed files with 717 additions and 51 deletions.
394 changes: 359 additions & 35 deletions dbms/src/Common/MyTime.cpp
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion dbms/src/Common/MyTime.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ struct MyDate : public MyTimeBase
String toString() const { return dateFormat("%Y-%m-%d"); }
};

Field parseMyDateTime(const String & str);
Field parseMyDateTime(const String & str, int8_t fsp = 6);

void convertTimeZone(UInt64 from_time, UInt64 & to_time, const DateLUTImpl & time_zone_from, const DateLUTImpl & time_zone_to);

Expand Down
8 changes: 7 additions & 1 deletion dbms/src/Common/TiFlashException.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,13 @@ namespace DB
E(Internal, "Encryption internal error.", \
"Please contact with developer, \n" \
"better providing information about your cluster(log, topology information etc.).", \
"");)
"");) \
C(MPP, \
E(Internal, "MPP internal error.", \
"Please contact with developer, \n" \
"better providing information about your cluster(log, topology information etc.).", \
"");) \
C(Types, E(Truncated, "Data is truncated during conversion.", "", ""); E(WrongValue, "Input value is in wrong format", "", "");)


/// TiFlashError is core struct of standard error,
Expand Down
3 changes: 3 additions & 0 deletions dbms/src/Common/tests/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,6 @@ target_link_libraries (persisted_container clickhouse_common_io)

add_executable(decimal_test_decimal_type gtest_decimal_type.cpp)
target_link_libraries(decimal_test_decimal_type clickhouse_common_io gtest_main)

add_executable(mytime_test gtest_mytime.cpp)
target_link_libraries(mytime_test clickhouse_common_io clickhouse_functions gtest_main)
157 changes: 157 additions & 0 deletions dbms/src/Common/tests/gtest_mytime.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
#include <Common/Exception.h>
#include <Common/MyTime.h>
#include <DataTypes/DataTypeMyDateTime.h>
#include <gtest/gtest.h>

#include <iostream>
#include <string>
#include <tuple>
#include <vector>

namespace DB
{
namespace tests
{

class TestMyTime : public testing::Test
{
protected:
virtual void SetUp() override {}
virtual void TearDown() override {}

public:
static void checkParseMyDateTime(const std::string & str, const std::string & expected, const DataTypeMyDateTime & type)
{
try
{
UInt64 res = parseMyDateTime(str, type.getFraction()).template safeGet<UInt64>();
MyDateTime datetime(res);
std::string actual = datetime.toString(type.getFraction());
EXPECT_EQ(actual, expected) << "Original datetime string: " << str;
}
catch (...)
{
std::cerr << "Error occurs when parsing: \"" << str << "\"" << std::endl;
throw;
}
}

static void checkParseMyDateTime(const std::string & str, MyDateTime & expected, const DataTypeMyDateTime & type)
{
try
{
UInt64 res = parseMyDateTime(str, type.getFraction()).template safeGet<UInt64>();
MyDateTime source(res);
EXPECT_EQ(source.year, expected.year) << "Original datetime string: " << str;
EXPECT_EQ(source.month, expected.month) << "Original datetime string: " << str;
EXPECT_EQ(source.day, expected.day) << "Original datetime string: " << str;
EXPECT_EQ(source.hour, expected.hour) << "Original datetime string: " << str;
EXPECT_EQ(source.minute, expected.minute) << "Original datetime string: " << str;
EXPECT_EQ(source.second, expected.second) << "Original datetime string: " << str;
EXPECT_EQ(source.micro_second, expected.micro_second) << "Original datetime string: " << str;
}
catch (...)
{
std::cerr << "Error occurs when parsing: \"" << str << "\"" << std::endl;
throw;
}
}
};

TEST_F(TestMyTime, ParseMyDateTimeWithFraction)
try
{
std::vector<std::tuple<std::string, std::string>> cases_with_fsp{
{"2020-12-10 11:11:11.123456", "2020-12-10 11:11:11.123456"}, // YYYY-MM-DD HH:MM:SS.mmmmmm
{"00-00-00 00:00:00.123", "2000-00-00 00:00:00.123000"},
{"1701020304.1", "2017-01-02 03:04:01.000000"},
{"1701020302.11", "2017-01-02 03:02:11.000000"},
{"170102037.11", "2017-01-02 03:07:11.000000"},
{"2018.01.01", "2018-01-01 00:00:00.000000"},
{"2020.10.10 10.10.10", "2020-10-10 10:10:10.000000"},
{"2020-10-10 10-10.10", "2020-10-10 10:10:10.000000"},
{"2020-10-10 10.10", "2020-10-10 10:10:00.000000"},
{"2018.01.01", "2018-01-01 00:00:00.000000"},
};
DataTypeMyDateTime type_with_fraction(6);
for (auto & [str, expected] : cases_with_fsp)
{
checkParseMyDateTime(str, expected, type_with_fraction);
}
}
catch (Exception & e)
{
std::cerr << e.displayText() << std::endl;
GTEST_FAIL();
}

TEST_F(TestMyTime, ParseMyDateTimeWithoutFraction)
try
{
std::vector<std::tuple<std::string, std::string>> cases_without_fsp{
{"2012-12-31 11:30:45", "2012-12-31 11:30:45"},
{"0000-00-00 00:00:00", "0000-00-00 00:00:00"},
{"0001-01-01 00:00:00", "0001-01-01 00:00:00"},
{"00-12-31 11:30:45", "2000-12-31 11:30:45"},
{"12-12-31 11:30:45", "2012-12-31 11:30:45"},
{"2012-12-31", "2012-12-31 00:00:00"},
{"20121231", "2012-12-31 00:00:00"},
{"121231", "2012-12-31 00:00:00"},
{"2012^12^31 11+30+45", "2012-12-31 11:30:45"},
{"2012^12^31T11+30+45", "2012-12-31 11:30:45"},
{"2012-2-1 11:30:45", "2012-02-01 11:30:45"},
{"12-2-1 11:30:45", "2012-02-01 11:30:45"},
{"20121231113045", "2012-12-31 11:30:45"},
{"121231113045", "2012-12-31 11:30:45"},
{"2012-02-29", "2012-02-29 00:00:00"},
{"00-00-00", "0000-00-00 00:00:00"},
{"11111111111", "2011-11-11 11:11:01"},
{"1701020301.", "2017-01-02 03:01:00"},
{"170102036", "2017-01-02 03:06:00"},
{"170102039.", "2017-01-02 03:09:00"},
{"2018-01-01 18", "2018-01-01 18:00:00"},
{"18-01-01 18", "2018-01-01 18:00:00"},
{"2018.01.01 00:00:00", "2018-01-01 00:00:00"},
{"2018/01/01-00:00:00", "2018-01-01 00:00:00"},
{"4710072", "2047-10-07 02:00:00"},
};
DataTypeMyDateTime type_without_fraction(0);
for (auto & [str, expected] : cases_without_fsp)
{
checkParseMyDateTime(str, expected, type_without_fraction);
}
}
catch (Exception & e)
{
std::cerr << e.displayText() << std::endl;
GTEST_FAIL();
}

TEST_F(TestMyTime, ParseMyDateTimeWithTimezone)
try
{
std::vector<std::tuple<std::string, MyDateTime>> cases{
{"2006-01-02T15:04:05Z", MyDateTime(2006, 1, 2, 15, 4, 5, 0)},
{"2020-10-21T16:05:10Z", MyDateTime(2020, 10, 21, 16, 5, 10, 0)},
{"2020-10-21T16:05:10.50+08", MyDateTime(2020, 10, 21, 8, 5, 10, 500 * 1000)},
{"2020-10-21T16:05:10.50-0700", MyDateTime(2020, 10, 21, 23, 5, 10, 500 * 1000)},
{"2020-10-21T16:05:10.50+09:00", MyDateTime(2020, 10, 21, 7, 5, 10, 500 * 1000)},
{"2006-01-02T15:04:05+09:00", MyDateTime(2006, 1, 2, 6, 4, 5, 0)},
{"2006-01-02T15:04:05-02:00", MyDateTime(2006, 1, 2, 17, 4, 5, 0)},
{"2006-01-02T15:04:05-14:00", MyDateTime(2006, 1, 3, 5, 4, 5, 0)},
};
DataTypeMyDateTime type(6);
for (auto & [str, expected] : cases)
{
checkParseMyDateTime(str, expected, type);
}
}
catch (Exception & e)
{
std::cerr << e.displayText() << std::endl;
GTEST_FAIL();
}

} // namespace tests

} // namespace DB
11 changes: 4 additions & 7 deletions dbms/src/DataTypes/DataTypeMyDateTime.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,13 @@
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>

#include <Columns/ColumnsNumber.h>
#include <Common/typeid_cast.h>
#include <DataTypes/DataTypeFactory.h>
#include <DataTypes/DataTypeMyDateTime.h>
#include <common/DateLUT.h>

#include <IO/Operators.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromString.h>

#include <IO/WriteHelpers.h>
#include <Parsers/ASTLiteral.h>
#include <common/DateLUT.h>


namespace DB
Expand Down Expand Up @@ -102,7 +99,7 @@ bool DataTypeMyDateTime::equals(const IDataType & rhs) const
{
/// DateTime with different timezones are equal, because:
/// "all types with different time zones are equivalent and may be used interchangingly."
return typeid(rhs) == typeid(*this);
return typeid(rhs) == typeid(*this) && getFraction() == dynamic_cast<const DataTypeMyDateTime *>(&rhs)->getFraction();
}


Expand Down
34 changes: 34 additions & 0 deletions dbms/src/Debug/dbgFuncCoprocessor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,22 @@ std::unordered_map<String, tipb::ScalarFuncSig> func_name_to_sig({
{"cast_decimal_decimal", tipb::ScalarFuncSig::CastDecimalAsDecimal},
{"cast_time_decimal", tipb::ScalarFuncSig::CastTimeAsDecimal},
{"cast_string_decimal", tipb::ScalarFuncSig::CastStringAsDecimal},
{"cast_int_string", tipb::ScalarFuncSig::CastIntAsString},
{"cast_real_string", tipb::ScalarFuncSig::CastRealAsString},
{"cast_decimal_string", tipb::ScalarFuncSig::CastDecimalAsString},
{"cast_time_string", tipb::ScalarFuncSig::CastTimeAsString},
{"cast_string_string", tipb::ScalarFuncSig::CastStringAsString},
{"cast_int_date", tipb::ScalarFuncSig::CastIntAsTime},
{"cast_real_date", tipb::ScalarFuncSig::CastRealAsTime},
{"cast_decimal_date", tipb::ScalarFuncSig::CastDecimalAsTime},
{"cast_time_date", tipb::ScalarFuncSig::CastTimeAsTime},
{"cast_string_date", tipb::ScalarFuncSig::CastStringAsTime},
{"cast_int_datetime", tipb::ScalarFuncSig::CastIntAsTime},
{"cast_real_datetime", tipb::ScalarFuncSig::CastRealAsTime},
{"cast_decimal_datetime", tipb::ScalarFuncSig::CastDecimalAsTime},
{"cast_time_datetime", tipb::ScalarFuncSig::CastTimeAsTime},
{"cast_string_datetime", tipb::ScalarFuncSig::CastStringAsTime},

});

void compileExpr(const DAGSchema & input, ASTPtr ast, tipb::Expr * expr, std::unordered_set<String> & referred_columns,
Expand Down Expand Up @@ -399,6 +415,24 @@ void compileExpr(const DAGSchema & input, ASTPtr ast, tipb::Expr * expr, std::un
expr->set_sig(tipb::ScalarFuncSig::DateFormatSig);
expr->mutable_field_type()->set_tp(TiDB::TypeString);
break;
case tipb::ScalarFuncSig::CastIntAsTime:
case tipb::ScalarFuncSig::CastRealAsTime:
case tipb::ScalarFuncSig::CastTimeAsTime:
case tipb::ScalarFuncSig::CastDecimalAsTime:
case tipb::ScalarFuncSig::CastStringAsTime:
{
expr->set_sig(it_sig->second);
auto * ft = expr->mutable_field_type();
if (it_sig->first.find("datetime"))
{
ft->set_tp(TiDB::TypeDatetime);
}
else
{
ft->set_tp(TiDB::TypeDate);
}
break;
}
default:
{
expr->set_sig(it_sig->second);
Expand Down
1 change: 1 addition & 0 deletions dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,7 @@ String DAGExpressionAnalyzer::convertToUInt8(ExpressionActionsPtr & actions, con
{
/// use tidb_cast to make it compatible with TiDB
tipb::FieldType field_type;
// TODO: Use TypeDouble as return type, to be compatible with TiDB
field_type.set_tp(TiDB::TypeLongLong);
tipb::Expr type_expr;
constructStringLiteralTiExpr(type_expr, "Nullable(Int64)");
Expand Down
9 changes: 9 additions & 0 deletions dbms/src/Functions/FunctionsConversion.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -438,6 +438,15 @@ struct FormatImpl<DataTypeMyDateTime>
}
};

template <typename DecimalType>
struct FormatImpl<DataTypeDecimal<DecimalType>>
{
static void execute(const typename DataTypeDecimal<DecimalType>::FieldType v, WriteBuffer & wb, const DataTypeDecimal<DecimalType> * tp, const DateLUTImpl *)
{
writeText(v, tp->getScale(), wb);
}
};

template <typename FieldType>
struct FormatImpl<DataTypeEnum<FieldType>>
{
Expand Down
Loading

0 comments on commit de0a49b

Please sign in to comment.