From e8b41981c3ac2c387221868ffb85cc069740df4e Mon Sep 17 00:00:00 2001 From: ruoxi Date: Mon, 26 Aug 2019 15:49:58 +0800 Subject: [PATCH] Fix date/datetime/bit encode error (#200) * Enhance dbg invoke and add dag as schemaful function * Add basic sql parse to dag * Column id starts from 1 * Fix value to ref * Add basic dag test * Fix dag bugs and pass 1st mock test * Make dag go normal routine and add mock dag * Add todo * Add comment * Fix gcc compile error * Enhance dag test * Address comments * Enhance mock sql -> dag compiler and add project test * Mock sql dag compiler support more expression types and add filter test * Add topn and limit test * Add agg for sql -> dag parser and agg test * Add dag specific codec * type * Update codec accordingly * Remove cop-test * Pass tests after merging master * Copy some changes from xufei * Enable date/datetime test * Enable date/datetime test * Refine code * Adjust date/datetime tiflash rep to UInt * Fix datetime to Int * Typo --- dbms/src/Debug/dbgFuncCoprocessor.cpp | 41 ++-- dbms/src/Debug/dbgTools.cpp | 30 ++- .../Coprocessor/DAGExpressionAnalyzer.cpp | 6 +- dbms/src/Flash/Coprocessor/DAGUtils.cpp | 22 ++- dbms/src/Functions/FunctionsComparison.h | 176 +++++++++++++++++- dbms/src/Storages/Transaction/Datum.cpp | 6 +- dbms/src/Storages/Transaction/TiDB.cpp | 12 +- dbms/src/Storages/Transaction/TiDB.h | 15 +- .../mutable-test/txn_dag/data_type_time.test | 18 +- 9 files changed, 271 insertions(+), 55 deletions(-) diff --git a/dbms/src/Debug/dbgFuncCoprocessor.cpp b/dbms/src/Debug/dbgFuncCoprocessor.cpp index 594a0c4ee1c..d7e2fb5bedf 100644 --- a/dbms/src/Debug/dbgFuncCoprocessor.cpp +++ b/dbms/src/Debug/dbgFuncCoprocessor.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -29,9 +30,12 @@ extern const int BAD_ARGUMENTS; extern const int LOGICA_ERROR; } // namespace ErrorCodes +using TiDB::DatumFlat; +using TiDB::TableInfo; + using DAGColumnInfo = std::pair; using DAGSchema = std::vector; -using SchemaFetcher = std::function; +using SchemaFetcher = std::function; std::tuple compileQuery( Context & context, const String & query, SchemaFetcher schema_fetcher, Timestamp start_ts); tipb::SelectResponse executeDAGRequest( @@ -138,7 +142,7 @@ void compileExpr(const DAGSchema & input, ASTPtr ast, tipb::Expr * expr, std::un { auto ft = std::find_if(input.begin(), input.end(), [&](const auto & field) { return field.first == id->getColumnName(); }); if (ft == input.end()) - throw DB::Exception("No such column " + id->getColumnName(), ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); + throw Exception("No such column " + id->getColumnName(), ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); expr->set_tp(tipb::ColumnRef); *(expr->mutable_field_type()) = columnInfoToFieldType((*ft).second); @@ -183,7 +187,7 @@ void compileExpr(const DAGSchema & input, ASTPtr ast, tipb::Expr * expr, std::un } else { - throw DB::Exception("Unsupported function: " + func_name_lowercase, ErrorCodes::LOGICAL_ERROR); + throw Exception("Unsupported function: " + func_name_lowercase, ErrorCodes::LOGICAL_ERROR); } expr->set_tp(tipb::ExprType::ScalarFunc); } @@ -221,13 +225,13 @@ void compileExpr(const DAGSchema & input, ASTPtr ast, tipb::Expr * expr, std::un encodeDAGBytes(lit->value.get(), ss); break; default: - throw DB::Exception(String("Unsupported literal type: ") + lit->value.getTypeName(), ErrorCodes::LOGICAL_ERROR); + throw Exception(String("Unsupported literal type: ") + lit->value.getTypeName(), ErrorCodes::LOGICAL_ERROR); } expr->set_val(ss.str()); } else { - throw DB::Exception("Unsupported expression " + ast->getColumnName(), ErrorCodes::LOGICAL_ERROR); + throw Exception("Unsupported expression " + ast->getColumnName(), ErrorCodes::LOGICAL_ERROR); } } @@ -262,7 +266,7 @@ std::tuple compileQuery( ASTSelectQuery & ast_query = typeid_cast(*ast); /// Get table metadata. - TiDB::TableInfo table_info; + TableInfo table_info; { String database_name, table_name; auto query_database = ast_query.database(); @@ -333,7 +337,7 @@ std::tuple compileQuery( { ASTOrderByElement * elem = typeid_cast(child.get()); if (!elem) - throw DB::Exception("Invalid order by element", ErrorCodes::LOGICAL_ERROR); + throw Exception("Invalid order by element", ErrorCodes::LOGICAL_ERROR); tipb::ByItem * by = topn->add_order_by(); by->set_desc(elem->direction < 0); tipb::Expr * expr = by->mutable_expr(); @@ -389,7 +393,7 @@ std::tuple compileQuery( { auto iter = std::find_if(last_output.begin(), last_output.end(), [&](const auto & field) { return field.first == pair.first; }); if (iter == last_output.end()) - throw DB::Exception("Column not found when pruning: " + pair.first, ErrorCodes::LOGICAL_ERROR); + throw Exception("Column not found when pruning: " + pair.first, ErrorCodes::LOGICAL_ERROR); std::stringstream ss; encodeDAGInt64(iter - last_output.begin(), ss); pair.second->set_val(ss.str()); @@ -414,7 +418,7 @@ std::tuple compileQuery( if (has_gby || has_agg_func) { if (last_executor->has_limit() || last_executor->has_topn()) - throw DB::Exception("Limit/TopN and Agg cannot co-exist.", ErrorCodes::LOGICAL_ERROR); + throw Exception("Limit/TopN and Agg cannot co-exist.", ErrorCodes::LOGICAL_ERROR); tipb::Executor * agg_exec = dag_request.add_executors(); agg_exec->set_tp(tipb::ExecType::TypeAggregation); @@ -424,7 +428,7 @@ std::tuple compileQuery( { const ASTFunction * func = typeid_cast(expr.get()); if (!func || !AggregateFunctionFactory::instance().isAggregateFunctionName(func->name)) - throw DB::Exception("Only agg function is allowed in select for a query with aggregation", ErrorCodes::LOGICAL_ERROR); + throw Exception("Only agg function is allowed in select for a query with aggregation", ErrorCodes::LOGICAL_ERROR); tipb::Expr * agg_func = agg->add_agg_func(); @@ -444,7 +448,7 @@ std::tuple compileQuery( // TODO: Other agg func. else { - throw DB::Exception("Unsupported agg function " + func->name, ErrorCodes::LOGICAL_ERROR); + throw Exception("Unsupported agg function " + func->name, ErrorCodes::LOGICAL_ERROR); } schema.emplace_back(std::make_pair(func->getColumnName(), fieldTypeToColumnInfo(agg_func->field_type()))); @@ -489,7 +493,7 @@ std::tuple compileQuery( } else { - throw DB::Exception("Unsupported expression type in select", ErrorCodes::LOGICAL_ERROR); + throw Exception("Unsupported expression type in select", ErrorCodes::LOGICAL_ERROR); } } @@ -501,7 +505,7 @@ std::tuple compileQuery( auto iter = std::find_if(last_output.begin(), last_output.end(), [&](const auto & last_field) { return last_field.first == field; }); if (iter == last_output.end()) - throw DB::Exception("Column not found after pruning: " + field, ErrorCodes::LOGICAL_ERROR); + throw Exception("Column not found after pruning: " + field, ErrorCodes::LOGICAL_ERROR); dag_request.add_output_offsets(iter - last_output.begin()); schema.push_back(*iter); } @@ -526,18 +530,18 @@ tipb::SelectResponse executeDAGRequest( BlockInputStreamPtr outputDAGResponse(Context &, const DAGSchema & schema, const tipb::SelectResponse & dag_response) { if (dag_response.has_error()) - throw DB::Exception(dag_response.error().msg(), dag_response.error().code()); + throw Exception(dag_response.error().msg(), dag_response.error().code()); BlocksList blocks; for (const auto & chunk : dag_response.chunks()) { - std::vector> rows; - std::vector curr_row; + std::vector> rows; + std::vector curr_row; const std::string & data = chunk.rows_data(); size_t cursor = 0; while (cursor < data.size()) { - curr_row.push_back(DB::DecodeDatum(cursor, data)); + curr_row.push_back(DecodeDatum(cursor, data)); if (curr_row.size() == schema.size()) { rows.emplace_back(std::move(curr_row)); @@ -558,7 +562,8 @@ BlockInputStreamPtr outputDAGResponse(Context &, const DAGSchema & schema, const { for (size_t i = 0; i < row.size(); i++) { - columns[i].column->assumeMutable()->insert(row[i]); + const Field & field = row[i]; + columns[i].column->assumeMutable()->insert(DatumFlat(field, schema[i].second.tp).field()); } } diff --git a/dbms/src/Debug/dbgTools.cpp b/dbms/src/Debug/dbgTools.cpp index 305ac3f8260..47874e73772 100644 --- a/dbms/src/Debug/dbgTools.cpp +++ b/dbms/src/Debug/dbgTools.cpp @@ -149,21 +149,24 @@ T convertNumber(const Field & field) } } -Field convertDecimal(UInt32 scale, const Field & field) +Field convertDecimal(const ColumnInfo & column_info, const Field & field) { switch (field.getType()) { case Field::Types::Int64: - return DecimalField(ToDecimal(field.get(), scale), scale); + return column_info.getDecimalValue(std::to_string(field.get())); case Field::Types::UInt64: - return DecimalField(ToDecimal(field.get(), scale), scale); + return column_info.getDecimalValue(std::to_string(field.get())); case Field::Types::Float64: - return DecimalField(ToDecimal(field.get(), scale), scale); + return column_info.getDecimalValue(std::to_string(field.get())); case Field::Types::Decimal32: + return column_info.getDecimalValue(field.get().toString(column_info.decimal)); case Field::Types::Decimal64: + return column_info.getDecimalValue(field.get().toString(column_info.decimal)); case Field::Types::Decimal128: + return column_info.getDecimalValue(field.get().toString(column_info.decimal)); case Field::Types::Decimal256: - return field; + return column_info.getDecimalValue(field.get().toString(column_info.decimal)); default: throw Exception(String("Unable to convert field type ") + field.getTypeName() + " to number", ErrorCodes::LOGICAL_ERROR); } @@ -204,9 +207,22 @@ Field convertField(const ColumnInfo & column_info, const Field & field) case TiDB::TypeDouble: return convertNumber(field); case TiDB::TypeDate: + { + auto text = field.get(); + ReadBufferFromMemory buf(text.data(), text.size()); + DayNum_t date; + readDateText(date, buf); + return static_cast(date); + } case TiDB::TypeDatetime: case TiDB::TypeTimestamp: - return DB::parseMyDatetime(field.get()); + { + auto text = field.get(); + ReadBufferFromMemory buf(text.data(), text.size()); + time_t dt; + readDateTimeText(dt, buf); + return static_cast(dt); + } case TiDB::TypeVarchar: case TiDB::TypeTinyBlob: case TiDB::TypeMediumBlob: @@ -221,7 +237,7 @@ Field convertField(const ColumnInfo & column_info, const Field & field) return Field(); case TiDB::TypeDecimal: case TiDB::TypeNewDecimal: - return convertDecimal(column_info.decimal, field); + return convertDecimal(column_info, field); case TiDB::TypeTime: throw Exception(String("Unable to convert field type ") + field.getTypeName() + " to Time", ErrorCodes::LOGICAL_ERROR); case TiDB::TypeYear: diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp index 9041060a295..155ce2b5ac0 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp @@ -265,9 +265,9 @@ String DAGExpressionAnalyzer::appendCastIfNeeded(const tipb::Expr & expr, Expres type_expr.set_tp(tipb::ExprType::String); std::stringstream ss; type_expr.set_val(expected_type->getName()); - auto type_field_type = type_expr.field_type(); - type_field_type.set_tp(0xfe); - type_field_type.set_flag(1); + auto * type_field_type = type_expr.mutable_field_type(); + type_field_type->set_tp(0xfe); + type_field_type->set_flag(1); getActions(type_expr, actions); Names cast_argument_names; diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index 2a334230573..9359bb7de06 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -84,6 +85,16 @@ String exprToString(const tipb::Expr & expr, const NamesAndTypesList & input_col else throw Exception("Not decimal literal" + expr.DebugString(), ErrorCodes::COP_BAD_DAG_REQUEST); } + case tipb::ExprType::MysqlTime: + { + if (!expr.has_field_type() + || (expr.field_type().tp() != TiDB::TypeDate && expr.field_type().tp() != TiDB::TypeDatetime + && expr.field_type().tp() != TiDB::TypeTimestamp)) + throw Exception("Invalid MySQL Time literal " + expr.DebugString(), ErrorCodes::COP_BAD_DAG_REQUEST); + auto t = decodeDAGUInt64(expr.val()); + // TODO: Use timezone in DAG request. + return std::to_string(TiDB::DatumFlat(t, static_cast(expr.field_type().tp())).field().get()); + } case tipb::ExprType::ColumnRef: column_id = decodeDAGInt64(expr.val()); if (column_id < 0 || column_id >= (ColumnID)input_col.size()) @@ -222,12 +233,21 @@ Field decodeLiteral(const tipb::Expr & expr) return decodeDAGBytes(expr.val()); case tipb::ExprType::MysqlDecimal: return decodeDAGDecimal(expr.val()); + case tipb::ExprType::MysqlTime: + { + if (!expr.has_field_type() + || (expr.field_type().tp() != TiDB::TypeDate && expr.field_type().tp() != TiDB::TypeDatetime + && expr.field_type().tp() != TiDB::TypeTimestamp)) + throw Exception("Invalid MySQL Time literal " + expr.DebugString(), ErrorCodes::COP_BAD_DAG_REQUEST); + auto t = decodeDAGUInt64(expr.val()); + // TODO: Use timezone in DAG request. + return TiDB::DatumFlat(t, static_cast(expr.field_type().tp())).field(); + } case tipb::ExprType::MysqlBit: case tipb::ExprType::MysqlDuration: case tipb::ExprType::MysqlEnum: case tipb::ExprType::MysqlHex: case tipb::ExprType::MysqlSet: - case tipb::ExprType::MysqlTime: case tipb::ExprType::MysqlJson: case tipb::ExprType::ValueList: throw Exception(tipb::ExprType_Name(expr.tp()) + " is not supported yet", ErrorCodes::UNSUPPORTED_METHOD); diff --git a/dbms/src/Functions/FunctionsComparison.h b/dbms/src/Functions/FunctionsComparison.h index 8c4a9e18113..f94fb6f6a2a 100644 --- a/dbms/src/Functions/FunctionsComparison.h +++ b/dbms/src/Functions/FunctionsComparison.h @@ -125,6 +125,116 @@ inline int memcmp16(const void * a, const void * b) } +inline time_t dateToDateTime(UInt32 date_data) +{ + DayNum_t day_num(date_data); + LocalDate local_date(day_num); + // todo use timezone info + return DateLUT::instance().makeDateTime(local_date.year(), local_date.month(), local_date.day(), 0, 0, 0); +} + + +template class Op, bool is_left_date> +struct DateDateTimeComparisonImpl +{ + static void NO_INLINE vector_vector(const PaddedPODArray & a, const PaddedPODArray & b, PaddedPODArray & c) + { + size_t size = a.size(); + const A * a_pos = &a[0]; + const B * b_pos = &b[0]; + UInt8 * c_pos = &c[0]; + const A * a_end = a_pos + size; + while (a_pos < a_end) + { + if (is_left_date) + { + using OpType = B; + time_t date_time = dateToDateTime(*a_pos); + *c_pos = Op::apply((OpType)date_time, *b_pos); + } + else + { + using OpType = A; + time_t date_time = dateToDateTime(*b_pos); + *c_pos = Op::apply(*a_pos, (OpType)date_time); + } + ++a_pos; + ++b_pos; + ++c_pos; + } + } + + static void NO_INLINE vector_constant(const PaddedPODArray & a, B b, PaddedPODArray & c) + { + if (!is_left_date) + { + // datetime vector with date constant + using OpType = A; + time_t date_time = dateToDateTime(b); + NumComparisonImpl>::vector_constant(a, (OpType) date_time, c); + } + else + { + using OpType = B; + size_t size = a.size(); + const A * a_pos = &a[0]; + UInt8 * c_pos = &c[0]; + const A * a_end = a_pos + size; + + while (a_pos < a_end) + { + time_t date_time = dateToDateTime(*a_pos); + *c_pos = Op::apply((OpType)date_time, b); + ++a_pos; + ++c_pos; + } + } + } + + static void constant_vector(A a, const PaddedPODArray & b, PaddedPODArray & c) + { + if (is_left_date) + { + // date constant with datetime vector + using OpType = B; + time_t date_time = dateToDateTime(a); + NumComparisonImpl>::constant_vector((OpType)date_time, b, c); + } + else + { + using OpType = A; + size_t size = b.size(); + const B * b_pos = &b[0]; + UInt8 * c_pos = &c[0]; + const B * b_end = b_pos + size; + + while (b_pos < b_end) + { + time_t date_time = dateToDateTime(*b_pos); + *c_pos = Op::apply(a, (OpType)date_time); + ++b_pos; + ++c_pos; + } + } + } + + static void constant_constant(A a, B b, UInt8 & c) { + if (is_left_date) + { + using OpType = B; + time_t date_time = dateToDateTime(a); + NumComparisonImpl>::constant_constant((OpType) date_time, b, c); + } + else + { + using OpType = A; + time_t date_time = dateToDateTime(b); + NumComparisonImpl>::constant_constant(a, (OpType) date_time, c); + } + } +}; + + template struct StringComparisonImpl { @@ -966,6 +1076,69 @@ class FunctionComparison : public IFunction } } + bool executeDateWithDateTimeOrDateTimeWithDate( + Block &block, size_t result, + const IColumn *col_left_untyped, const IColumn *col_right_untyped, + const DataTypePtr &left_type, const DataTypePtr &right_type) + { + if ((checkDataType(left_type.get()) && checkDataType(right_type.get())) + || (checkDataType(left_type.get()) && checkDataType(right_type.get()))) + { + bool is_left_date = checkDataType(left_type.get()); + if (is_left_date) + { + return executeDateAndDateTimeCompare(block, result, col_left_untyped, col_right_untyped); + } + else + { + return executeDateAndDateTimeCompare(block, result, col_left_untyped, col_right_untyped); + } + } + return false; + } + + template + bool executeDateAndDateTimeCompare(Block & block, size_t result, const IColumn * c0, const IColumn * c1) + { + bool c0_const = c0->isColumnConst(); + bool c1_const = c1->isColumnConst(); + + if (c0_const && c1_const) + { + UInt8 res = 0; + DateDateTimeComparisonImpl::constant_constant( + checkAndGetColumnConst>(c0)->template getValue(), + checkAndGetColumnConst>(c1)-> template getValue(), res); + block.getByPosition(result).column = DataTypeUInt8().createColumnConst(c0->size(), toField(res)); + } + else + { + auto c_res = ColumnUInt8::create(); + ColumnUInt8::Container & vec_res = c_res->getData(); + vec_res.resize(c0->size()); + if (c0_const) + { + DateDateTimeComparisonImpl::constant_vector( + checkAndGetColumnConst>(c0)-> template getValue(), + checkAndGetColumn>(c1)->getData(), vec_res); + } + else if (c1_const) + { + DateDateTimeComparisonImpl::vector_constant( + checkAndGetColumn>(c0)->getData(), + checkAndGetColumnConst>(c1)-> template getValue(), vec_res); + } + else + { + DateDateTimeComparisonImpl::vector_vector( + checkAndGetColumn>(c0)->getData(), + checkAndGetColumn>(c1)->getData(), vec_res); + } + block.getByPosition(result).column = std::move(c_res); + } + return true; + } + public: String getName() const override { @@ -1084,7 +1257,8 @@ class FunctionComparison : public IFunction if (left_is_num && right_is_num) { - if (!( executeNumLeftType(block, result, col_left_untyped, col_right_untyped) + if (!(executeDateWithDateTimeOrDateTimeWithDate(block, result, col_left_untyped, col_right_untyped, col_with_type_and_name_left.type, col_with_type_and_name_right.type) + || executeNumLeftType(block, result, col_left_untyped, col_right_untyped) || executeNumLeftType(block, result, col_left_untyped, col_right_untyped) || executeNumLeftType(block, result, col_left_untyped, col_right_untyped) || executeNumLeftType(block, result, col_left_untyped, col_right_untyped) diff --git a/dbms/src/Storages/Transaction/Datum.cpp b/dbms/src/Storages/Transaction/Datum.cpp index 72301b9fa9b..0b7c9d775e2 100644 --- a/dbms/src/Storages/Transaction/Datum.cpp +++ b/dbms/src/Storages/Transaction/Datum.cpp @@ -44,7 +44,7 @@ struct DatumOp(date); + copy = static_cast(date); } else { @@ -86,7 +86,7 @@ struct DatumOp(orig.get())); + time_t date_time(orig.get()); values = date_lut.getValues(date_time); hour = date_lut.toHour(date_time); minute = date_lut.toMinute(date_time); @@ -100,7 +100,7 @@ struct DatumOp struct DatumOp::type> { diff --git a/dbms/src/Storages/Transaction/TiDB.cpp b/dbms/src/Storages/Transaction/TiDB.cpp index bbdca6b2877..6d3a8376789 100644 --- a/dbms/src/Storages/Transaction/TiDB.cpp +++ b/dbms/src/Storages/Transaction/TiDB.cpp @@ -56,7 +56,7 @@ Field ColumnInfo::defaultValueToField() const return Field(); case TypeDecimal: case TypeNewDecimal: - return getDecimalDefaultValue(value.convert()); + return getDecimalValue(value.convert()); case TypeTime: case TypeYear: case TypeSet: @@ -68,9 +68,9 @@ Field ColumnInfo::defaultValueToField() const return Field(); } -DB::Field ColumnInfo::getDecimalDefaultValue(const String & str) const +DB::Field ColumnInfo::getDecimalValue(const String & decimal_text) const { - DB::ReadBufferFromString buffer(str); + DB::ReadBufferFromString buffer(decimal_text); auto precision = flen; auto scale = decimal; @@ -102,16 +102,16 @@ DB::Field ColumnInfo::getDecimalDefaultValue(const String & str) const } // FIXME it still has bug: https://github.com/pingcap/tidb/issues/11435 -Int64 ColumnInfo::getEnumIndex(const String & default_str) const +Int64 ColumnInfo::getEnumIndex(const String & enum_id_or_text) const { for (const auto & elem : elems) { - if (elem.first == default_str) + if (elem.first == enum_id_or_text) { return elem.second; } } - int num = std::stoi(default_str); + int num = std::stoi(enum_id_or_text); return num; } diff --git a/dbms/src/Storages/Transaction/TiDB.h b/dbms/src/Storages/Transaction/TiDB.h index efa0e8af5ed..7e0cb03996a 100644 --- a/dbms/src/Storages/Transaction/TiDB.h +++ b/dbms/src/Storages/Transaction/TiDB.h @@ -43,16 +43,16 @@ using DB::Timestamp; M(Float, 4, Float, Float32, false) \ M(Double, 5, Float, Float64, false) \ M(Null, 6, Nil, Nothing, false) \ - M(Timestamp, 7, Int, DateTime, false) \ + M(Timestamp, 7, UInt, DateTime, false) \ M(LongLong, 8, Int, Int64, false) \ M(Int24, 9, VarInt, Int32, true) \ - M(Date, 10, Int, Date, false) \ + M(Date, 10, UInt, Date, false) \ M(Time, 11, Duration, Int64, false) \ - M(Datetime, 12, Int, DateTime, false) \ + M(Datetime, 12, UInt, DateTime, false) \ M(Year, 13, Int, Int16, false) \ M(NewDate, 14, Int, Date, false) \ M(Varchar, 15, CompactBytes, String, false) \ - M(Bit, 16, CompactBytes, UInt64, false) \ + M(Bit, 16, VarInt, UInt64, false) \ M(JSON, 0xf5, Json, String, false) \ M(NewDecimal, 0xf6, Decimal, Decimal32, false) \ M(Enum, 0xf7, VarUInt, Enum16, false) \ @@ -179,13 +179,10 @@ struct ColumnInfo COLUMN_FLAGS(M) #undef M - DB::Field convertField(const DB::Field &) const; DB::Field defaultValueToField() const; - Int64 getEnumIndex(const String &) const; CodecFlag getCodecFlag() const; - -private: - DB::Field getDecimalDefaultValue(const String & str) const; + DB::Field getDecimalValue(const String &) const; + Int64 getEnumIndex(const String &) const; }; enum PartitionType diff --git a/tests/mutable-test/txn_dag/data_type_time.test b/tests/mutable-test/txn_dag/data_type_time.test index e86de40cb4a..ce43ca8c732 100644 --- a/tests/mutable-test/txn_dag/data_type_time.test +++ b/tests/mutable-test/txn_dag/data_type_time.test @@ -10,14 +10,18 @@ => DBGInvoke __mock_tidb_table(default, test, 'col_1 Date, col_2 DateTime') => DBGInvoke __refresh_schemas() => DBGInvoke __put_region(4, 0, 100, default, test) -# not supported -#=> DBGInvoke __raft_insert_row(default, test, 4, 50, '2019-06-10', '2019-06-10 09:00:00') +=> DBGInvoke __raft_insert_row(default, test, 4, 50, '2019-06-10', '2019-06-10 09:00:00') -# DAG read full table scan -#=> DBGInvoke dag('select * from default.test') " --dag_planner="optree -#┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┬─col_11─┐ -#│ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ 666.88 │ -#└───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┴────────┘ +=> DBGInvoke dag('select * from default.test') " --dag_planner="optree +┌──────col_1─┬───────────────col_2─┐ +│ 2019-06-10 │ 2019-06-10 09:00:00 │ +└────────────┴─────────────────────┘ + +# Mock DAG doesn't support date/datetime comparison with string, may need type inference and do implicit conversion to literal. +# => DBGInvoke dag('select * from default.test where col_1 = \'2019-06-06\' and col_2 = \'2019-06-10 09:00:00\'') " --dag_planner="optree +# ┌──────col_1─┬───────────────col_2─┐ +# │ 2019-06-10 │ 2019-06-10 09:00:00 │ +# └────────────┴─────────────────────┘ # Clean up. => DBGInvoke __drop_tidb_table(default, test)