diff --git a/dbms/src/Common/FailPoint.h b/dbms/src/Common/FailPoint.h index 368a4e8edc5..f1edb3e1e61 100644 --- a/dbms/src/Common/FailPoint.h +++ b/dbms/src/Common/FailPoint.h @@ -15,11 +15,12 @@ extern const int FAIL_POINT_ERROR; #define FAIL_POINT_REGISTER(name) static constexpr char name[] = #name ""; -#define FAIL_POINT_ENABLE(trigger, name) else if (trigger == name) fiu_enable(name, 1, nullptr, FIU_ONETIME); +#define FAIL_POINT_ENABLE(trigger, name) else if (trigger == name) { fiu_enable(name, 1, nullptr, FIU_ONETIME); } FAIL_POINT_REGISTER(exception_between_drop_meta_and_data) FAIL_POINT_REGISTER(exception_between_alter_data_and_meta) FAIL_POINT_REGISTER(exception_drop_table_during_remove_meta) +FAIL_POINT_REGISTER(exception_between_rename_table_data_and_metadata); #define FAIL_POINT_TRIGGER_EXCEPTION(fail_point) \ fiu_do_on(fail_point, throw Exception("Fail point " #fail_point " is triggered.", ErrorCodes::FAIL_POINT_ERROR);) @@ -33,7 +34,8 @@ class FailPointHelper FAIL_POINT_ENABLE(fail_point_name, exception_between_alter_data_and_meta) FAIL_POINT_ENABLE(fail_point_name, exception_between_drop_meta_and_data) FAIL_POINT_ENABLE(fail_point_name, exception_drop_table_during_remove_meta) + FAIL_POINT_ENABLE(fail_point_name, exception_between_rename_table_data_and_metadata) else throw Exception("Cannot find fail point " + fail_point_name, ErrorCodes::FAIL_POINT_ERROR); } }; -} // namespace DB \ No newline at end of file +} // namespace DB diff --git a/dbms/src/Core/ColumnWithTypeAndName.cpp b/dbms/src/Core/ColumnWithTypeAndName.cpp index eed70a04db3..ee88f705a39 100644 --- a/dbms/src/Core/ColumnWithTypeAndName.cpp +++ b/dbms/src/Core/ColumnWithTypeAndName.cpp @@ -1,7 +1,7 @@ #include +#include #include #include -#include namespace DB @@ -9,14 +9,7 @@ namespace DB ColumnWithTypeAndName ColumnWithTypeAndName::cloneEmpty() const { - ColumnWithTypeAndName res; - - res.name = name; - res.type = type; - res.column_id = column_id; - if (column) - res.column = column->cloneEmpty(); - + ColumnWithTypeAndName res{(column != nullptr ? column->cloneEmpty() : nullptr), type, name, column_id, default_value}; return res; } @@ -24,8 +17,7 @@ ColumnWithTypeAndName ColumnWithTypeAndName::cloneEmpty() const bool ColumnWithTypeAndName::operator==(const ColumnWithTypeAndName & other) const { // TODO should we check column_id here? - return name == other.name - && ((!type && !other.type) || (type && other.type && type->equals(*other.type))) + return name == other.name && ((!type && !other.type) || (type && other.type && type->equals(*other.type))) && ((!column && !other.column) || (column && other.column && column->getName() == other.column->getName())); } @@ -43,6 +35,8 @@ void ColumnWithTypeAndName::dumpStructure(WriteBuffer & out) const out << ' ' << column->dumpStructure(); else out << " nullptr"; + + out << " " << column_id; } String ColumnWithTypeAndName::dumpStructure() const @@ -52,4 +46,4 @@ String ColumnWithTypeAndName::dumpStructure() const return out.str(); } -} +} // namespace DB diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 1c0764bc798..85de32ac0d7 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -328,6 +328,9 @@ void DatabaseOrdinary::removeTable( try { + // If tiflash crash before remove metadata, next time it restart, will + // full apply schema from TiDB. And the old table's metadata and data + // will be removed. FAIL_POINT_TRIGGER_EXCEPTION(exception_drop_table_during_remove_meta); Poco::File(table_metadata_path).remove(); } @@ -410,6 +413,9 @@ void DatabaseOrdinary::renameTable( throw Exception{e}; } + // TODO: Atomic rename table is not fixed. + FAIL_POINT_TRIGGER_EXCEPTION(exception_between_rename_table_data_and_metadata); + ASTPtr ast = getQueryFromMetadata(detail::getTableMetadataPath(metadata_path, table_name)); if (!ast) throw Exception("There is no metadata file for table " + table_name, ErrorCodes::FILE_DOESNT_EXIST); @@ -417,6 +423,7 @@ void DatabaseOrdinary::renameTable( ast_create_query.table = to_table_name; /// NOTE Non-atomic. + // Create new metadata and remove old metadata. to_database_concrete->createTable(context, to_table_name, table, ast); removeTable(context, table_name); } diff --git a/dbms/src/Debug/MockTiDB.cpp b/dbms/src/Debug/MockTiDB.cpp index fef86c8ad57..b740535e9e3 100644 --- a/dbms/src/Debug/MockTiDB.cpp +++ b/dbms/src/Debug/MockTiDB.cpp @@ -191,7 +191,7 @@ TableID MockTiDB::newTable(const String & database_name, const String & table_na else if (engine_type == "buggy") table_info.engine_type = TiDB::StorageEngine::DEBUGGING_MEMORY; else - throw Exception("Unknown engine type : " + engine_type + ", must be 'tmt' or 'dm'", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Unknown engine type : " + engine_type + ", must be 'tmt' or 'dt'", ErrorCodes::BAD_ARGUMENTS); auto table = std::make_shared(database_name, table_name, std::move(table_info)); tables_by_id.emplace(table->table_info.id, table); diff --git a/dbms/src/Debug/dbgFuncMockTiDBTable.cpp b/dbms/src/Debug/dbgFuncMockTiDBTable.cpp index 6cccf890a81..a037520c258 100644 --- a/dbms/src/Debug/dbgFuncMockTiDBTable.cpp +++ b/dbms/src/Debug/dbgFuncMockTiDBTable.cpp @@ -26,7 +26,7 @@ extern const int LOGICAL_ERROR; void MockTiDBTable::dbgFuncMockTiDBTable(Context & context, const ASTs & args, DBGInvoker::Printer output) { if (args.size() != 3 && args.size() != 4 && args.size() != 5) - throw Exception("Args not matched, should be: database-name, table-name, schema-string [, handle_pk_name], [, engine-type(tmt|dm|buggy)]", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Args not matched, should be: database-name, table-name, schema-string [, handle_pk_name], [, engine-type(tmt|dt|buggy)]", ErrorCodes::BAD_ARGUMENTS); const String & database_name = typeid_cast(*args[0]).name; const String & table_name = typeid_cast(*args[1]).name; diff --git a/dbms/src/Debug/dbgFuncMockTiDBTable.h b/dbms/src/Debug/dbgFuncMockTiDBTable.h index 4dd0ece00a7..28f8d7cbed6 100644 --- a/dbms/src/Debug/dbgFuncMockTiDBTable.h +++ b/dbms/src/Debug/dbgFuncMockTiDBTable.h @@ -15,7 +15,7 @@ struct MockTiDBTable // Inject mocked TiDB table. // Usage: // ./storages-client.sh "DBGInvoke mock_tidb_table(database_name, table_name, 'col1 type1, col2 type2, ...'[, engine])" - // engine: [tmt, dm, buggy], tmt by default + // engine: [tmt, dt, buggy], tmt by default static void dbgFuncMockTiDBTable(Context & context, const ASTs & args, DBGInvoker::Printer output); // Inject mocked TiDB table. diff --git a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp index 1576a817a0b..9ce709ab469 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp @@ -432,7 +432,16 @@ void InterpreterDAG::executeTS(const tipb::TableScan & ts, Pipeline & pipeline) info.range_in_table = current_region->getHandleRangeByTable(table_id); query_info.mvcc_query_info->regions_query_info.push_back(info); query_info.mvcc_query_info->concurrent = 0.0; - pipeline.streams = storage->read(required_columns, query_info, context, from_stage, max_block_size, max_streams); + try + { + pipeline.streams = storage->read(required_columns, query_info, context, from_stage, max_block_size, max_streams); + } + catch (DB::Exception & e) + { + e.addMessage("(while creating InputStreams from storage `" + storage->getDatabaseName() + "`.`" + storage->getTableName() + + "`, table_id: " + DB::toString(table_id) + ")"); + throw; + } if (pipeline.streams.empty()) { diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp index 79369ad9c48..7954994a58d 100644 --- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp @@ -109,7 +109,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() Block sample_block = getSampleBlock(); MutableColumns res_columns = sample_block.cloneEmptyColumns(); - OrderedNameSet filtered_names = MutableSupport::instance().hiddenColumns(table->getName()); + const OrderedNameSet filtered_names = MutableSupport::instance().hiddenColumns(table->getName()); for (const auto & column : columns) { diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 3708ee6269b..0a9c8f1cd60 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -1,15 +1,14 @@ -#include #include - +#include +#include #include #include #include #include #include +#include #include #include -#include -#include namespace DB @@ -17,12 +16,12 @@ namespace DB namespace ErrorCodes { - extern const int TABLE_WAS_NOT_DROPPED; - extern const int DATABASE_NOT_EMPTY; - extern const int UNKNOWN_DATABASE; - extern const int READONLY; - extern const int FAIL_POINT_ERROR; -} +extern const int TABLE_WAS_NOT_DROPPED; +extern const int DATABASE_NOT_EMPTY; +extern const int UNKNOWN_DATABASE; +extern const int READONLY; +extern const int FAIL_POINT_ERROR; +} // namespace ErrorCodes InterpreterDropQuery::InterpreterDropQuery(const ASTPtr & query_ptr_, Context & context_) : query_ptr(query_ptr_), context(context_) {} @@ -57,8 +56,8 @@ BlockIO InterpreterDropQuery::execute() { if (drop.database.empty() && !drop.temporary) { - LOG_WARNING((&Logger::get("InterpreterDropQuery")), - "It is recommended to use `DROP TEMPORARY TABLE` to delete temporary tables"); + LOG_WARNING( + (&Logger::get("InterpreterDropQuery")), "It is recommended to use `DROP TEMPORARY TABLE` to delete temporary tables"); } table->shutdown(); /// If table was already dropped by anyone, an exception will be thrown @@ -119,7 +118,8 @@ BlockIO InterpreterDropQuery::execute() if (!drop.detach) { if (!table.first->checkTableCanBeDropped()) - throw Exception("Table " + database_name + "." + table.first->getTableName() + " couldn't be dropped due to failed pre-drop check", + throw Exception( + "Table " + database_name + "." + table.first->getTableName() + " couldn't be dropped due to failed pre-drop check", ErrorCodes::TABLE_WAS_NOT_DROPPED); } @@ -147,13 +147,8 @@ BlockIO InterpreterDropQuery::execute() table.first->is_dropped = true; - // drop is complete, then clean tmt context; - if (auto storage = std::static_pointer_cast(table.first)) - storage->removeFromTMTContext(); - - String database_data_path = database->getDataPath(); - /// If it is not virtual database like Dictionary then drop remaining data dir + const String database_data_path = database->getDataPath(); if (!database_data_path.empty()) { String table_data_path = database_data_path + "/" + escapeForFileName(current_table_name); @@ -161,6 +156,10 @@ BlockIO InterpreterDropQuery::execute() if (Poco::File(table_data_path).exists()) Poco::File(table_data_path).remove(true); } + + // drop is complete, then clean tmt context + if (auto storage = std::dynamic_pointer_cast(table.first); storage) + storage->removeFromTMTContext(); } } @@ -214,4 +213,4 @@ void InterpreterDropQuery::checkAccess(const ASTDropQuery & drop) throw Exception("Cannot drop table in readonly mode", ErrorCodes::READONLY); } -} +} // namespace DB diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index b0cf3ac3113..ea19d6ec5ed 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -447,22 +448,24 @@ int Server::main(const std::vector & /*args*/) } /// "tmt" engine ONLY support disable_bg_flush = false. - /// "dm" engine by default disable_bg_flush = true. + /// "dt" engine ONLY support disable_bg_flush = true. String disable_bg_flush_conf = "raft.disable_bg_flush"; if (engine == ::TiDB::StorageEngine::TMT) { if (config().has(disable_bg_flush_conf) && config().getBool(disable_bg_flush_conf)) - throw Exception( - "Illegal arguments: disable background flush while using engine TxnMergeTree.", ErrorCodes::INVALID_CONFIG_PARAMETER); + throw Exception("Illegal arguments: disable background flush while using engine " + MutableSupport::txn_storage_name, + ErrorCodes::INVALID_CONFIG_PARAMETER); disable_bg_flush = false; } else if (engine == ::TiDB::StorageEngine::DT) { - if (config().has(disable_bg_flush_conf)) - disable_bg_flush = config().getBool(disable_bg_flush_conf); - else - disable_bg_flush = true; + /// If background flush is enabled, read will not triggle schema sync. + /// Which means that we may get the wrong result with outdated schema. + if (config().has(disable_bg_flush_conf) && !config().getBool(disable_bg_flush_conf)) + throw Exception("Illegal arguments: enable background flush while using engine " + MutableSupport::delta_tree_storage_name, + ErrorCodes::INVALID_CONFIG_PARAMETER); + disable_bg_flush = true; } } diff --git a/dbms/src/Storages/DeltaMerge/Delta/CompactDelta.cpp b/dbms/src/Storages/DeltaMerge/Delta/CompactDelta.cpp index d8c62e1d6a9..946cc0d3e1d 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/CompactDelta.cpp +++ b/dbms/src/Storages/DeltaMerge/Delta/CompactDelta.cpp @@ -118,10 +118,13 @@ bool DeltaValueSpace::compact(DMContext & context) if (unlikely(pack->isDeleteRange())) throw Exception("Unexpectedly selected a delete range to compact", ErrorCodes::LOGICAL_ERROR); + // We ensure schema of all packs are the same Block block = pack->isCached() ? readPackFromCache(pack) : readPackFromDisk(pack, reader); size_t block_rows = block.rows(); for (size_t i = 0; i < schema.columns(); ++i) + { compact_columns[i]->insertRangeFrom(*block.getByPosition(i).column, 0, block_rows); + } wbs.removed_log.delPage(pack->data_page); } diff --git a/dbms/src/Storages/DeltaMerge/Delta/Pack.cpp b/dbms/src/Storages/DeltaMerge/Delta/Pack.cpp index 3fa837de455..20082392102 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/Pack.cpp +++ b/dbms/src/Storages/DeltaMerge/Delta/Pack.cpp @@ -1,10 +1,10 @@ -#include - #include #include #include #include +#include #include +#include namespace DB::DM { @@ -185,35 +185,36 @@ Columns readPackFromCache(const PackPtr & pack, const ColumnDefines & column_def // TODO: should be able to use cache data directly, without copy. std::scoped_lock lock(pack->cache->mutex); - auto & cache_block = pack->cache->block; + const auto & cache_block = pack->cache->block; + if constexpr (0) + { + if (pack->schema == nullptr || !checkSchema(cache_block, *pack->schema)) + { + const String pack_schema_str = pack->schema ? pack->schema->dumpStructure() : "(none)"; + const String cache_schema_str = cache_block.dumpStructure(); + throw Exception("Pack[" + pack->toString() + "] schema not match its cache_block! pack: " + pack_schema_str + + ", cache: " + cache_schema_str, + ErrorCodes::LOGICAL_ERROR); + } + } Columns columns; for (size_t i = col_start; i < col_end; ++i) { - auto & col = column_defines[i]; - auto it = pack->colid_to_offset.find(col.id); - if (it == pack->colid_to_offset.end()) + const auto & cd = column_defines[i]; + if (auto it = pack->colid_to_offset.find(cd.id); it != pack->colid_to_offset.end()) { - // TODO: support DDL. - throw Exception("Cannot find column with id" + DB::toString(col.id)); + auto col_offset = it->second; + // Copy data from cache + auto [type, col_data] = pack->getDataTypeAndEmptyColumn(cd.id); + col_data->insertRangeFrom(*cache_block.getByPosition(col_offset).column, pack->cache_offset, pack->rows); + // Cast if need + auto col_converted = convertColumnByColumnDefineIfNeed(type, std::move(col_data), cd); + columns.push_back(std::move(col_converted)); } else { - auto col_offset = it->second; - auto col_data = col.type->createColumn(); - auto & cache_col = cache_block.getByPosition(col_offset).column; - if (unlikely(col_offset >= cache_block.columns() || !cache_col)) - { - String msg = "read column at " + DB::toString(col_offset) // - + ", cache block: columns=" + DB::toString(cache_block.columns()) // - + ", rows=" + DB::toString(cache_block.rows()) // - + ", read col_id: " + DB::toString(col.id) // - + ", pack: " + pack->toString(); - LOG_ERROR(&Logger::get(__FUNCTION__), msg); - throw Exception(msg); - } - - col_data->insertRangeFrom(*cache_col, pack->cache_offset, pack->rows); - columns.push_back(std::move(col_data)); + ColumnPtr column = createColumnWithDefaultValue(cd, pack->rows - pack->cache_offset); + columns.emplace_back(std::move(column)); } } return columns; @@ -253,37 +254,47 @@ Columns readPackFromDisk(const PackPtr & pack, // size_t col_start, size_t col_end) { + const size_t num_columns_read = col_end - col_start; + + Columns columns(num_columns_read); // allocate empty columns + PageReadFields fields; fields.first = pack->data_page; for (size_t index = col_start; index < col_end; ++index) { - auto col_id = column_defines[index].id; - auto it = pack->colid_to_offset.find(col_id); - if (it == pack->colid_to_offset.end()) - // TODO: support DDL. - throw Exception("Cannot find column with id" + DB::toString(col_id)); - else + const auto & cd = column_defines[index]; + if (auto it = pack->colid_to_offset.find(cd.id); it != pack->colid_to_offset.end()) { auto col_index = it->second; fields.second.push_back(col_index); } + else + { + // New column after ddl is not exist in this pack, fill with default value + columns[index - col_start] = createColumnWithDefaultValue(cd, pack->rows); + } } auto page_map = page_reader.read({fields}); Page page = page_map[pack->data_page]; - - Columns columns; for (size_t index = col_start; index < col_end; ++index) { + const size_t index_in_read_columns = index - col_start; + if (columns[index_in_read_columns] != nullptr) + { + // the column is fill with default values. + continue; + } auto col_id = column_defines[index].id; auto col_index = pack->colid_to_offset[col_id]; auto data_buf = page.getFieldData(col_index); - auto & cd = column_defines[index]; - auto col = cd.type->createColumn(); - deserializeColumn(*col, cd.type, data_buf, pack->rows); + const auto & cd = column_defines[index]; + // Deserialize column by pack's schema + auto [type, col_data] = pack->getDataTypeAndEmptyColumn(cd.id); + deserializeColumn(*col_data, type, data_buf, pack->rows); - columns.push_back(std::move(col)); + columns[index_in_read_columns] = convertColumnByColumnDefineIfNeed(type, std::move(col_data), cd); } return columns; diff --git a/dbms/src/Storages/DeltaMerge/Delta/Pack.h b/dbms/src/Storages/DeltaMerge/Delta/Pack.h index 446dc2addc3..75713652267 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/Pack.h +++ b/dbms/src/Storages/DeltaMerge/Delta/Pack.h @@ -23,10 +23,17 @@ void serializeColumn(MemoryWriteBuffer & buf, const IColumn & column, const Data void serializeSavedPacks(WriteBuffer & buf, const Packs & packs); Packs deserializePacks(ReadBuffer & buf); -String packsToString(const Packs & packs); -Block readPackFromCache(const PackPtr & pack); +// Debugging string +String packsToString(const Packs & packs); + +// Read a block from cache / disk according to `pack->schema` +Block readPackFromCache(const PackPtr & pack); +Block readPackFromDisk(const PackPtr & pack, const PageReader & page_reader); + +// Read a block of columns in `column_defines` from cache / disk, +// if `pack->schema` is not match with `column_defines`, take good care of +// ddl cast Columns readPackFromCache(const PackPtr & pack, const ColumnDefines & column_defines, size_t col_start, size_t col_end); -Block readPackFromDisk(const PackPtr & pack, const PageReader & page_reader); Columns readPackFromDisk(const PackPtr & pack, // const PageReader & page_reader, const ColumnDefines & column_defines, diff --git a/dbms/src/Storages/DeltaMerge/Delta/Snapshot.cpp b/dbms/src/Storages/DeltaMerge/Delta/Snapshot.cpp index eb9101975af..4862df37f60 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/Snapshot.cpp +++ b/dbms/src/Storages/DeltaMerge/Delta/Snapshot.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB::DM { @@ -169,6 +170,7 @@ std::pair findPack(const Packs & packs, size_t rows_offset, size const Columns & DeltaValueSpace::Snapshot::getColumnsOfPack(size_t pack_index, size_t col_num) { + // If some columns is already read in this snapshot, we can reuse `packs_data` auto & columns = packs_data[pack_index]; if (columns.size() < col_num) { @@ -211,8 +213,9 @@ size_t DeltaValueSpace::Snapshot::read(const HandleRange & range, MutableColumns if (rows_start_in_pack == rows_end_in_pack) continue; + // TODO: this get the full columns of pack, which may cause unnecessary copying auto & columns = getColumnsOfPack(pack_index, output_columns.size()); - auto & handle_col_data = toColumnVectorData(columns[0]); + auto & handle_col_data = toColumnVectorData(columns[0]); // TODO: Magic number of fixed position of pk if (rows_in_pack_limit == 1) { if (range.check(handle_col_data[rows_start_in_pack])) diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h b/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h index 66769a7b52d..e96ff5d7e68 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h @@ -152,11 +152,16 @@ inline void addColumnToBlock(Block & block, block.insert(std::move(column)); } -inline Block toEmptyBlock(const ColumnDefines & columns) +/// Generate a block from column_defines +inline Block toEmptyBlock(const ColumnDefines & column_defines) { Block block; - for (auto & c : columns) + for (const auto & c : column_defines) + { + /// Usually we use this function to get a header block, + /// maybe columns of all nullptr in this block is enough? addColumnToBlock(block, c.id, c.name, c.type, c.type->createColumn(), c.default_value); + } return block; } @@ -194,7 +199,7 @@ inline bool checkSchema(const Block & a, const Block & b) for (size_t i = 0; i < a.columns(); ++i) { auto & ca = a.getByPosition(i); - auto & cb = a.getByPosition(i); + auto & cb = b.getByPosition(i); bool col_ok = ca.column_id == cb.column_id; bool name_ok = ca.name == cb.name; diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index 20898231ea2..c26ec91ea12 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -71,6 +71,27 @@ DeltaMergeStore::BackgroundTask DeltaMergeStore::MergeDeltaTaskPool::nextTask(Lo // DeltaMergeStore // ================================================ +namespace +{ +// Actually we will always store a column of `_tidb_rowid`, no matter it +// exist in `table_columns` or not. +ColumnDefinesPtr getStoreColumns(const ColumnDefines & table_columns) +{ + auto columns = std::make_shared(); + // First three columns are always _tidb_rowid, _INTERNAL_VERSION, _INTERNAL_DELMARK + columns->emplace_back(getExtraHandleColumnDefine()); + columns->emplace_back(getVersionColumnDefine()); + columns->emplace_back(getTagColumnDefine()); + // Add other columns + for (const auto & col : table_columns) + { + if (col.name != EXTRA_HANDLE_COLUMN_NAME && col.name != VERSION_COLUMN_NAME && col.name != TAG_COLUMN_NAME) + columns->emplace_back(col); + } + return columns; +} +} // namespace + DeltaMergeStore::DeltaMergeStore(Context & db_context, const String & path_, const String & db_name_, @@ -99,20 +120,14 @@ DeltaMergeStore::DeltaMergeStore(Context & db_context, original_table_columns.emplace_back(original_table_handle_define); original_table_columns.emplace_back(getVersionColumnDefine()); original_table_columns.emplace_back(getTagColumnDefine()); - - store_columns = std::make_shared(); - store_columns->emplace_back(getExtraHandleColumnDefine()); - store_columns->emplace_back(getVersionColumnDefine()); - store_columns->emplace_back(getTagColumnDefine()); - - for (auto & col : columns) + for (const auto & col : columns) { if (col.name != original_table_handle_define.name && col.name != VERSION_COLUMN_NAME && col.name != TAG_COLUMN_NAME) original_table_columns.emplace_back(col); - if (col.name != EXTRA_HANDLE_COLUMN_NAME && col.name != VERSION_COLUMN_NAME && col.name != TAG_COLUMN_NAME) - store_columns->emplace_back(col); } - original_header = toEmptyBlock(original_table_columns); + + store_columns = getStoreColumns(original_table_columns); + auto dm_context = newDMContext(db_context, db_context.getSettingsRef()); @@ -192,12 +207,24 @@ DeltaMergeStore::~DeltaMergeStore() { LOG_INFO(log, "Release DeltaMerge Store start [" << db_name << "." << table_name << "]"); - background_pool.removeTask(gc_handle); - background_pool.removeTask(background_task_handle); + shutdown(); LOG_INFO(log, "Release DeltaMerge Store end [" << db_name << "." << table_name << "]"); } +void DeltaMergeStore::shutdown() +{ + bool v = false; + if (!shutdown_called.compare_exchange_strong(v, true)) + return ; + + background_pool.removeTask(gc_handle); + gc_handle = nullptr; + + background_pool.removeTask(background_task_handle); + background_task_handle = nullptr; +} + DMContextPtr DeltaMergeStore::newDMContext(const Context & db_context, const DB::Settings & db_settings) { std::shared_lock lock(read_write_mutex); @@ -1282,24 +1309,28 @@ void DeltaMergeStore::check(const Context & /*db_context*/) } +Block DeltaMergeStore::getHeader() const +{ + return toEmptyBlock(original_table_columns); +} + void DeltaMergeStore::applyAlters(const AlterCommands & commands, const OptionTableInfoConstRef table_info, ColumnID & max_column_id_used, - const Context & /*context*/) + const Context & /* context */) { std::unique_lock lock(read_write_mutex); - ColumnDefines original_table_columns_copy(original_table_columns.begin(), original_table_columns.end()); - auto store_columns_copy = std::make_shared(store_columns->begin(), store_columns->end()); - + ColumnDefines new_original_table_columns(original_table_columns.begin(), original_table_columns.end()); for (const auto & command : commands) { - applyAlter(original_table_columns_copy, command, table_info, max_column_id_used); - applyAlter(*store_columns_copy, command, table_info, max_column_id_used); + applyAlter(new_original_table_columns, command, table_info, max_column_id_used); } - original_table_columns.swap(original_table_columns); - store_columns.swap(store_columns_copy); + auto new_store_columns = getStoreColumns(new_original_table_columns); + + original_table_columns.swap(new_original_table_columns); + store_columns.swap(new_store_columns); } diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h index 464d4d6294d..a56bca76cef 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h @@ -222,6 +222,12 @@ class DeltaMergeStore : private boost::noncopyable const Settings & settings_); ~DeltaMergeStore(); + const String & getDatabaseName() const { return db_name; } + const String & getTableName() const { return table_name; } + + // Stop all background tasks. + void shutdown(); + void write(const Context & db_context, const DB::Settings & db_settings, const Block & block); void deleteRange(const Context & db_context, const DB::Settings & db_settings, const HandleRange & delete_range); @@ -263,7 +269,7 @@ class DeltaMergeStore : private boost::noncopyable const ColumnDefines & getTableColumns() const { return original_table_columns; } const ColumnDefine & getHandle() const { return original_table_handle_define; } - const Block & getHeader() const { return original_header; } + Block getHeader() const; const Settings & getSettings() const { return settings; } DataTypePtr getPKDataType() const { return original_table_handle_define.type; } SortDescription getPrimarySortDescription() const; @@ -306,11 +312,12 @@ class DeltaMergeStore : private boost::noncopyable ColumnDefines original_table_columns; const ColumnDefine original_table_handle_define; - Block original_header; // The columns we actually store. ColumnDefinesPtr store_columns; + std::atomic shutdown_called{false}; + BackgroundProcessingPool & background_pool; BackgroundProcessingPool::TaskHandle gc_handle; BackgroundProcessingPool::TaskHandle background_task_handle; diff --git a/dbms/src/Storages/DeltaMerge/DeltaValueSpace.cpp b/dbms/src/Storages/DeltaMerge/DeltaValueSpace.cpp index 12a03b3b623..4893ef8ed03 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaValueSpace.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaValueSpace.cpp @@ -19,7 +19,6 @@ namespace DB { namespace DM { - const UInt64 DeltaValueSpace::CURRENT_VERSION = 1; using BlockPtr = DeltaValueSpace::BlockPtr; @@ -317,6 +316,18 @@ bool DeltaValueSpace::appendToCache(DMContext & context, const Block & block, si cache = std::make_shared(block); } + if constexpr (0) + { + if (unlikely(!checkSchema(cache->block, block))) + { + const String block_schema_str = block.dumpStructure(); + const String cache_schema_str = cache->block.dumpStructure(); + throw Exception("Try to append block(rows:" + DB::toString(block.rows()) + + ") to a cache but schema not match! block: " + block_schema_str + ", cache: " + cache_schema_str, + ErrorCodes::LOGICAL_ERROR); + } + } + size_t cache_offset; { std::scoped_lock cache_lock(cache->mutex); diff --git a/dbms/src/Storages/DeltaMerge/DeltaValueSpace.h b/dbms/src/Storages/DeltaMerge/DeltaValueSpace.h index 2752ecdd669..7c903dbba26 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaValueSpace.h +++ b/dbms/src/Storages/DeltaMerge/DeltaValueSpace.h @@ -83,6 +83,14 @@ class DeltaValueSpace : public std::enable_shared_from_this, pr colid_to_offset.emplace(schema->getByPosition(i).column_id, i); } + std::pair getDataTypeAndEmptyColumn(ColId column_id) const + { + // Note that column_id must exist + auto index = colid_to_offset.at(column_id); + auto col_type = schema->getByPosition(index).type; + return { col_type, col_type->createColumn() }; + } + String toString() { String s = "{rows:" + DB::toString(rows) // @@ -136,19 +144,25 @@ class DeltaValueSpace : public std::enable_shared_from_this, pr } } - size_t getPackCount() { return packs.size(); } - size_t getRows() { return rows; } - size_t getDeletes() { return deletes; } + size_t getPackCount() const { return packs.size(); } + size_t getRows() const { return rows; } + size_t getDeletes() const { return deletes; } void prepare(const DMContext & context, const ColumnDefines & column_defines_); BlockInputStreamPtr prepareForStream(const DMContext & context, const ColumnDefines & column_defines_); const Columns & getColumnsOfPack(size_t pack_index, size_t col_num); - size_t read(const HandleRange & range, MutableColumns & output_columns, size_t offset, size_t limit); - Block read(size_t col_num, size_t offset, size_t limit); - Block read(size_t pack_index); + // Get blocks or delete_ranges of `ExtraHandleColumn` and `VersionColumn`. + // If there are continuous blocks, they will be squashed into one block. + // We use the result to update DeltaTree. BlockOrDeletes getMergeBlocks(size_t rows_begin, size_t deletes_begin, size_t rows_end, size_t deletes_end); + + Block read(size_t pack_index); + size_t read(const HandleRange & range, MutableColumns & output_columns, size_t offset, size_t limit); + + private: + Block read(size_t col_num, size_t offset, size_t limit); }; using SnapshotPtr = std::shared_ptr; @@ -216,7 +230,7 @@ class DeltaValueSpace : public std::enable_shared_from_this, pr throw Exception("Try to abandon a already abandoned DeltaValueSpace", ErrorCodes::LOGICAL_ERROR); } - bool hasAbandoned() { return abandoned.load(std::memory_order_relaxed); } + bool hasAbandoned() const { return abandoned.load(std::memory_order_relaxed); } /// Restore the metadata of this instance. /// Only called after reboot. diff --git a/dbms/src/Storages/DeltaMerge/File/DMFile.cpp b/dbms/src/Storages/DeltaMerge/File/DMFile.cpp index 2bb574b2e41..b396b300262 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFile.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFile.cpp @@ -1,19 +1,26 @@ -#include -#include -#include - #include #include #include #include +#include #include #include +#include +#include + namespace DB { namespace DM { +static constexpr const char * NGC_FILE_NAME = "NGC"; + +String DMFile::ngcPath() const +{ + return path() + "/" + NGC_FILE_NAME; +} + DMFilePtr DMFile::create(UInt64 file_id, const String & parent_path) { Logger * log = &Logger::get("DMFile"); diff --git a/dbms/src/Storages/DeltaMerge/File/DMFile.h b/dbms/src/Storages/DeltaMerge/File/DMFile.h index 0fdde57f665..2f012c6288d 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFile.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFile.h @@ -15,8 +15,6 @@ class DMFile; using DMFilePtr = std::shared_ptr; using DMFiles = std::vector; -static const String NGC_FILE_NAME = "NGC"; - class DMFile : private boost::noncopyable { public: @@ -65,26 +63,18 @@ class DMFile : private boost::noncopyable void enableGC(); void remove(); - UInt64 fileId() { return file_id; } - UInt64 refId() { return ref_id; } - String path() { return parent_path + (status == Status::READABLE ? "/dmf_" : "/.tmp.dmf_") + DB::toString(file_id); } - String metaPath() { return path() + "/meta.txt"; } - String packStatPath() { return path() + "/pack"; } + UInt64 fileId() const { return file_id; } + UInt64 refId() const { return ref_id; } + String path() const { return parent_path + (status == Status::READABLE ? "/dmf_" : "/.tmp.dmf_") + DB::toString(file_id); } + String metaPath() const { return path() + "/meta.txt"; } + String packStatPath() const { return path() + "/pack"; } // Do not gc me. - String ngcPath() { return path() + "/" + NGC_FILE_NAME; } - String colDataPath(const String & file_name_base) { return path() + "/" + file_name_base + ".dat"; } - String colIndexPath(const String & file_name_base) { return path() + "/" + file_name_base + ".idx"; } - String colMarkPath(const String & file_name_base) { return path() + "/" + file_name_base + ".mrk"; } + String ngcPath() const; + String colDataPath(const String & file_name_base) const { return path() + "/" + file_name_base + ".dat"; } + String colIndexPath(const String & file_name_base) const { return path() + "/" + file_name_base + ".idx"; } + String colMarkPath(const String & file_name_base) const { return path() + "/" + file_name_base + ".mrk"; } - const ColumnStat & getColumnStat(ColId col_id) - { - auto it = column_stats.find(col_id); - if (it == column_stats.end()) - throw Exception("Column [" + DB::toString(col_id) + "] not found in dm file [" + path() + "]"); - return it->second; - } - - size_t getRows() + size_t getRows() const { size_t rows = 0; for (auto & s : pack_stats) @@ -92,7 +82,7 @@ class DMFile : private boost::noncopyable return rows; } - size_t getBytes() + size_t getBytes() const { size_t bytes = 0; for (auto & s : pack_stats) @@ -100,11 +90,22 @@ class DMFile : private boost::noncopyable return bytes; } - size_t getPacks() { return pack_stats.size(); } - const PackStats & getPackStats() { return pack_stats; } - const PackStat & getPackStat(size_t pack_index) { return pack_stats[pack_index]; } - const ColumnStats & getColumnStats() { return column_stats; } - Status getStatus() { return status; } + size_t getPacks() const { return pack_stats.size(); } + const PackStats & getPackStats() const { return pack_stats; } + const PackStat & getPackStat(size_t pack_index) const { return pack_stats[pack_index]; } + + const ColumnStats & getColumnStats() const { return column_stats; } + const ColumnStat & getColumnStat(ColId col_id) const + { + if (auto it = column_stats.find(col_id); it != column_stats.end()) + { + return it->second; + } + throw Exception("Column [" + DB::toString(col_id) + "] not found in dm file [" + path() + "]"); + } + bool isColumnExist(ColId col_id) const { return column_stats.find(col_id) != column_stats.end(); } + + Status getStatus() const { return status; } static String getFileNameBase(ColId col_id, const IDataType::SubstreamPath & substream = {}) { diff --git a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h index 935b42934cb..826ebfde96c 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h @@ -151,4 +151,4 @@ class DMFilePackFilter }; } // namespace DM -} // namespace DB \ No newline at end of file +} // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp index a0a14a79d05..f52955f2d62 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp @@ -1,8 +1,8 @@ -#include - #include #include +#include #include +#include #include namespace DB @@ -18,8 +18,8 @@ DMFileReader::Stream::Stream(DMFileReader & reader, // Logger * log) : avg_size_hint(reader.dmfile->getColumnStat(col_id).avg_size) { - String mark_path = reader.dmfile->colMarkPath(file_name_base); - String data_path = reader.dmfile->colDataPath(file_name_base); + const String mark_path = reader.dmfile->colMarkPath(file_name_base); + const String data_path = reader.dmfile->colDataPath(file_name_base); auto mark_load = [&]() -> MarksInCompressedFilePtr { auto res = std::make_shared(reader.dmfile->getPacks()); @@ -39,7 +39,7 @@ DMFileReader::Stream::Stream(DMFileReader & reader, // marks = mark_load(); size_t data_file_size = Poco::File(data_path).getSize(); - size_t packs = reader.dmfile->getPacks(); + size_t packs = reader.dmfile->getPacks(); size_t buffer_size = 0; size_t estimated_size = 0; for (size_t i = 0; i < packs;) @@ -125,19 +125,27 @@ DMFileReader::DMFileReader(bool enable_clean_read_, throw Exception("DMFile [" + DB::toString(dmfile->fileId()) + "] is expected to be in READABLE status, but: " + DMFile::statusString(dmfile->getStatus())); - for (auto & cd : read_columns) + for (const auto & cd : read_columns) { + // New inserted column, fill them with default value later + if (!dmfile->isColumnExist(cd.id)) + continue; + auto callback = [&](const IDataType::SubstreamPath & substream) { String stream_name = DMFile::getFileNameBase(cd.id, substream); - auto stream = std::make_unique(*this, // - cd.id, - stream_name, - aio_threshold, - max_read_buffer_size, - log); + auto stream = std::make_unique( // + *this, + cd.id, + stream_name, + aio_threshold, + max_read_buffer_size, + log); column_streams.emplace(stream_name, std::move(stream)); }; - cd.type->enumerateStreams(callback, {}); + + // Load stream according to DataType in disk + const auto data_type = dmfile->getColumnStat(cd.id).type; + data_type->enumerateStreams(callback, {}); } } @@ -174,7 +182,7 @@ Block DMFileReader::read() // Find max continuing rows we can read. size_t start_pack_id = next_pack_id; - auto & pack_stats = dmfile->getPackStats(); + auto & pack_stats = dmfile->getPackStats(); size_t read_rows = 0; size_t not_clean_rows = 0; @@ -188,7 +196,7 @@ Block DMFileReader::read() not_clean_rows += pack_stats[next_pack_id].not_clean; } - if (!read_rows) + if (read_rows == 0) return {}; Block res; @@ -232,28 +240,42 @@ Block DMFileReader::read() } else { - String stream_name = DMFile::getFileNameBase(cd.id); - auto & stream = column_streams.at(stream_name); - if (shouldSeek(start_pack_id) || skip_packs_by_column[i] > 0) + const String stream_name = DMFile::getFileNameBase(cd.id); + if (auto iter = column_streams.find(stream_name); iter != column_streams.end()) { - auto & mark = (*stream->marks)[start_pack_id]; - stream->buf->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); + auto & stream = iter->second; + if (shouldSeek(start_pack_id) || skip_packs_by_column[i] > 0) + { + auto & mark = (*stream->marks)[start_pack_id]; + stream->buf->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); + } + + auto data_type = dmfile->getColumnStat(cd.id).type; + auto column = data_type->createColumn(); + data_type->deserializeBinaryBulkWithMultipleStreams( // + *column, + [&](const IDataType::SubstreamPath & substream_path) { + String substream_name = DMFile::getFileNameBase(cd.id, substream_path); + auto & sub_stream = column_streams.at(substream_name); + return sub_stream->buf.get(); + }, + read_rows, + stream->avg_size_hint, + true, + {}); + IDataType::updateAvgValueSizeHint(*column, stream->avg_size_hint); + + // Cast column's data from DataType in disk to what we need now + auto converted_column = convertColumnByColumnDefineIfNeed(data_type, std::move(column), cd); + + res.insert(ColumnWithTypeAndName{std::move(converted_column), cd.type, cd.name, cd.id}); + } + else + { + // New column after ddl is not exist in this DMFile, fill with default value + ColumnPtr column = createColumnWithDefaultValue(cd, read_rows); + res.insert(ColumnWithTypeAndName{std::move(column), cd.type, cd.name, cd.id}); } - - auto column = cd.type->createColumn(); - cd.type->deserializeBinaryBulkWithMultipleStreams(*column, // - [&](const IDataType::SubstreamPath & substream) { - String name = DMFile::getFileNameBase(cd.id, substream); - auto & stream = column_streams.at(name); - return stream->buf.get(); - }, - read_rows, - stream->avg_size_hint, - true, - {}); - IDataType::updateAvgValueSizeHint(*column, stream->avg_size_hint); - - res.insert(ColumnWithTypeAndName{std::move(column), cd.type, cd.name, cd.id}); skip_packs_by_column[i] = 0; } diff --git a/dbms/src/Storages/DeltaMerge/SchemaUpdate.cpp b/dbms/src/Storages/DeltaMerge/SchemaUpdate.cpp index 45849090036..ebb30826cd2 100644 --- a/dbms/src/Storages/DeltaMerge/SchemaUpdate.cpp +++ b/dbms/src/Storages/DeltaMerge/SchemaUpdate.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -11,15 +12,21 @@ namespace DB namespace DM { +String astToDebugString(const IAST * const ast) +{ + std::stringstream ss; + ast->dumpTree(ss); + return ss.str(); +} + inline void setColumnDefineDefaultValue(const AlterCommand & command, ColumnDefine & define) { - std::function castDefaultValue; // for lazy bind - castDefaultValue = [&](Field value, DataTypePtr type) -> Field { + std::function castDefaultValue; // for lazy bind + castDefaultValue = [&](const Field & value, const DataTypePtr & type) -> Field { switch (type->getTypeId()) { case TypeIndex::Float32: - case TypeIndex::Float64: - { + case TypeIndex::Float64: { if (value.getType() == Field::Types::Float64) { Float64 res = applyVisitor(FieldVisitorConvertToNumber(), value); @@ -39,117 +46,148 @@ inline void setColumnDefineDefaultValue(const AlterCommand & command, ColumnDefi } else { - throw Exception("Unknown float number literal"); + throw Exception("Unknown float number literal: " + applyVisitor(FieldVisitorToString(), value) + + ", value type: " + value.getTypeName()); } } - case TypeIndex::FixedString: - { + case TypeIndex::String: + case TypeIndex::FixedString: { String res = get(value); return toField(res); } case TypeIndex::Int8: case TypeIndex::Int16: case TypeIndex::Int32: - case TypeIndex::Int64: - { + case TypeIndex::Int64: { Int64 res = applyVisitor(FieldVisitorConvertToNumber(), value); return toField(res); } case TypeIndex::UInt8: case TypeIndex::UInt16: case TypeIndex::UInt32: - case TypeIndex::UInt64: - { + case TypeIndex::UInt64: { UInt64 res = applyVisitor(FieldVisitorConvertToNumber(), value); return toField(res); } - case TypeIndex::DateTime: - { + case TypeIndex::DateTime: { auto date = safeGet(value); time_t time = 0; ReadBufferFromMemory buf(date.data(), date.size()); readDateTimeText(time, buf); return toField((Int64)time); } - case TypeIndex::Decimal32: - { - auto dec = std::dynamic_pointer_cast(type); - Int64 v = applyVisitor(FieldVisitorConvertToNumber(), value); - ScaleType scale = dec->getScale(); - return DecimalField(Decimal32(v), scale); + case TypeIndex::Decimal32: { + auto v = safeGet>(value); + return v; } - case TypeIndex::Decimal64: - { - auto dec = std::dynamic_pointer_cast(type); - Int64 v = applyVisitor(FieldVisitorConvertToNumber(), value); - ScaleType scale = dec->getScale(); - return DecimalField(Decimal64(v), scale); + case TypeIndex::Decimal64: { + auto v = safeGet>(value); + return v; } - case TypeIndex::Decimal128: - { - auto dec = std::dynamic_pointer_cast(type); - Int64 v = applyVisitor(FieldVisitorConvertToNumber(), value); - ScaleType scale = dec->getScale(); - return DecimalField(Decimal128(v), scale); + case TypeIndex::Decimal128: { + auto v = safeGet>(value); + return v; } - case TypeIndex::Decimal256: - { - auto dec = std::dynamic_pointer_cast(type); - Int64 v = applyVisitor(FieldVisitorConvertToNumber(), value); - ScaleType scale = dec->getScale(); - return DecimalField(Decimal256(v), scale); + case TypeIndex::Decimal256: { + auto v = safeGet>(value); + return v; } - case TypeIndex::Nullable: - { + case TypeIndex::Enum16: { + // According to `Storages/Transaction/TiDB.h` and MySQL 5.7 + // document(https://dev.mysql.com/doc/refman/5.7/en/enum.html), + // enum support 65,535 distinct value at most, so only Enum16 is supported here. + // Default value of Enum should be store as a Int64 Field (Storages/Transaction/Datum.cpp) + Int64 res = applyVisitor(FieldVisitorConvertToNumber(), value); + return toField(res); + } + case TypeIndex::MyDate: + case TypeIndex::MyDateTime: { + static_assert(std::is_same_v); + static_assert(std::is_same_v); + UInt64 res = applyVisitor(FieldVisitorConvertToNumber(), value); + return toField(res); + } + case TypeIndex::Nullable: { if (value.isNull()) return value; auto nullable = std::dynamic_pointer_cast(type); DataTypePtr nested_type = nullable->getNestedType(); - return castDefaultValue(value, nested_type); + return castDefaultValue(value, nested_type); // Recursive call on nested type } default: - throw Exception("Unsupported data type: " + type->getName()); + throw Exception("Unsupported to setColumnDefineDefaultValue with data type: " + type->getName() + + " value: " + applyVisitor(FieldVisitorToString(), value) + ", type: " + value.getTypeName()); } }; if (command.default_expression) { - // a cast function - // change column_define.default_value - - if (auto default_literal = typeid_cast(command.default_expression.get()); - default_literal && default_literal->value.getType() == Field::Types::String) - { - define.default_value = default_literal->value; - } - else if (auto default_cast_expr = typeid_cast(command.default_expression.get()); - default_cast_expr && default_cast_expr->name == "CAST" /* ParserCastExpression::name */) + try { - // eg. CAST('1.234' AS Float32); CAST(999 AS Int32) - if (default_cast_expr->arguments->children.size() != 2) + // a cast function + // change column_define.default_value + + if (auto default_literal = typeid_cast(command.default_expression.get()); + default_literal && default_literal->value.getType() == Field::Types::String) { - throw Exception("Unknown CAST expression in default expr", ErrorCodes::NOT_IMPLEMENTED); + define.default_value = default_literal->value; } - - auto default_literal_in_cast = typeid_cast(default_cast_expr->arguments->children[0].get()); - if (default_literal_in_cast) + else if (auto default_cast_expr = typeid_cast(command.default_expression.get()); + default_cast_expr && default_cast_expr->name == "CAST" /* ParserCastExpression::name */) { - Field default_value = castDefaultValue(default_literal_in_cast->value, define.type); - define.default_value = default_value; + // eg. CAST('1.234' AS Float32); CAST(999 AS Int32) + if (default_cast_expr->arguments->children.size() != 2) + { + throw Exception("Unknown CAST expression in default expr", ErrorCodes::NOT_IMPLEMENTED); + } + + auto default_literal_in_cast = typeid_cast(default_cast_expr->arguments->children[0].get()); + if (default_literal_in_cast) + { + Field default_value = castDefaultValue(default_literal_in_cast->value, define.type); + define.default_value = default_value; + } + else + { + throw Exception("Invalid CAST expression", ErrorCodes::BAD_ARGUMENTS); + } } else { - throw Exception("Invalid CAST expression", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Default value must be a string or CAST('...' AS WhatType)", ErrorCodes::BAD_ARGUMENTS); } } - else + catch (DB::Exception & e) + { + e.addMessage("(in setColumnDefineDefaultValue for default_expression:" + astToDebugString(command.default_expression.get()) + + ")"); + throw; + } + catch (const Poco::Exception & e) + { + DB::Exception ex(e); + ex.addMessage("(in setColumnDefineDefaultValue for default_expression:" + astToDebugString(command.default_expression.get()) + + ")"); + throw ex; + } + catch (std::exception & e) { - throw Exception("Default value must be a string or CAST('...' AS WhatType)", ErrorCodes::BAD_ARGUMENTS); + std::stringstream ss; + ss << "std::exception: " << e.what() + << " (in setColumnDefineDefaultValue for default_expression:" + astToDebugString(command.default_expression.get()) << ")"; + DB::Exception ex(ss.str(), ErrorCodes::LOGICAL_ERROR); + throw ex; } } } +inline void setColumnDefineDefaultValue(const OptionTableInfoConstRef table_info, ColumnDefine & define) +{ + const auto col_info = table_info->get().getColumnInfo(define.id); + define.default_value = col_info.defaultValueToField(); +} + void applyAlter(ColumnDefines & table_columns, const AlterCommand & command, const OptionTableInfoConstRef table_info, @@ -170,7 +208,14 @@ void applyAlter(ColumnDefines & table_columns, { exist_column = true; column_define.type = command.data_type; - setColumnDefineDefaultValue(command, column_define); + if (table_info) + { + setColumnDefineDefaultValue(table_info, column_define); + } + else + { + setColumnDefineDefaultValue(command, column_define); + } break; } } @@ -206,12 +251,13 @@ void applyAlter(ColumnDefines & table_columns, if (table_info) { define.id = table_info->get().getColumnID(command.column_name); + setColumnDefineDefaultValue(table_info, define); } else { define.id = max_column_id_used++; + setColumnDefineDefaultValue(command, define); } - setColumnDefineDefaultValue(command, define); table_columns.emplace_back(std::move(define)); } else if (command.type == AlterCommand::DROP_COLUMN) @@ -238,4 +284,4 @@ void applyAlter(ColumnDefines & table_columns, } } // namespace DM -} // namespace DB \ No newline at end of file +} // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp b/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp index 29e98fea9b8..324fe82f5a3 100644 --- a/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp +++ b/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp @@ -100,15 +100,16 @@ SkippableBlockInputStreamPtr StableValueSpace::getInputStream(const DMContext & SkippableBlockInputStreams streams; for (auto & file : files) { - streams.push_back(std::make_shared(context.db_context, // - max_data_version, - enable_clean_read, - context.hash_salt, - file, - read_columns, - handle_range, - filter, - IdSetPtr{})); + streams.push_back(std::make_shared( // + context.db_context, + max_data_version, + enable_clean_read, + context.hash_salt, + file, + read_columns, + handle_range, + filter, + IdSetPtr{})); } return std::make_shared(streams); } diff --git a/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.cpp b/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.cpp new file mode 100644 index 00000000000..362e4aa20c7 --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.cpp @@ -0,0 +1,360 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace DM +{ + +namespace +{ + +/// some helper functions for casting column data type + +template +void insertRangeFromWithNumericTypeCast(const ColumnPtr & from_col, // + const ColumnPtr & null_map, + const ColumnDefine & read_define, + MutableColumnPtr & to_col, + size_t rows_offset, + size_t rows_limit) +{ + // Caller should ensure that both from_col / to_col + // * is numeric + // * no nullable wrapper + // * both signed or unsigned + static_assert(std::is_integral_v); + static_assert(std::is_integral_v); + constexpr bool is_both_signed_or_unsigned = !(std::is_unsigned_v ^ std::is_unsigned_v); + static_assert(is_both_signed_or_unsigned); + assert(from_col != nullptr); + assert(to_col != nullptr); + assert(from_col->isNumeric()); + assert(to_col->isNumeric()); + assert(!from_col->isColumnNullable()); + assert(!to_col->isColumnNullable()); + assert(!from_col->isColumnConst()); + assert(!to_col->isColumnConst()); + + // Something like `insertRangeFrom(from_col, rows_offset, rows_limit)` with static_cast + const PaddedPODArray & from_array = toColumnVectorData(from_col); + PaddedPODArray * to_array_ptr = toMutableColumnVectorDataPtr(to_col); + to_array_ptr->reserve(rows_limit); + for (size_t i = 0; i < rows_limit; ++i) + { + (*to_array_ptr).emplace_back(static_cast(from_array[rows_offset + i])); + } + + if (unlikely(null_map)) + { + /// We are applying cast from nullable to not null, scan to fill "NULL" with default value + + TypeTo default_value = 0; // if read_define.default_value is empty, fill with 0 + if (read_define.default_value.isNull()) + { + // Do nothing + } + else if (read_define.default_value.getType() == Field::Types::Int64) + { + default_value = read_define.default_value.safeGet(); + } + else if (read_define.default_value.getType() == Field::Types::UInt64) + { + default_value = read_define.default_value.safeGet(); + } + else + { + throw Exception("Invalid column value type", ErrorCodes::BAD_ARGUMENTS); + } + + const size_t to_offset_before_inserted = to_array_ptr->size() - rows_limit; + + for (size_t i = 0; i < rows_limit; ++i) + { + const size_t to_offset = to_offset_before_inserted + i; + if (null_map->getInt(rows_offset + i) != 0) + { + // `from_col[rows_offset + i]` is "NULL", fill `to_col[x]` with default value + (*to_array_ptr)[to_offset] = static_cast(default_value); + } + } + } +} + + +bool castNonNullNumericColumn(const DataTypePtr & disk_type_not_null_, + const ColumnPtr & disk_col_not_null, + const ColumnDefine & read_define, + const ColumnPtr & null_map, + MutableColumnPtr & memory_col_not_null, + size_t rows_offset, + size_t rows_limit) +{ + /// Caller should ensure that type is not nullable + assert(disk_type_not_null_ != nullptr); + assert(disk_col_not_null != nullptr); + assert(read_define.type != nullptr); + assert(memory_col_not_null != nullptr); + + const IDataType * disk_type_not_null = disk_type_not_null_.get(); + const IDataType * read_type_not_null = read_define.type.get(); + + /// Caller should ensure nullable is unwrapped + assert(!disk_type_not_null->isNullable()); + assert(!read_type_not_null->isNullable()); + + /// Caller should ensure that dist_type != read_type + assert(!disk_type_not_null->equals(*read_type_not_null)); + + if (checkDataType(disk_type_not_null)) + { + using FromType = UInt32; + if (checkDataType(read_type_not_null)) + { + insertRangeFromWithNumericTypeCast( + disk_col_not_null, null_map, read_define, memory_col_not_null, rows_offset, rows_limit); + return true; + } + } + else if (checkDataType(disk_type_not_null)) + { + using FromType = Int32; + if (checkDataType(read_type_not_null)) + { + insertRangeFromWithNumericTypeCast( + disk_col_not_null, null_map, read_define, memory_col_not_null, rows_offset, rows_limit); + return true; + } + } + else if (checkDataType(disk_type_not_null)) + { + using FromType = UInt16; + if (checkDataType(read_type_not_null)) + { + insertRangeFromWithNumericTypeCast( + disk_col_not_null, null_map, read_define, memory_col_not_null, rows_offset, rows_limit); + return true; + } + else if (checkDataType(read_type_not_null)) + { + insertRangeFromWithNumericTypeCast( + disk_col_not_null, null_map, read_define, memory_col_not_null, rows_offset, rows_limit); + return true; + } + } + else if (checkDataType(disk_type_not_null)) + { + using FromType = Int16; + if (checkDataType(read_type_not_null)) + { + insertRangeFromWithNumericTypeCast( + disk_col_not_null, null_map, read_define, memory_col_not_null, rows_offset, rows_limit); + return true; + } + else if (checkDataType(read_type_not_null)) + { + insertRangeFromWithNumericTypeCast( + disk_col_not_null, null_map, read_define, memory_col_not_null, rows_offset, rows_limit); + return true; + } + } + else if (checkDataType(disk_type_not_null)) + { + using FromType = UInt8; + if (checkDataType(read_type_not_null)) + { + insertRangeFromWithNumericTypeCast( + disk_col_not_null, null_map, read_define, memory_col_not_null, rows_offset, rows_limit); + return true; + } + else if (checkDataType(read_type_not_null)) + { + insertRangeFromWithNumericTypeCast( + disk_col_not_null, null_map, read_define, memory_col_not_null, rows_offset, rows_limit); + return true; + } + else if (checkDataType(read_type_not_null)) + { + insertRangeFromWithNumericTypeCast( + disk_col_not_null, null_map, read_define, memory_col_not_null, rows_offset, rows_limit); + return true; + } + } + else if (checkDataType(disk_type_not_null)) + { + using FromType = Int8; + if (checkDataType(read_type_not_null)) + { + insertRangeFromWithNumericTypeCast( + disk_col_not_null, null_map, read_define, memory_col_not_null, rows_offset, rows_limit); + return true; + } + else if (checkDataType(read_type_not_null)) + { + insertRangeFromWithNumericTypeCast( + disk_col_not_null, null_map, read_define, memory_col_not_null, rows_offset, rows_limit); + return true; + } + else if (checkDataType(read_type_not_null)) + { + insertRangeFromWithNumericTypeCast( + disk_col_not_null, null_map, read_define, memory_col_not_null, rows_offset, rows_limit); + return true; + } + } + + // else is not support + return false; +} + +} // namespace + +void convertColumnByColumnDefine(const DataTypePtr & disk_type, + const ColumnPtr & disk_col, + const ColumnDefine & read_define, + MutableColumnPtr memory_col, + size_t rows_offset, + size_t rows_limit) +{ + const DataTypePtr & read_type = read_define.type; + + // Unwrap nullable(what) + ColumnPtr disk_col_not_null; + MutableColumnPtr memory_col_not_null; + ColumnPtr null_map; + DataTypePtr disk_type_not_null = disk_type; + DataTypePtr read_type_not_null = read_type; + if (disk_type->isNullable() && read_type->isNullable()) + { + // nullable -> nullable, copy null map + const auto & disk_nullable_col = typeid_cast(*disk_col); + const auto & disk_null_map = disk_nullable_col.getNullMapData(); + auto & memory_nullable_col = typeid_cast(*memory_col); + auto & memory_null_map = memory_nullable_col.getNullMapData(); + memory_null_map.insert(disk_null_map.begin(), disk_null_map.end()); + + disk_col_not_null = disk_nullable_col.getNestedColumnPtr(); + memory_col_not_null = memory_nullable_col.getNestedColumn().getPtr(); + + const auto * type_nullable = typeid_cast(disk_type.get()); + disk_type_not_null = type_nullable->getNestedType(); + type_nullable = typeid_cast(read_type.get()); + read_type_not_null = type_nullable->getNestedType(); + } + else if (!disk_type->isNullable() && read_type->isNullable()) + { + // not null -> nullable, set null map to all not null + auto & memory_nullable_col = typeid_cast(*memory_col); + auto & nullmap_data = memory_nullable_col.getNullMapData(); + nullmap_data.resize_fill(rows_offset + rows_limit, 0); + + disk_col_not_null = disk_col; + memory_col_not_null = memory_nullable_col.getNestedColumn().getPtr(); + + const auto * type_nullable = typeid_cast(read_type.get()); + read_type_not_null = type_nullable->getNestedType(); + } + else if (disk_type->isNullable() && !read_type->isNullable()) + { + // nullable -> not null, fill "NULL" values with default value later + const auto & disk_nullable_col = typeid_cast(*disk_col); + null_map = disk_nullable_col.getNullMapColumnPtr(); + disk_col_not_null = disk_nullable_col.getNestedColumnPtr(); + memory_col_not_null = std::move(memory_col); + + const auto * type_nullable = typeid_cast(disk_type.get()); + disk_type_not_null = type_nullable->getNestedType(); + } + else + { + // not null -> not null + disk_col_not_null = disk_col; + memory_col_not_null = std::move(memory_col); + } + + assert(memory_col_not_null != nullptr); + assert(disk_col_not_null != nullptr); + assert(read_type_not_null != nullptr); + assert(disk_type_not_null != nullptr); + + ColumnDefine read_define_not_null(read_define); + read_define_not_null.type = read_type_not_null; + if (disk_type_not_null->equals(*read_type_not_null)) + { + // just change from nullable -> not null / not null -> nullable + memory_col_not_null->insertRangeFrom(*disk_col_not_null, rows_offset, rows_limit); + + if (null_map) + { + /// We are applying cast from nullable to not null, scan to fill "NULL" with default value + + for (size_t i = 0; i < rows_limit; ++i) + { + if (unlikely(null_map->getInt(i) != 0)) + { + // `from_col[i]` is "NULL", fill `to_col[rows_offset + i]` with default value + // TiDB/MySQL don't support this, should not call here. + throw Exception("Reading mismatch data type pack. Cast from " + disk_type->getName() + " to " + read_type->getName() + + " with \"NULL\" value is NOT supported!", + ErrorCodes::NOT_IMPLEMENTED); + } + } + } + } + else if (!castNonNullNumericColumn( + disk_type_not_null, disk_col_not_null, read_define_not_null, null_map, memory_col_not_null, rows_offset, rows_limit)) + { + throw Exception("Reading mismatch data type pack. Cast and assign from " + disk_type->getName() + " to " + read_type->getName() + + " is NOT supported!", + ErrorCodes::NOT_IMPLEMENTED); + } +} + +ColumnPtr convertColumnByColumnDefineIfNeed(const DataTypePtr & from_type, ColumnPtr && from_col, const ColumnDefine & to_column_define) +{ + // No need to convert + if (likely(from_type->equals(*to_column_define.type))) + return std::move(from_col); + + // Check if support + if (unlikely(!isSupportedDataTypeCast(from_type, to_column_define.type))) + { + throw Exception("Reading mismatch data type pack. Cast from " + from_type->getName() + " to " + to_column_define.type->getName() + + " is NOT supported!", + ErrorCodes::NOT_IMPLEMENTED); + } + + // Cast column's data from DataType in disk to what we need now + auto to_col = to_column_define.type->createColumn(); + to_col->reserve(from_col->size()); + convertColumnByColumnDefine(from_type, from_col, to_column_define, to_col->getPtr(), 0, from_col->size()); + return to_col; +} + +ColumnPtr createColumnWithDefaultValue(const ColumnDefine & column_define, size_t num_rows) +{ + ColumnPtr column; + // Read default value from `column_define.default_value` + if (column_define.default_value.isNull()) + { + column = column_define.type->createColumnConstWithDefaultValue(num_rows); + } + else + { + column = column_define.type->createColumnConst(num_rows, column_define.default_value); + } + column = column->convertToFullColumnIfConst(); + return column; +} + +} // namespace DM +} // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.h b/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.h new file mode 100644 index 00000000000..f1a914851cb --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.h @@ -0,0 +1,22 @@ +#include +#include +#include + +namespace DB +{ +namespace DM +{ +//========================================================================================== +// Functions for casting column data when disk data type mismatch with read data type. +//========================================================================================== + +// If `from_type` is the same as `to_column_define.type`, simply return `from_col`. +// If `from_type` is different from `to_column_define.type`, check if we can apply +// cast on read, if not, throw exception. +ColumnPtr convertColumnByColumnDefineIfNeed(const DataTypePtr & from_type, ColumnPtr && from_col, const ColumnDefine & to_column_define); + +// Create a column with `num_rows`, fill with column_define.default_value or column's default. +ColumnPtr createColumnWithDefaultValue(const ColumnDefine & column_define, size_t num_rows); + +} // namespace DM +} // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_convert_column.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_convert_column.cpp new file mode 100644 index 00000000000..24e9c4cfaaf --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_convert_column.cpp @@ -0,0 +1,246 @@ +#include +#include +#include +#include + +#include "dm_basic_include.h" + +namespace DB +{ +namespace DM +{ +namespace tests +{ + +TEST(ConvertColumnType_test, CastNumeric) +{ + { + const Strings to_types = {"UInt8", "UInt16", "UInt32", "UInt64"}; + + DataTypePtr disk_data_type = typeFromString("UInt8"); + MutableColumnPtr disk_col = disk_data_type->createColumn(); + disk_col->insert(Field(UInt64(15))); + disk_col->insert(Field(UInt64(255))); + + for (const String & to_type : to_types) + { + ColumnDefine read_define(0, "c", typeFromString(to_type)); + auto memory_column = convertColumnByColumnDefineIfNeed(disk_data_type, disk_col->getPtr(), read_define); + + UInt64 val1 = memory_column->getUInt(0); + ASSERT_EQ(val1, 15UL); + UInt64 val2 = memory_column->getUInt(1); + ASSERT_EQ(val2, 255UL); + } + } + + { + const Strings to_types = {"UInt16", "UInt32", "UInt64"}; + + DataTypePtr disk_data_type = typeFromString("UInt16"); + MutableColumnPtr disk_col = disk_data_type->createColumn(); + disk_col->insert(Field(UInt64(15))); + disk_col->insert(Field(UInt64(255))); + + for (const String & to_type : to_types) + { + ColumnDefine read_define(0, "c", typeFromString(to_type)); + auto memory_column = convertColumnByColumnDefineIfNeed(disk_data_type, disk_col->getPtr(), read_define); + + UInt64 val1 = memory_column->getUInt(0); + ASSERT_EQ(val1, 15UL); + UInt64 val2 = memory_column->getUInt(1); + ASSERT_EQ(val2, 255UL); + } + } + + { + const Strings to_types = {"UInt32", "UInt64"}; + + DataTypePtr disk_data_type = typeFromString("UInt32"); + MutableColumnPtr disk_col = disk_data_type->createColumn(); + disk_col->insert(Field(UInt64(15))); + disk_col->insert(Field(UInt64(255))); + + for (const String & to_type : to_types) + { + ColumnDefine read_define(0, "c", typeFromString(to_type)); + auto memory_column = convertColumnByColumnDefineIfNeed(disk_data_type, disk_col->getPtr(), read_define); + + UInt64 val1 = memory_column->getUInt(0); + ASSERT_EQ(val1, 15UL); + UInt64 val2 = memory_column->getUInt(1); + ASSERT_EQ(val2, 255UL); + } + } + + { + const Strings to_types = {"Int8", "Int16", "Int32", "Int64"}; + + DataTypePtr disk_data_type = typeFromString("Int8"); + MutableColumnPtr disk_col = disk_data_type->createColumn(); + disk_col->insert(Field(Int64(127))); + disk_col->insert(Field(Int64(-1))); + + for (const String & to_type : to_types) + { + ColumnDefine read_define(0, "c", typeFromString(to_type)); + auto memory_column = convertColumnByColumnDefineIfNeed(disk_data_type, disk_col->getPtr(), read_define); + + Int64 val1 = memory_column->getInt(0); + ASSERT_EQ(val1, 127L); + Int64 val2 = memory_column->getInt(1); + ASSERT_EQ(val2, -1L); + } + } + + { + const Strings to_types = {"Int16", "Int32", "Int64"}; + + DataTypePtr disk_data_type = typeFromString("Int16"); + MutableColumnPtr disk_col = disk_data_type->createColumn(); + disk_col->insert(Field(Int64(127))); + disk_col->insert(Field(Int64(-1))); + + for (const String & to_type : to_types) + { + ColumnDefine read_define(0, "c", typeFromString(to_type)); + auto memory_column = convertColumnByColumnDefineIfNeed(disk_data_type, disk_col->getPtr(), read_define); + + Int64 val1 = memory_column->getInt(0); + ASSERT_EQ(val1, 127L); + Int64 val2 = memory_column->getInt(1); + ASSERT_EQ(val2, -1L); + } + } + + { + const Strings to_types = {"Int32", "Int64"}; + + DataTypePtr disk_data_type = typeFromString("Int32"); + MutableColumnPtr disk_col = disk_data_type->createColumn(); + disk_col->insert(Field(Int64(127))); + disk_col->insert(Field(Int64(-1))); + + for (const String & to_type : to_types) + { + ColumnDefine read_define(0, "c", typeFromString(to_type)); + auto memory_column = convertColumnByColumnDefineIfNeed(disk_data_type, disk_col->getPtr(), read_define); + + Int64 val1 = memory_column->getInt(0); + ASSERT_EQ(val1, 127L); + Int64 val2 = memory_column->getInt(1); + ASSERT_EQ(val2, -1L); + } + } +} + +TEST(ConvertColumnType_test, CastNullableToNotNull) +{ + const Strings to_types = {"Int16", "Int32", "Int64"}; + + DataTypePtr disk_data_type = typeFromString("Nullable(Int8)"); + MutableColumnPtr disk_col = disk_data_type->createColumn(); + disk_col->insert(Field()); // a "NULL" value + disk_col->insert(Field(Int64(127))); + disk_col->insert(Field(Int64(-1))); + + for (const String & to_type : to_types) + { + ColumnDefine read_define(0, "c", typeFromString(to_type)); + auto memory_column = convertColumnByColumnDefineIfNeed(disk_data_type, disk_col->getPtr(), read_define); + + Int64 val1 = memory_column->getInt(0); + ASSERT_EQ(val1, 0); // "NULL" value is cast to 0 + Int64 val2 = memory_column->getInt(1); + ASSERT_EQ(val2, 127L); + Int64 val3 = memory_column->getUInt(2); + ASSERT_EQ(val3, -1L); + } +} + +TEST(ConvertColumnType_test, CastNullableToNotNullWithNonZeroDefaultValue) +{ + const Strings to_types = {"Int16", "Int32", "Int64"}; + + DataTypePtr disk_data_type = typeFromString("Nullable(Int8)"); + MutableColumnPtr disk_col = disk_data_type->createColumn(); + disk_col->insert(Field()); // a "NULL" value + disk_col->insert(Field(Int64(127))); + disk_col->insert(Field(Int64(-1))); + + for (const String & to_type : to_types) + { + ColumnDefine read_define(0, "c", typeFromString(to_type)); + read_define.default_value = Field(Int64(5)); + auto memory_column = convertColumnByColumnDefineIfNeed(disk_data_type, disk_col->getPtr(), read_define); + + Int64 val1 = memory_column->getInt(0); + ASSERT_EQ(val1, 5); // "NULL" value is cast to default value (5) + Int64 val2 = memory_column->getInt(1); + ASSERT_EQ(val2, 127L); + Int64 val3 = memory_column->getUInt(2); + ASSERT_EQ(val3, -1L); + } +} + +TEST(ConvertColumnType_test, CastNullableToNullable) +{ + const Strings to_types = {"Nullable(Int8)", "Nullable(Int16)", "Nullable(Int32)", "Nullable(Int64)"}; + + DataTypePtr disk_data_type = typeFromString("Nullable(Int8)"); + MutableColumnPtr disk_col = disk_data_type->createColumn(); + disk_col->insert(Field()); // a "NULL" value + disk_col->insert(Field(Int64(127))); + disk_col->insert(Field(Int64(-1))); + + for (const String & to_type : to_types) + { + ColumnDefine read_define(0, "c", typeFromString(to_type)); + auto memory_column = convertColumnByColumnDefineIfNeed(disk_data_type, disk_col->getPtr(), read_define); + + ASSERT_TRUE(memory_column->isNullAt(0)); + Field f = (*memory_column)[0]; + ASSERT_TRUE(f.isNull()); + + ASSERT_FALSE(memory_column->isNullAt(1)); + f = (*memory_column)[1]; + ASSERT_EQ(f.getType(), Field::Types::Int64); + ASSERT_EQ(f.get(), 127L); + + ASSERT_FALSE(memory_column->isNullAt(2)); + f = (*memory_column)[2]; + ASSERT_EQ(f.getType(), Field::Types::Int64); + ASSERT_EQ(f.get(), -1L); + } +} + +TEST(ConvertColumnType_test, CastNotNullToNullable) +{ + const Strings to_types = {"Nullable(Int8)", "Nullable(Int16)", "Nullable(Int32)", "Nullable(Int64)"}; + + DataTypePtr disk_data_type = typeFromString("Int8"); + MutableColumnPtr disk_col = disk_data_type->createColumn(); + disk_col->insert(Field(Int64(127))); + disk_col->insert(Field(Int64(-1))); + + for (const String & to_type : to_types) + { + ColumnDefine read_define(0, "c", typeFromString(to_type)); + auto memory_column = convertColumnByColumnDefineIfNeed(disk_data_type, disk_col->getPtr(), read_define); + + ASSERT_FALSE(memory_column->isNullAt(0)); + Field f = (*memory_column)[0]; + ASSERT_EQ(f.getType(), Field::Types::Int64); + ASSERT_EQ(f.get(), 127L); + + ASSERT_FALSE(memory_column->isNullAt(1)); + f = (*memory_column)[1]; + ASSERT_EQ(f.getType(), Field::Types::Int64); + ASSERT_EQ(f.get(), -1L); + } +} + +} // namespace tests +} // namespace DM +} // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp index af39b138943..022f5b32791 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp @@ -25,9 +25,12 @@ class DMFile_Test : public ::testing::Test void SetUp() override { dropFiles(); - storage_pool = std::make_unique("test.t1", path, DB::Settings()); - dm_file = DMFile::create(0, path); - db_context = std::make_unique(DMTestEnv::getContext(DB::Settings())); + + auto settings = DB::Settings(); + storage_pool = std::make_unique("test.t1", path, settings); + dm_file = DMFile::create(0, path); + db_context = std::make_unique(DMTestEnv::getContext(settings)); + table_columns_ = std::make_shared(); reload(); } @@ -41,17 +44,21 @@ class DMFile_Test : public ::testing::Test } } + // Update dm_context. void reload(const ColumnDefinesPtr & cols = DMTestEnv::getDefaultColumns()) { - dm_context = std::make_unique(*db_context, - path, - db_context->getExtraPaths(), - *storage_pool, - 0, - cols, - 0, - settings.not_compress_columns, - db_context->getSettingsRef()); + *table_columns_ = *cols; + + dm_context = std::make_unique( // + *db_context, + path, + db_context->getExtraPaths(), + *storage_pool, + /*hash_salt*/ 0, + table_columns_, + 0, + settings.not_compress_columns, + db_context->getSettingsRef()); } @@ -63,10 +70,10 @@ class DMFile_Test : public ::testing::Test String path; std::unique_ptr db_context; std::unique_ptr dm_context; - + /// all these var live as ref in dm_context std::unique_ptr storage_pool; - - DeltaMergeStore::Settings settings; + ColumnDefinesPtr table_columns_; + DeltaMergeStore::Settings settings; protected: DMFilePtr dm_file; @@ -76,7 +83,8 @@ class DMFile_Test : public ::testing::Test TEST_F(DMFile_Test, WriteRead) try { - auto cols = DMTestEnv::getDefaultColumns(); + auto cols = DMTestEnv::getDefaultColumns(); + const size_t num_rows_write = 128; { @@ -92,15 +100,16 @@ try { // Test read - auto stream = std::make_shared(dbContext(), // - std::numeric_limits::max(), - false, - dmContext().hash_salt, - dm_file, - *cols, - HandleRange::newAll(), - RSOperatorPtr{}, - IdSetPtr{}); + auto stream = std::make_shared( // + dbContext(), + std::numeric_limits::max(), + false, + dmContext().hash_salt, + dm_file, + *cols, + HandleRange::newAll(), + RSOperatorPtr{}, + IdSetPtr{}); size_t num_rows_read = 0; stream->readPrefix(); @@ -122,7 +131,7 @@ try stream->readSuffix(); ASSERT_EQ(num_rows_read, num_rows_write); } -} // namespace tests +} CATCH TEST_F(DMFile_Test, NumberTypes) @@ -130,26 +139,27 @@ try { auto cols = DMTestEnv::getDefaultColumns(); // Prepare columns - ColumnDefine i64_col(2, "i64", DataTypeFactory::instance().get("Int64")); - ColumnDefine f64_col(3, "f64", DataTypeFactory::instance().get("Float64")); + ColumnDefine i64_col(2, "i64", typeFromString("Int64")); + ColumnDefine f64_col(3, "f64", typeFromString("Float64")); cols->push_back(i64_col); cols->push_back(f64_col); reload(cols); + const size_t num_rows_write = 128; { // Prepare write - Block block = DMTestEnv::prepareSimpleWriteBlock(0, 128, false); + Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); auto col = i64_col.type->createColumn(); - for (int i = 0; i < 128; i++) + for (size_t i = 0; i < num_rows_write; i++) { col->insert(toField(Int64(i))); } ColumnWithTypeAndName i64(std::move(col), i64_col.type, i64_col.name, i64_col.id); col = f64_col.type->createColumn(); - for (int i = 0; i < 128; i++) + for (size_t i = 0; i < num_rows_write; i++) { col->insert(toField(Float64(0.125))); } @@ -166,15 +176,16 @@ try { // Test Read - auto stream = std::make_unique(dbContext(), // - std::numeric_limits::max(), - false, - dmContext().hash_salt, - dm_file, - *cols, - HandleRange::newAll(), - RSOperatorPtr{}, - IdSetPtr{}); + auto stream = std::make_unique( // + dbContext(), + std::numeric_limits::max(), + false, + dmContext().hash_salt, + dm_file, + *cols, + HandleRange::newAll(), + RSOperatorPtr{}, + IdSetPtr{}); size_t num_rows_read = 0; stream->readPrefix(); @@ -204,7 +215,7 @@ try num_rows_read += in.rows(); } stream->readSuffix(); - ASSERT_EQ(num_rows_read, 128UL); + ASSERT_EQ(num_rows_read, num_rows_write); } } CATCH @@ -213,17 +224,18 @@ TEST_F(DMFile_Test, StringType) { auto cols = DMTestEnv::getDefaultColumns(); // Prepare columns - ColumnDefine fixed_str_col(2, "str", DataTypeFactory::instance().get("FixedString(5)")); + ColumnDefine fixed_str_col(2, "str", typeFromString("FixedString(5)")); cols->push_back(fixed_str_col); reload(cols); + const size_t num_rows_write = 128; { // Prepare write - Block block = DMTestEnv::prepareSimpleWriteBlock(0, 128, false); + Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); auto col = fixed_str_col.type->createColumn(); - for (int i = 0; i < 128; i++) + for (size_t i = 0; i < num_rows_write; i++) { col->insert(toField(String("hello"))); } @@ -239,15 +251,16 @@ TEST_F(DMFile_Test, StringType) { // Test Read - auto stream = std::make_unique(dbContext(), // - std::numeric_limits::max(), - false, - dmContext().hash_salt, - dm_file, - *cols, - HandleRange::newAll(), - RSOperatorPtr{}, - IdSetPtr{}); + auto stream = std::make_unique( // + dbContext(), + std::numeric_limits::max(), + false, + dmContext().hash_salt, + dm_file, + *cols, + HandleRange::newAll(), + RSOperatorPtr{}, + IdSetPtr{}); size_t num_rows_read = 0; stream->readPrefix(); @@ -269,7 +282,7 @@ TEST_F(DMFile_Test, StringType) num_rows_read += in.rows(); } stream->readSuffix(); - ASSERT_EQ(num_rows_read, 128UL); + ASSERT_EQ(num_rows_read, num_rows_write); } } @@ -279,23 +292,24 @@ try auto cols = DMTestEnv::getDefaultColumns(); { // Prepare columns - ColumnDefine nullable_col(2, "i32_null", DataTypeFactory::instance().get("Nullable(Int32)")); + ColumnDefine nullable_col(2, "i32_null", typeFromString("Nullable(Int32)")); cols->emplace_back(nullable_col); } reload(cols); + const size_t num_rows_write = 128; { // Prepare write - Block block = DMTestEnv::prepareSimpleWriteBlock(0, 128, false); + Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); - ColumnWithTypeAndName nullable_col({}, DataTypeFactory::instance().get("Nullable(Int32)"), "i32_null", 2); + ColumnWithTypeAndName nullable_col({}, typeFromString("Nullable(Int32)"), "i32_null", 2); auto col = nullable_col.type->createColumn(); - for (int i = 0; i < 64; i++) + for (size_t i = 0; i < 64; i++) { - col->insert(toField(i)); + col->insert(toField(Int64(i))); } - for (int i = 64; i < 128; i++) + for (size_t i = 64; i < num_rows_write; i++) { col->insertDefault(); } @@ -310,15 +324,17 @@ try { // Test read - auto stream = std::make_shared(dbContext(), // - std::numeric_limits::max(), - false, - dmContext().hash_salt, - dm_file, - *cols, - HandleRange::newAll(), - RSOperatorPtr{}, - IdSetPtr{}); + auto stream = std::make_shared( // + dbContext(), + std::numeric_limits::max(), + false, + dmContext().hash_salt, + dm_file, + *cols, + HandleRange::newAll(), + RSOperatorPtr{}, + IdSetPtr{}); + size_t num_rows_read = 0; stream->readPrefix(); while (Block in = stream->read()) @@ -353,8 +369,373 @@ try } num_rows_read += in.rows(); } - ASSERT_EQ(num_rows_read, 128UL); + ASSERT_EQ(num_rows_read, num_rows_write); + stream->readSuffix(); + } +} +CATCH + +/// DDL test cases +class DMFile_DDL_Test : public DMFile_Test +{ +public: + /// Write some data into DMFile. + /// return rows write, schema + std::pair prepareSomeDataToDMFile(bool i8_is_nullable = false) + { + size_t num_rows_write = 128; + auto cols_before_ddl = DMTestEnv::getDefaultColumns(); + + ColumnDefine i8_col(2, "i8", i8_is_nullable ? typeFromString("Nullable(Int8)") : typeFromString("Int8")); + ColumnDefine f64_col(3, "f64", typeFromString("Float64")); + cols_before_ddl->push_back(i8_col); + cols_before_ddl->push_back(f64_col); + + reload(cols_before_ddl); + + { + // Prepare write + Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); + + auto col = i8_col.type->createColumn(); + for (size_t i = 0; i < num_rows_write; i++) + { + Field field; // Null by default + if (!i8_is_nullable || (i8_is_nullable && i < num_rows_write / 2)) + field = toField(Int64(i) * (-1 * (i % 2))); + col->insert(field); + } + ColumnWithTypeAndName i64(std::move(col), i8_col.type, i8_col.name, i8_col.id); + + col = f64_col.type->createColumn(); + for (size_t i = 0; i < num_rows_write; i++) + { + col->insert(toField(Float64(0.125))); + } + ColumnWithTypeAndName f64(std::move(col), f64_col.type, f64_col.name, f64_col.id); + + block.insert(i64); + block.insert(f64); + + auto stream = std::make_unique(dbContext(), dm_file, *cols_before_ddl); + stream->writePrefix(); + stream->write(block, 0); + stream->writeSuffix(); + + return {num_rows_write, *cols_before_ddl}; + } + } +}; + +TEST_F(DMFile_DDL_Test, AddColumn) +try +{ + // Prepare some data before ddl + const auto [num_rows_write, cols_before_ddl] = prepareSomeDataToDMFile(); + + // Mock that we add new column after ddl + auto cols_after_ddl = std::make_shared(); + *cols_after_ddl = cols_before_ddl; + // A new string column + ColumnDefine new_s_col(100, "s", typeFromString("String")); + cols_after_ddl->emplace_back(new_s_col); + // A new int64 column with default value 5 + ColumnDefine new_i_col_with_default(101, "i", typeFromString("Int64")); + new_i_col_with_default.default_value = Field(Int64(5)); + cols_after_ddl->emplace_back(new_i_col_with_default); + + { + // Test read with new columns after ddl + auto stream = std::make_unique( // + dbContext(), + std::numeric_limits::max(), + false, + dmContext().hash_salt, + dm_file, + *cols_after_ddl, + HandleRange::newAll(), + RSOperatorPtr{}, + IdSetPtr{}); + + size_t num_rows_read = 0; + stream->readPrefix(); + while (Block in = stream->read()) + { + ASSERT_TRUE(in.has("i8")); + ASSERT_TRUE(in.has("f64")); + ASSERT_TRUE(in.has(new_s_col.name)); + ASSERT_TRUE(in.has(new_i_col_with_default.name)); + for (auto itr : in) + { + auto c = itr.column; + if (itr.name == new_s_col.name) + { + EXPECT_EQ(itr.column_id, new_s_col.id); + EXPECT_TRUE(itr.type->equals(*new_s_col.type)); + for (size_t i = 0; i < c->size(); i++) + { + Field value = (*c)[i]; + ASSERT_EQ(value.getType(), Field::Types::String); + // Empty default value + ASSERT_EQ(value, new_s_col.type->getDefault()); + } + } + else if (itr.name == new_i_col_with_default.name) + { + EXPECT_EQ(itr.column_id, new_i_col_with_default.id); + EXPECT_TRUE(itr.type->equals(*new_i_col_with_default.type)); + for (size_t i = 0; i < c->size(); i++) + { + auto value = c->getInt(i); + ASSERT_EQ(value, 5); // Should fill with default value + } + } + // Check old columns before ddl + else if (itr.name == "i8") + { + EXPECT_EQ(itr.column_id, 2L); + EXPECT_TRUE(itr.type->equals(*typeFromString("Int8"))); + for (size_t i = 0; i < c->size(); i++) + { + EXPECT_EQ(c->getInt(i), Int64(i * (-1 * (i % 2)))); + } + } + else if (itr.name == "f64") + { + EXPECT_EQ(itr.column_id, 3L); + EXPECT_TRUE(itr.type->equals(*typeFromString("Float64"))); + for (size_t i = 0; i < c->size(); i++) + { + Field value = (*c)[i]; + Float64 v = value.get(); + EXPECT_EQ(v, 0.125); + } + } + } + num_rows_read += in.rows(); + } + stream->readSuffix(); + ASSERT_EQ(num_rows_read, num_rows_write); + } +} +CATCH + +TEST_F(DMFile_DDL_Test, UpcastColumnType) +try +{ + // Prepare some data before ddl + const auto [num_rows_write, cols_before_ddl] = prepareSomeDataToDMFile(); + + // Mock that we achange a column type from int8 -> int32, and its name to "i8_new" after ddl + auto cols_after_ddl = std::make_shared(); + *cols_after_ddl = cols_before_ddl; + const ColumnDefine old_col = cols_before_ddl[3]; + ASSERT_TRUE(old_col.type->equals(*typeFromString("Int8"))); + ColumnDefine new_col = old_col; + new_col.type = typeFromString("Int32"); + new_col.name = "i32_new"; + (*cols_after_ddl)[3] = new_col; + + { + // Test read with new columns after ddl + auto stream = std::make_unique( // + dbContext(), + std::numeric_limits::max(), + false, + dmContext().hash_salt, + dm_file, + *cols_after_ddl, + HandleRange::newAll(), + RSOperatorPtr{}, + IdSetPtr{}); + + size_t num_rows_read = 0; + stream->readPrefix(); + while (Block in = stream->read()) + { + ASSERT_TRUE(in.has(new_col.name)); + ASSERT_TRUE(!in.has("i8")); + ASSERT_TRUE(in.has("f64")); + for (auto itr : in) + { + auto c = itr.column; + if (itr.name == new_col.name) + { + EXPECT_EQ(itr.column_id, new_col.id); + EXPECT_TRUE(itr.type->equals(*new_col.type)); + for (size_t i = 0; i < c->size(); i++) + { + auto value = c->getInt(Int64(i)); + ASSERT_EQ(value, (Int64)(i * (-1 * (i % 2)))); + } + } + // Check old columns before ddl + else if (itr.name == "f64") + { + EXPECT_EQ(itr.column_id, 3L); + EXPECT_TRUE(itr.type->equals(*typeFromString("Float64"))); + for (size_t i = 0; i < c->size(); i++) + { + Field value = (*c)[i]; + Float64 v = value.get(); + EXPECT_EQ(v, 0.125); + } + } + } + num_rows_read += in.rows(); + } + stream->readSuffix(); + ASSERT_EQ(num_rows_read, num_rows_write); + } +} +CATCH + +TEST_F(DMFile_DDL_Test, NotNullToNull) +try +{ + // Prepare some data before ddl + const auto [num_rows_write, cols_before_ddl] = prepareSomeDataToDMFile(); + + // Mock that we achange a column type from int8 -> Nullable(int32), and its name to "i8_new" after ddl + auto cols_after_ddl = std::make_shared(); + *cols_after_ddl = cols_before_ddl; + const ColumnDefine old_col = cols_before_ddl[3]; + ASSERT_TRUE(old_col.type->equals(*typeFromString("Int8"))); + ColumnDefine new_col = old_col; + new_col.type = typeFromString("Nullable(Int32)"); + new_col.name = "i32_nullable"; + (*cols_after_ddl)[3] = new_col; + + { + // Test read with new columns after ddl + auto stream = std::make_unique( // + dbContext(), + std::numeric_limits::max(), + false, + dmContext().hash_salt, + dm_file, + *cols_after_ddl, + HandleRange::newAll(), + RSOperatorPtr{}, + IdSetPtr{}); + + size_t num_rows_read = 0; + stream->readPrefix(); + while (Block in = stream->read()) + { + ASSERT_TRUE(in.has(new_col.name)); + ASSERT_TRUE(!in.has("i8")); + ASSERT_TRUE(in.has("f64")); + for (auto itr : in) + { + auto c = itr.column; + if (itr.name == new_col.name) + { + EXPECT_EQ(itr.column_id, new_col.id); + EXPECT_TRUE(itr.type->equals(*new_col.type)); + for (size_t i = 0; i < c->size(); i++) + { + auto value = (*c)[i]; + ASSERT_FALSE(value.isNull()); + ASSERT_EQ(value, (Int64)(i * (-1 * (i % 2)))); + } + } + // Check old columns before ddl + else if (itr.name == "f64") + { + EXPECT_EQ(itr.column_id, 3L); + EXPECT_TRUE(itr.type->equals(*typeFromString("Float64"))); + for (size_t i = 0; i < c->size(); i++) + { + Field value = (*c)[i]; + Float64 v = value.get(); + EXPECT_EQ(v, 0.125); + } + } + } + num_rows_read += in.rows(); + } + stream->readSuffix(); + ASSERT_EQ(num_rows_read, num_rows_write); + } +} +CATCH + +TEST_F(DMFile_DDL_Test, NullToNotNull) +try +{ + // Prepare some data before ddl + const auto [num_rows_write, cols_before_ddl] = prepareSomeDataToDMFile(true); + + // Mock that we achange a column type from Nullable(int8) -> int32, and its name to "i32" after ddl + auto cols_after_ddl = std::make_shared(); + *cols_after_ddl = cols_before_ddl; + const ColumnDefine old_col = cols_before_ddl[3]; + ASSERT_TRUE(old_col.type->equals(*typeFromString("Nullable(Int8)"))); + ColumnDefine new_col = old_col; + new_col.type = typeFromString("Int32"); + new_col.name = "i32"; + (*cols_after_ddl)[3] = new_col; + + { + // Test read with new columns after ddl + auto stream = std::make_unique( // + dbContext(), + std::numeric_limits::max(), + false, + dmContext().hash_salt, + dm_file, + *cols_after_ddl, + HandleRange::newAll(), + RSOperatorPtr{}, + IdSetPtr{}); + + size_t num_rows_read = 0; + stream->readPrefix(); + while (Block in = stream->read()) + { + ASSERT_TRUE(in.has(new_col.name)); + ASSERT_TRUE(!in.has("i8")); + ASSERT_TRUE(in.has("f64")); + for (auto itr : in) + { + auto c = itr.column; + if (itr.name == new_col.name) + { + EXPECT_EQ(itr.column_id, new_col.id); + EXPECT_TRUE(itr.type->equals(*new_col.type)); + for (size_t i = 0; i < c->size(); i++) + { + auto value = (*c)[i]; + if (i < num_rows_write / 2) + { + ASSERT_FALSE(value.isNull()) << " at row: " << i; + ASSERT_EQ(value, (Int64)(i * (-1 * (i % 2)))) << " at row: " << i; + } + else + { + ASSERT_FALSE(value.isNull()) << " at row: " << i; + ASSERT_EQ(value, (Int64)0) << " at row: " << i; + } + } + } + // Check old columns before ddl + else if (itr.name == "f64") + { + EXPECT_EQ(itr.column_id, 3L); + EXPECT_TRUE(itr.type->equals(*typeFromString("Float64"))); + for (size_t i = 0; i < c->size(); i++) + { + Field value = (*c)[i]; + Float64 v = value.get(); + EXPECT_EQ(v, 0.125); + } + } + } + num_rows_read += in.rows(); + } stream->readSuffix(); + ASSERT_EQ(num_rows_read, num_rows_write); } } CATCH diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp index cb60b5b793e..726ed340f58 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp @@ -45,8 +45,10 @@ class Segment_test : public ::testing::Test void SetUp() override { - db_context = std::make_unique(DMTestEnv::getContext(DB::Settings())); + db_context = std::make_unique(DMTestEnv::getContext(DB::Settings())); + table_columns_ = std::make_shared(); dropDataInDisk(); + segment = reload(); ASSERT_EQ(segment->segmentId(), DELTA_MERGE_FIRST_SEGMENT_ID); } @@ -67,7 +69,7 @@ class Segment_test : public ::testing::Test // setColumns should update dm_context at the same time void setColumns(const ColumnDefinesPtr & columns) { - table_columns_ = columns; + *table_columns_ = *columns; dm_context_ = std::make_unique(*db_context, path, @@ -834,34 +836,51 @@ try } CATCH + +enum class SegmentWriteType +{ + ToDisk, + ToCache +}; +class Segment_DDL_test : public Segment_test, // + public testing::WithParamInterface> +{ +}; +String paramToString(const ::testing::TestParamInfo & info) +{ + const auto [write_type, flush_before_ddl] = info.param; + + String name = (write_type == SegmentWriteType::ToDisk) ? "ToDisk_" : "ToCache"; + name += (flush_before_ddl ? "_FlushCache" : "_NotFlushCache"); + return name; +} + /// Mock a col from i8 -> i32 -TEST_F(Segment_test, DISABLED_DDLAlterInt8ToInt32) +TEST_P(Segment_DDL_test, AlterInt8ToInt32) try { const String column_name_i8_to_i32 = "i8_to_i32"; const ColumnID column_id_i8_to_i32 = 4; - const ColumnDefine column_i8_before_ddl(column_id_i8_to_i32, column_name_i8_to_i32, DataTypeFactory::instance().get("Int8")); - const ColumnDefine column_i32_after_ddl(column_id_i8_to_i32, column_name_i8_to_i32, DataTypeFactory::instance().get("Int32")); + const ColumnDefine column_i8_before_ddl(column_id_i8_to_i32, column_name_i8_to_i32, typeFromString("Int8")); + const ColumnDefine column_i32_after_ddl(column_id_i8_to_i32, column_name_i8_to_i32, typeFromString("Int32")); + const auto [write_type, flush_before_ddl] = GetParam(); + + // Prepare some data before ddl + const size_t num_rows_write = 100; { + /// set columns before ddl auto columns_before_ddl = DMTestEnv::getDefaultColumns(); columns_before_ddl->emplace_back(column_i8_before_ddl); - // Not cache any rows DB::Settings db_settings; - db_settings.dm_segment_delta_cache_limit_rows = 0; - segment = reload(columns_before_ddl, std::move(db_settings)); - } - const size_t num_rows_write = 100; - { - // write to segment + /// write to segment Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); - // add int8_col and later read it as int32 // (mock ddl change int8 -> int32) const size_t num_rows = block.rows(); - ColumnWithTypeAndName int8_col(column_i8_before_ddl.type, column_i8_before_ddl.name); + ColumnWithTypeAndName int8_col(nullptr, column_i8_before_ddl.type, column_i8_before_ddl.name, column_id_i8_to_i32); { IColumn::MutablePtr m_col = int8_col.type->createColumn(); auto & column_data = typeid_cast &>(*m_col).getData(); @@ -873,29 +892,31 @@ try int8_col.column = std::move(m_col); } block.insert(int8_col); - - segment->write(dmContext(), std::move(block)); + switch (write_type) + { + case SegmentWriteType::ToDisk: + segment->write(dmContext(), std::move(block)); + break; + case SegmentWriteType::ToCache: + segment->writeToCache(dmContext(), block, 0, num_rows_write); + break; + } } + ColumnDefinesPtr columns_to_read = std::make_shared(); { - ColumnDefines columns_to_read{ - column_i32_after_ddl, - }; - - BlockInputStreamPtr in; - try + *columns_to_read = *DMTestEnv::getDefaultColumns(); + columns_to_read->emplace_back(column_i32_after_ddl); + if (flush_before_ddl) { - // read written data - in = segment->getInputStream(dmContext(), *tableColumns()); + segment->flushCache(dmContext()); } - catch (const Exception & e) - { - const auto text = e.displayText(); - std::cerr << "Code: " << e.code() << ". " << text << std::endl << std::endl; - std::cerr << "Stack trace:" << std::endl << e.getStackTrace().toString(); + setColumns(columns_to_read); + } - throw; - } + { + // read written data + BlockInputStreamPtr in = segment->getInputStream(dmContext(), *columns_to_read); // check that we can read correct values size_t num_rows_read = 0; @@ -903,25 +924,89 @@ try while (Block block = in->read()) { num_rows_read += block.rows(); + ASSERT_TRUE(block.has(column_name_i8_to_i32)); const ColumnWithTypeAndName & col = block.getByName(column_name_i8_to_i32); - ASSERT_TRUE(col.type->equals(*column_i32_after_ddl.type)) - << "col.type: " + col.type->getName() + " expect type: " + column_i32_after_ddl.type->getName(); + ASSERT_DATATYPE_EQ(col.type, column_i32_after_ddl.type); ASSERT_EQ(col.name, column_i32_after_ddl.name); ASSERT_EQ(col.column_id, column_i32_after_ddl.id); for (size_t i = 0; i < block.rows(); ++i) { auto value = col.column->getInt(i); const auto expected = static_cast(-1 * (i % 2 ? 1 : -1) * i); - ASSERT_EQ(value, expected); + ASSERT_EQ(value, expected) << "at row: " << i; } } in->readSuffix(); ASSERT_EQ(num_rows_read, num_rows_write); } + + + /// Write some data after ddl, replacing som origin rows + { + /// write to segment, replacing some origin rows + Block block = DMTestEnv::prepareSimpleWriteBlock(num_rows_write / 2, num_rows_write * 2, false, /* tso= */ 3); + + const size_t num_rows = block.rows(); + ColumnWithTypeAndName int32_col(nullptr, column_i32_after_ddl.type, column_i32_after_ddl.name, column_id_i8_to_i32); + { + IColumn::MutablePtr m_col = int32_col.type->createColumn(); + auto & column_data = typeid_cast &>(*m_col).getData(); + column_data.resize(num_rows); + for (size_t i = 0; i < num_rows; ++i) + { + column_data[i] = static_cast(-1 * (i % 2 ? 1 : -1) * i); + } + int32_col.column = std::move(m_col); + } + block.insert(int32_col); + switch (write_type) + { + case SegmentWriteType::ToDisk: + segment->write(dmContext(), std::move(block)); + break; + case SegmentWriteType::ToCache: + segment->writeToCache(dmContext(), block, 0, num_rows); + break; + } + } + + { + // read written data + BlockInputStreamPtr in = segment->getInputStream(dmContext(), *columns_to_read); + + // check that we can read correct values + size_t num_rows_read = 0; + in->readPrefix(); + while (Block block = in->read()) + { + num_rows_read += block.rows(); + ASSERT_TRUE(block.has(column_name_i8_to_i32)); + const ColumnWithTypeAndName & col = block.getByName(column_name_i8_to_i32); + ASSERT_DATATYPE_EQ(col.type, column_i32_after_ddl.type); + ASSERT_EQ(col.name, column_i32_after_ddl.name); + ASSERT_EQ(col.column_id, column_i32_after_ddl.id); + for (size_t i = 0; i < block.rows(); ++i) + { + auto value = col.column->getInt(i); + auto expected = 0; + if (i < num_rows_write / 2) + expected = static_cast(-1 * (i % 2 ? 1 : -1) * i); + else + { + auto r = i - num_rows_write / 2; + expected = static_cast(-1 * (r % 2 ? 1 : -1) * r); + } + // std::cerr << " row: " << i << " "<< value << std::endl; + ASSERT_EQ(value, expected) << "at row: " << i; + } + } + in->readSuffix(); + ASSERT_EQ(num_rows_read, (size_t)(num_rows_write * 2)); + } } CATCH -TEST_F(Segment_test, DISABLED_DDLAddColumnWithDefaultValue) +TEST_P(Segment_DDL_test, AddColumn) try { const String new_column_name = "i8"; @@ -930,12 +1015,12 @@ try const Int8 new_column_default_value_int = 16; new_column_define.default_value = toField(new_column_default_value_int); + const auto [write_type, flush_before_ddl] = GetParam(); + { auto columns_before_ddl = DMTestEnv::getDefaultColumns(); // Not cache any rows DB::Settings db_settings; - db_settings.dm_segment_delta_cache_limit_rows = 0; - segment = reload(columns_before_ddl, std::move(db_settings)); } @@ -943,23 +1028,32 @@ try { // write to segment Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); - segment->write(dmContext(), std::move(block)); + switch (write_type) + { + case SegmentWriteType::ToDisk: + segment->write(dmContext(), std::move(block)); + break; + case SegmentWriteType::ToCache: + segment->writeToCache(dmContext(), block, 0, num_rows_write); + break; + } } + auto columns_after_ddl = DMTestEnv::getDefaultColumns(); { // DDL add new column with default value - auto columns_after_ddl = DMTestEnv::getDefaultColumns(); columns_after_ddl->emplace_back(new_column_define); + if (flush_before_ddl) + { + // If write to cache, before apply ddl changes (change column data type), segment->flushCache must be called. + segment->flushCache(dmContext()); + } setColumns(columns_after_ddl); } { - ColumnDefines columns_to_read{ - new_column_define, - }; - // read written data - auto in = segment->getInputStream(dmContext(), *tableColumns()); + auto in = segment->getInputStream(dmContext(), *columns_after_ddl); // check that we can read correct values size_t num_rows_read = 0; @@ -980,9 +1074,80 @@ try in->readSuffix(); ASSERT_EQ(num_rows_read, num_rows_write); } + + + /// Write some data after ddl, replacing som origin rows + { + /// write to segment, replacing some origin rows + Block block = DMTestEnv::prepareSimpleWriteBlock(num_rows_write / 2, num_rows_write * 2, false, /* tso= */ 3); + + const size_t num_rows = block.rows(); + ColumnWithTypeAndName int8_col(nullptr, new_column_define.type, new_column_define.name, new_column_id); + { + IColumn::MutablePtr m_col = int8_col.type->createColumn(); + auto & column_data = typeid_cast &>(*m_col).getData(); + column_data.resize(num_rows); + for (size_t i = 0; i < num_rows; ++i) + { + column_data[i] = static_cast(-1 * (i % 2 ? 1 : -1) * i); + } + int8_col.column = std::move(m_col); + } + block.insert(int8_col); + switch (write_type) + { + case SegmentWriteType::ToDisk: + segment->write(dmContext(), std::move(block)); + break; + case SegmentWriteType::ToCache: + segment->writeToCache(dmContext(), block, 0, num_rows); + break; + } + } + + { + // read written data + BlockInputStreamPtr in = segment->getInputStream(dmContext(), *columns_after_ddl); + + // check that we can read correct values + size_t num_rows_read = 0; + in->readPrefix(); + while (Block block = in->read()) + { + num_rows_read += block.rows(); + ASSERT_TRUE(block.has(new_column_name)); + const ColumnWithTypeAndName & col = block.getByName(new_column_name); + ASSERT_DATATYPE_EQ(col.type, new_column_define.type); + ASSERT_EQ(col.name, new_column_define.name); + ASSERT_EQ(col.column_id, new_column_define.id); + for (size_t i = 0; i < block.rows(); ++i) + { + int8_t value = col.column->getInt(i); + int8_t expected = 0; + if (i < num_rows_write / 2) + expected = new_column_default_value_int; + else + { + auto r = i - num_rows_write / 2; + expected = static_cast(-1 * (r % 2 ? 1 : -1) * r); + } + // std::cerr << " row: " << i << " "<< value << std::endl; + ASSERT_EQ(value, expected) << "at row: " << i; + } + } + in->readSuffix(); + ASSERT_EQ(num_rows_read, (size_t)(num_rows_write * 2)); + } } CATCH +INSTANTIATE_TEST_CASE_P(SegmentWriteType, + Segment_DDL_test, + ::testing::Combine( // + ::testing::Values(SegmentWriteType::ToDisk, SegmentWriteType::ToCache), + ::testing::Bool()), + paramToString); + } // namespace tests } // namespace DM } // namespace DB diff --git a/dbms/src/Storages/IManageableStorage.h b/dbms/src/Storages/IManageableStorage.h index 7142d4253d5..8d094096c3a 100644 --- a/dbms/src/Storages/IManageableStorage.h +++ b/dbms/src/Storages/IManageableStorage.h @@ -60,6 +60,10 @@ class IManageableStorage : public IStorage const AlterCommands & commands, const TiDB::TableInfo & table_info, const String & database_name, const Context & context) = 0; + + /// Remove this storage from TMTContext. Should be called after its metadata and data have been removed from disk. + virtual void removeFromTMTContext() = 0; + PKType getPKType() const { static const DataTypeInt64 & dataTypeInt64 = {}; diff --git a/dbms/src/Storages/MutableSupport.h b/dbms/src/Storages/MutableSupport.h index 49841cb2e83..cd4a1fb65fc 100644 --- a/dbms/src/Storages/MutableSupport.h +++ b/dbms/src/Storages/MutableSupport.h @@ -26,7 +26,7 @@ class MutableSupport : public ext::singleton const OrderedNameSet & hiddenColumns(const String& table_type_name) { - if (mmt_storage_name == table_type_name || txn_storage_name == table_type_name) + if (mmt_storage_name == table_type_name || txn_storage_name == table_type_name || delta_tree_storage_name == table_type_name) return mutable_hidden; return empty; } diff --git a/dbms/src/Storages/StorageDebugging.cpp b/dbms/src/Storages/StorageDebugging.cpp index cf3965a5427..1f823016033 100644 --- a/dbms/src/Storages/StorageDebugging.cpp +++ b/dbms/src/Storages/StorageDebugging.cpp @@ -198,7 +198,10 @@ void StorageDebugging::shutdown() return; shutdown_called = true; +} +void StorageDebugging::removeFromTMTContext() +{ // remove this table from TMTContext TMTContext & tmt_context = global_context.getTMTContext(); tmt_context.getStorages().remove(tidb_table_info.id); diff --git a/dbms/src/Storages/StorageDebugging.h b/dbms/src/Storages/StorageDebugging.h index 937a7e07d72..13f0958fb6d 100644 --- a/dbms/src/Storages/StorageDebugging.h +++ b/dbms/src/Storages/StorageDebugging.h @@ -70,6 +70,7 @@ class StorageDebugging : public ext::shared_ptr_helper, public void shutdown() override; + void removeFromTMTContext() override; private: String database_name; diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index 926b336ec95..dd8f26fc7bf 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -260,6 +260,7 @@ class DMBlockOutputStream : public IBlockOutputStream Block getHeader() const override { return header; } void write(const Block & block) override + try { if (db_settings.dm_insert_max_rows == 0) { @@ -286,6 +287,11 @@ class DMBlockOutputStream : public IBlockOutputStream } } } + catch (DB::Exception & e) + { + e.addMessage("(while writing to table `" + store->getDatabaseName() + "`.`" + store->getTableName() + "`)"); + throw; + } private: DeltaMergeStorePtr store; @@ -1110,11 +1116,15 @@ void StorageDeltaMerge::startup() void StorageDeltaMerge::shutdown() { - if (shutdown_called) + bool v = false; + if (!shutdown_called.compare_exchange_strong(v, true)) return; - shutdown_called = true; + store->shutdown(); +} +void StorageDeltaMerge::removeFromTMTContext() +{ // remove this table from TMTContext TMTContext & tmt_context = global_context.getTMTContext(); tmt_context.getStorages().remove(tidb_table_info.id); diff --git a/dbms/src/Storages/StorageDeltaMerge.h b/dbms/src/Storages/StorageDeltaMerge.h index 92b8d8184e6..ac39aaf7acc 100644 --- a/dbms/src/Storages/StorageDeltaMerge.h +++ b/dbms/src/Storages/StorageDeltaMerge.h @@ -73,6 +73,8 @@ class StorageDeltaMerge : public ext::shared_ptr_helper, publ void shutdown() override; + void removeFromTMTContext() override; + SortDescription getPrimarySortDescription() const override; const OrderedNameSet & getHiddenColumnsImpl() const override { return hidden_columns; } diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 92e98d577ac..dedc2270ba5 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -33,7 +33,7 @@ class StorageMergeTree : public ext::shared_ptr_helper, public public: void startup() override; void shutdown() override; - void removeFromTMTContext(); + void removeFromTMTContext() override; ~StorageMergeTree() override; std::string getName() const override { return data.merging_params.getModeName() + "MergeTree"; } diff --git a/dbms/src/Storages/Transaction/RegionTable.cpp b/dbms/src/Storages/Transaction/RegionTable.cpp index ec628aef59e..d5ab53cbb65 100644 --- a/dbms/src/Storages/Transaction/RegionTable.cpp +++ b/dbms/src/Storages/Transaction/RegionTable.cpp @@ -244,7 +244,10 @@ void RegionTable::removeRegion(const RegionID region_id) TableID table_id = it->second; auto & table = tables.find(table_id)->second; + do { + /// Some region of this table is removed, if it is a DeltaTree, write deleteRange. + /// Now we assume that StorageDeltaMerge::deleteRange do not block for long time and do it in sync mode. /// If this block for long time, consider to do this in background threads. TMTContext & tmt = context->getTMTContext(); @@ -253,22 +256,28 @@ void RegionTable::removeRegion(const RegionID region_id) { // acquire lock so that no other threads can change storage's structure auto storage_lock = storage->lockStructure(true, __PRETTY_FUNCTION__); + // Check if it has been dropped by other thread + if (storage->is_dropped) + break; // continue to remove region on `regions`, `table.regions` + auto dm_storage = std::dynamic_pointer_cast(storage); auto region_it = table.regions.find(region_id); - if (region_it == table.regions.end()) - return; - HandleRange handle_range = region_it->second.range_in_table; + if (dm_storage == nullptr || region_it == table.regions.end()) + break; + HandleRange handle_range = region_it->second.range_in_table; auto dm_handle_range = toDMHandleRange(handle_range); dm_storage->deleteRange(dm_handle_range, context->getSettingsRef()); - dm_storage->flushCache(*context, dm_handle_range); + dm_storage->flushCache(*context, dm_handle_range); // flush to disk } - } + } while (0); regions.erase(it); table.regions.erase(region_id); if (table.regions.empty()) { + /// All regions of this table is removed, the storage maybe drop or pd + /// move it to another node, we can optimize outdated data. table_to_optimize.insert(table_id); tables.erase(table_id); } diff --git a/dbms/src/Storages/Transaction/TiDB.cpp b/dbms/src/Storages/Transaction/TiDB.cpp index c40193f0546..fd4b109d9b9 100644 --- a/dbms/src/Storages/Transaction/TiDB.cpp +++ b/dbms/src/Storages/Transaction/TiDB.cpp @@ -150,14 +150,12 @@ UInt64 ColumnInfo::getSetValue(const String & set_str) const if (marked.empty()) return value; - throw DB::Exception( - std::string(__PRETTY_FUNCTION__) + ": can't parse set type value."); + throw DB::Exception(std::string(__PRETTY_FUNCTION__) + ": can't parse set type value."); } Int64 ColumnInfo::getTimeValue(const String & time_str) const { - const static long fractional_seconds_multiplier[] = {1000000000, 100000000, 10000000, 1000000, - 100000, 10000, 1000, 100, 10, 1}; + const static long fractional_seconds_multiplier[] = {1000000000, 100000000, 10000000, 1000000, 100000, 10000, 1000, 100, 10, 1}; bool negative = time_str[0] == '-'; Poco::StringTokenizer second_and_fsp(time_str, "."); Poco::StringTokenizer string_tokens(second_and_fsp[0], ":"); @@ -171,8 +169,7 @@ Int64 ColumnInfo::getTimeValue(const String & time_str) const fs_length = second_and_fsp[1].length(); fs_value = std::stol(second_and_fsp[1]); } - ret = ret * fractional_seconds_multiplier[0] - + fs_value * fractional_seconds_multiplier[fs_length]; + ret = ret * fractional_seconds_multiplier[0] + fs_value * fractional_seconds_multiplier[fs_length]; return negative ? -ret : ret; } @@ -186,7 +183,8 @@ Int64 ColumnInfo::getYearValue(Int64 val) const return val; } -Poco::JSON::Object::Ptr ColumnInfo::getJSONObject() const try +Poco::JSON::Object::Ptr ColumnInfo::getJSONObject() const +try { Poco::JSON::Object::Ptr json = new Poco::JSON::Object(); @@ -229,7 +227,8 @@ catch (const Poco::Exception & e) std::string(__PRETTY_FUNCTION__) + ": Serialize TiDB schema JSON failed (ColumnInfo): " + e.displayText(), DB::Exception(e)); } -void ColumnInfo::deserialize(Poco::JSON::Object::Ptr json) try +void ColumnInfo::deserialize(Poco::JSON::Object::Ptr json) +try { id = json->getValue("id"); name = json->getObject("name")->getValue("L"); @@ -263,7 +262,8 @@ catch (const Poco::Exception & e) PartitionDefinition::PartitionDefinition(Poco::JSON::Object::Ptr json) { deserialize(json); } -Poco::JSON::Object::Ptr PartitionDefinition::getJSONObject() const try +Poco::JSON::Object::Ptr PartitionDefinition::getJSONObject() const +try { Poco::JSON::Object::Ptr json = new Poco::JSON::Object(); json->set("id", id); @@ -284,7 +284,8 @@ catch (const Poco::Exception & e) std::string(__PRETTY_FUNCTION__) + ": Serialize TiDB schema JSON failed (PartitionDef): " + e.displayText(), DB::Exception(e)); } -void PartitionDefinition::deserialize(Poco::JSON::Object::Ptr json) try +void PartitionDefinition::deserialize(Poco::JSON::Object::Ptr json) +try { id = json->getValue("id"); name = json->getObject("name")->getValue("L"); @@ -299,7 +300,8 @@ catch (const Poco::Exception & e) PartitionInfo::PartitionInfo(Poco::JSON::Object::Ptr json) { deserialize(json); } -Poco::JSON::Object::Ptr PartitionInfo::getJSONObject() const try +Poco::JSON::Object::Ptr PartitionInfo::getJSONObject() const +try { Poco::JSON::Object::Ptr json = new Poco::JSON::Object(); @@ -328,7 +330,8 @@ catch (const Poco::Exception & e) std::string(__PRETTY_FUNCTION__) + ": Serialize TiDB schema JSON failed (PartitionInfo): " + e.displayText(), DB::Exception(e)); } -void PartitionInfo::deserialize(Poco::JSON::Object::Ptr json) try +void PartitionInfo::deserialize(Poco::JSON::Object::Ptr json) +try { type = static_cast(json->getValue("type")); expr = json->getValue("expr"); @@ -352,7 +355,8 @@ catch (const Poco::Exception & e) TableInfo::TableInfo(const String & table_info_json) { deserialize(table_info_json); } -String TableInfo::serialize() const try +String TableInfo::serialize() const +try { std::stringstream buf; @@ -401,7 +405,8 @@ catch (const Poco::Exception & e) std::string(__PRETTY_FUNCTION__) + ": Serialize TiDB schema JSON failed (TableInfo): " + e.displayText(), DB::Exception(e)); } -void DBInfo::deserialize(const String & json_str) try +void DBInfo::deserialize(const String & json_str) +try { Poco::JSON::Parser parser; Poco::Dynamic::Var result = parser.parse(json_str); @@ -419,7 +424,8 @@ catch (const Poco::Exception & e) DB::Exception(e)); } -void TableInfo::deserialize(const String & json_str) try +void TableInfo::deserialize(const String & json_str) +try { if (json_str.empty()) { @@ -542,6 +548,21 @@ String TableInfo::getColumnName(const ColumnID id) const DB::ErrorCodes::LOGICAL_ERROR); } +const ColumnInfo & TableInfo::getColumnInfo(const ColumnID id) const +{ + for (const auto & col : columns) + { + if (id == col.id) + { + return col; + } + } + + throw DB::Exception( + std::string(__PRETTY_FUNCTION__) + ": Invalidate column id " + std::to_string(id) + " for table " + db_name + "." + name, + DB::ErrorCodes::LOGICAL_ERROR); +} + std::optional> TableInfo::getPKHandleColumn() const { if (!pk_is_handle) diff --git a/dbms/src/Storages/Transaction/TiDB.h b/dbms/src/Storages/Transaction/TiDB.h index 657262a739e..05bf979bca1 100644 --- a/dbms/src/Storages/Transaction/TiDB.h +++ b/dbms/src/Storages/Transaction/TiDB.h @@ -1,13 +1,13 @@ #pragma once -#include - #include #include #include #include -#include #include +#include + +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" @@ -288,6 +288,8 @@ struct TableInfo ColumnID getColumnID(const String & name) const; String getColumnName(const ColumnID id) const; + const ColumnInfo & getColumnInfo(const ColumnID id) const; + std::optional> getPKHandleColumn() const; TableInfo producePartitionTableInfo(TableID table_or_partition_id) const; diff --git a/dbms/src/test_utils/TiflashTestBasic.h b/dbms/src/test_utils/TiflashTestBasic.h index db654fa9cc2..10eed176958 100644 --- a/dbms/src/test_utils/TiflashTestBasic.h +++ b/dbms/src/test_utils/TiflashTestBasic.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -24,6 +25,21 @@ namespace tests throw; \ } +/// helper functions for comparing DataType +inline ::testing::AssertionResult DataTypeCompare( // + const char * lhs_expr, + const char * rhs_expr, + const DataTypePtr & lhs, + const DataTypePtr & rhs) +{ + if (lhs->equals(*rhs)) + return ::testing::AssertionSuccess(); + else + return ::testing::internal::EqFailure(lhs_expr, rhs_expr, lhs->getName(), rhs->getName(), false); +} +#define ASSERT_DATATYPE_EQ(val1, val2) ASSERT_PRED_FORMAT2(::DB::tests::DataTypeCompare, val1, val2) +#define EXPECT_DATATYPE_EQ(val1, val2) EXPECT_PRED_FORMAT2(::DB::tests::DataTypeCompare, val1, val2) + // A simple helper for getting DataType from type name inline DataTypePtr typeFromString(const String & str) { diff --git a/tests/_env.sh b/tests/_env.sh index bd70de7c84e..c8f2efb2ef3 100644 --- a/tests/_env.sh +++ b/tests/_env.sh @@ -10,9 +10,6 @@ fi export storage_bin="$build_dir/dbms/src/Server/tiflash" -# Serve config for launching -export storage_server_config="../../running/config/config.xml" - # Server address for connecting export storage_server="127.0.0.1" diff --git a/tests/delta-merge-test/ddl/alter.test b/tests/delta-merge-test/ddl/alter.test index 7c6f49a9a29..b5ef46d0718 100644 --- a/tests/delta-merge-test/ddl/alter.test +++ b/tests/delta-merge-test/ddl/alter.test @@ -1,6 +1,3 @@ -#TODO: enable ddl tests for DeltaMerge -#RETURN - >> drop table if exists dm_test ## Prepare table @@ -19,22 +16,14 @@ >> alter table dm_test drop column a Received exception from server (version {#WORD}): Code: 36. DB::Exception: Received from {#WORD} DB::Exception: Storage engine DeltaMerge doesn't support drop primary key / hidden column: a. -#>> show create table dm_test -#β”Œβ”€statement───────────────────────────────────────────────────────────────┐ -#β”‚ CREATE TABLE default.dm_test ( a Int8, b Int32) ENGINE = DeltaMerge(a) β”‚ -#β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ >> alter table dm_test drop column b -#>> show create table dm_test -#β”Œβ”€statement─────────────────────────────────────────────────────┐ -#β”‚ CREATE TABLE default.dm_test ( a Int8) ENGINE = DeltaMerge(a) β”‚ -#β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ >> select * from dm_test β”Œβ”€a─┐ β”‚ 1 β”‚ β””β”€β”€β”€β”˜ -# add a column which name is the same as before +# add a column which name is the same as before, but column b value should be default value(0) >> alter table dm_test add column b Int32 >> select * from dm_test β”Œβ”€a─┬─b─┐ diff --git a/tests/delta-merge-test/ddl/alter_default_value.test b/tests/delta-merge-test/ddl/alter_default_value.test deleted file mode 100644 index d3df83dc84b..00000000000 --- a/tests/delta-merge-test/ddl/alter_default_value.test +++ /dev/null @@ -1,50 +0,0 @@ -#TODO: enable ddl tests for DeltaMerge -#RETURN - -# modify column default value - ->> drop table if exists dm_test -## Prepare table ->> create table dm_test ( - pk Int8 - ) engine = DeltaMerge(pk) - ->> insert into table dm_test values(1) - -## create new column without default value. the row which pk==1, will filled with 'zero' value ->> alter table dm_test add column i0 Int32 ->> alter table dm_test add column f32_0 Float32 ->> alter table dm_test add column f64_0 Float64 ->> alter table dm_test add column dec0 Decimal(10,4) ->> alter table dm_test add column s0 String ->> alter table dm_test add column fs0 FixedString(4) ->> alter table dm_test add column dt0 DateTime - -## create new column with default value. the row which pk==1, will filled with those default value -## See FLASH-453 -#>> alter table dm_test add column s2 String default 'non-empty' - -## These alter command will throw exception now -## See FLASH-453 ->> alter table dm_test add column i1 Int32 default 999 ->> alter table dm_test add column f32_1 Float32 default 1.234 ->> alter table dm_test add column f64_1 Float64 default 1.234 ->> alter table dm_test add column dec1 Decimal(10,4) default 3.1415 ->> alter table dm_test add column fs1 FixedString(4) default 'aaa' ->> alter table dm_test add column dt1 DateTime default '1999-09-09 12:34:56' - ->> select * from dm_test where pk = 1 -β”Œβ”€pk─┬─i0─┬─f32_0─┬─f64_0─┬─dec0───┬─s0─┬─fs0──────┬─────────────────dt0─┬──i1─┬─f32_1─┬─f64_1─┬─dec1───┬─fs1───┬─────────────────dt1─┐ -β”‚ 1 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0.0000 β”‚ β”‚ \0\0\0\0 β”‚ 0000-00-00 00:00:00 β”‚ 999 β”‚ 1.234 β”‚ 1.234 β”‚ 3.1415 β”‚ aaa\0 β”‚ 1999-09-09 12:34:56 β”‚ -β””β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - - -## insert a row, missing fields will be filled with default value ->> insert into table dm_test(pk) values(3) ->> select * from dm_test where pk = 3 -β”Œβ”€pk─┬─i0─┬─f32_0─┬─f64_0─┬─dec0───┬─s0─┬─fs0──────┬─────────────────dt0─┬──i1─┬─f32_1─┬─f64_1─┬─dec1───┬─fs1───┬─────────────────dt1─┐ -β”‚ 3 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0.0000 β”‚ β”‚ \0\0\0\0 β”‚ 0000-00-00 00:00:00 β”‚ 999 β”‚ 1.234 β”‚ 1.234 β”‚ 3.1415 β”‚ aaa\0 β”‚ 1999-09-09 12:34:56 β”‚ -β””β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - -## clean up ->> drop table if exists dm_test diff --git a/tests/delta-merge-test/ddl/alter_joint_primary_key.test b/tests/delta-merge-test/ddl/alter_joint_primary_key.test index 73c8ee4800d..c5857235612 100644 --- a/tests/delta-merge-test/ddl/alter_joint_primary_key.test +++ b/tests/delta-merge-test/ddl/alter_joint_primary_key.test @@ -1,5 +1,3 @@ -#TODO: enable ddl tests for DeltaMerge -#RETURN >> drop table if exists dm_test @@ -26,17 +24,23 @@ Code: 36. DB::Exception: Received from {#WORD} DB::Exception: Storage engine Del β”Œβ”€a─┬─b─┬─c─────────────┬─d─────────────────────────────┐ β”‚ 1 β”‚ 2 β”‚ hello TiFlash β”‚ hello world\0\0\0\0\0\0\0\0\0 β”‚ β””β”€β”€β”€β”΄β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ->> show create table dm_test -β”Œβ”€statement─────────────────────────────────────────────────────────────────────────────────────────────────────┐ -β”‚ CREATE TABLE default.dm_test ( a Int32, b Int32, c String, d FixedString(20)) ENGINE = DeltaMerge((a, b)) β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - +>> desc dm_test +β”Œβ”€name──────────────┬─type────────────┬─default_type─┬─default_expression─┐ +β”‚ a β”‚ Int32 β”‚ β”‚ β”‚ +β”‚ b β”‚ Int32 β”‚ β”‚ β”‚ +β”‚ c β”‚ String β”‚ β”‚ β”‚ +β”‚ d β”‚ FixedString(20) β”‚ β”‚ β”‚ +β”‚ _tidb_rowid β”‚ Int64 β”‚ β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ >> alter table dm_test drop column c -#>> show create table dm_test -#β”Œβ”€statement──────────────────────────────────────────────────────────────────────────────────────────┐ -#β”‚ CREATE TABLE default.dm_test ( a Int32, b Int32, d FixedString(200)) ENGINE = DeltaMerge((a, b)) β”‚ -#β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +>> desc dm_test +β”Œβ”€name──────────────┬─type────────────┬─default_type─┬─default_expression─┐ +β”‚ a β”‚ Int32 β”‚ β”‚ β”‚ +β”‚ b β”‚ Int32 β”‚ β”‚ β”‚ +β”‚ d β”‚ FixedString(20) β”‚ β”‚ β”‚ +β”‚ _tidb_rowid β”‚ Int64 β”‚ β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ >> select * from dm_test β”Œβ”€a─┬─b─┬─d─────────────────────────────┐ β”‚ 1 β”‚ 2 β”‚ hello world\0\0\0\0\0\0\0\0\0 β”‚ @@ -48,10 +52,12 @@ Code: 36. DB::Exception: Received from {#WORD} DB::Exception: Storage engine Del β”Œβ”€a─┬─b─┐ β”‚ 1 β”‚ 2 β”‚ β””β”€β”€β”€β”΄β”€β”€β”€β”˜ -#>> show create table dm_test -#β”Œβ”€statement─────────────────────────────────────────────────────────────────────┐ -#β”‚ CREATE TABLE default.dm_test ( a Int32, b Int32) ENGINE = DeltaMerge((a, b)) β”‚ -#β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +>> desc dm_test +β”Œβ”€name──────────────┬─type───┬─default_type─┬─default_expression─┐ +β”‚ a β”‚ Int32 β”‚ β”‚ β”‚ +β”‚ b β”‚ Int32 β”‚ β”‚ β”‚ +β”‚ _tidb_rowid β”‚ Int64 β”‚ β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ## Clean up >> drop table if exists dm_test diff --git a/tests/delta-merge-test/ddl/alter_nullable.test b/tests/delta-merge-test/ddl/alter_nullable.test index 3a4cc8108be..c75e06bfd02 100644 --- a/tests/delta-merge-test/ddl/alter_nullable.test +++ b/tests/delta-merge-test/ddl/alter_nullable.test @@ -1,5 +1,3 @@ -#TODO: enable ddl tests for DeltaMerge -#RETURN >> drop table if exists dm_test diff --git a/tests/delta-merge-test/raft/bugs/FLASH-484.test b/tests/delta-merge-test/raft/bugs/FLASH-484.test index 9e2e306f7ad..abecde66cdc 100644 --- a/tests/delta-merge-test/raft/bugs/FLASH-484.test +++ b/tests/delta-merge-test/raft/bugs/FLASH-484.test @@ -19,7 +19,6 @@ ## insert data and get exception, then rowid==51 is not flush to Storage => DBGInvoke __raft_insert_row(default, test, 4, 51, 'test51') -=> DBGInvoke __try_flush_region(4) Received exception from server (version {#WORD}): Code: 48. DB::Exception: Received from {#WORD} DB::Exception: Reject first write for test, engine: BuggyMemory-RejectFirstWrite. => select count(*) from default.test @@ -31,7 +30,6 @@ Code: 48. DB::Exception: Received from {#WORD} DB::Exception: Reject first write ## insert more data, and this time we get all 3 rows(including rowid==51) flush to Storage => DBGInvoke __raft_insert_row(default, test, 4, 52, 'test52') => DBGInvoke __raft_insert_row(default, test, 4, 19, 'test19') -=> DBGInvoke __try_flush_region(4) => select * from default.test order by _tidb_rowid β”Œβ”€col_1──┬─_tidb_rowid─┐ β”‚ test19 β”‚ 19 β”‚ diff --git a/tests/delta-merge-test/raft/schema/alter_for_nullable.test b/tests/delta-merge-test/raft/schema/alter_for_nullable.test new file mode 100644 index 00000000000..d61cc50b39d --- /dev/null +++ b/tests/delta-merge-test/raft/schema/alter_for_nullable.test @@ -0,0 +1,70 @@ + +# Preparation. +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __refresh_schemas() + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +# Sync add column by reading. +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String, col_2 Int8, col_3 Int32', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test', 1, 3) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 'test', 2, 4) +=> DBGInvoke __try_flush_region(4) + +# test add nullable flag and change type at the same time. +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(Int32)') +# test trigger by background worker. +=> DBGInvoke __refresh_schemas() +=> select col_2 from default.test +β”Œβ”€col_2─┐ +β”‚ 1 β”‚ +β”‚ 2 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ + +# test only add nullable. +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_3 Nullable(Int32)') + +=> DBGInvoke __put_region(5, 100, 150, default, test) +=> DBGInvoke __raft_insert_row(default, test, 5, 100, 'test', 1, NULL) +=> DBGInvoke __raft_insert_row(default, test, 5, 101, 'test', 2, NULL) +# test trigger by flush worker. +=> DBGInvoke __try_flush_region(5) + +=> select col_3 from default.test +β”Œβ”€col_2─┐ +β”‚ 3 β”‚ +β”‚ 4 β”‚ +β”‚ \N β”‚ +β”‚ \N β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +# Test convert nullable type to not-null type. +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String, col_2 Nullable(Int8)', '', 'dt') +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test', 1) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 'test', 2) +=> DBGInvoke __try_flush_region(4) +=> select col_2 from default.test +β”Œβ”€col_2─┐ +β”‚ 1 β”‚ +β”‚ 2 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Int16') +=> DBGInvoke __refresh_schemas() +=> select col_2 from default.test +β”Œβ”€col_2─┐ +β”‚ 1 β”‚ +β”‚ 2 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __refresh_schemas() diff --git a/tests/delta-merge-test/raft/schema/alter_on_read.test b/tests/delta-merge-test/raft/schema/alter_on_read.test new file mode 100644 index 00000000000..46c3cb4daf5 --- /dev/null +++ b/tests/delta-merge-test/raft/schema/alter_on_read.test @@ -0,0 +1,215 @@ +# Preparation. +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +# Sync add column by reading. +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_2 Nullable(Int8)') +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test1', 1) +# Data in region cache with tso greater than read tso will be ignored. +=> select * from default.test " --read_tso "1500000000000000 +# Data in region cache with tso less than read tso will be force decoded (extra column will be discarded), even under an out-of-date schema. +=> select col_1 from default.test +β”Œβ”€col_1─┐ +β”‚ test1 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ + +# For engine DeltaTree, each write will trigger schema sync. So schema is fresh here. +## Verify this schema is truely out-of-date. +#=> select col_2 from default.test +#Received exception from server (version {#WORD}): +#Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_2. +# Read with specified bigger schema version will trigger schema sync. +=> select col_2 from default.test " --schema_version "10000000 +β”Œβ”€col_2─┐ +β”‚ 1 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ + +# Sync drop column by reading. +=> DBGInvoke __try_flush_region(4) +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_1) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 2) + +# For engine DeltaTree, each write will trigger schema sync. So schema is fresh here. +## Data in region cache with tso less than read tso will be force decoded (missing column will be filled with default value or null), even under an out-of-date schema. +#=> select col_1 from default.test +#β”Œβ”€col_1─┐ +#β”‚ test1 β”‚ +#β”‚ β”‚ +#β””β”€β”€β”€β”€β”€β”€β”€β”˜ +# Read with specified bigger schema version will trigger schema sync. +=> select col_1 from default.test " --schema_version "10000000 +Received exception from server (version {#WORD}): +Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_1. + +# Sync type change by checking sign overflow in CH when flushing. +=> DBGInvoke __try_flush_region(4) +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(Int16)') +=> DBGInvoke __raft_insert_row(default, test, 4, 52, -128) +=> select col_2 from default.test +β”Œβ”€col_2─┐ +β”‚ 1 β”‚ +β”‚ 2 β”‚ +β”‚ -128 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ +=> desc default.test +β”Œβ”€name────────┬─type───────────┬─default_type─┬─default_expression─┐ +β”‚ _tidb_rowid β”‚ Int64 β”‚ β”‚ β”‚ +β”‚ col_2 β”‚ Nullable(Int8) β”‚ β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +=> DBGInvoke __raft_insert_row(default, test, 4, 53, 128) + +# For engine DeltaTree, each write will trigger schema sync. So schema is fresh here. +## 128 will overflow when decoding using out-of-date schema (Int8). +#=> select col_2 from default.test +#Received exception from server (version {#WORD}): +#Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow when decoding data 128 of column col_2 with type Nullable(Int8). +# Read with specified bigger schema version will trigger schema sync. +=> select col_2 from default.test " --schema_version "10000000 +β”Œβ”€col_2─┐ +β”‚ 1 β”‚ +β”‚ 2 β”‚ +β”‚ -128 β”‚ +β”‚ 128 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ + +# Sync type change by checking value overflow in CH when flushing. +=> DBGInvoke __try_flush_region(4) +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(Int64)') +=> DBGInvoke __raft_insert_row(default, test, 4, 54, 65536) + +# For engine DeltaTree, each write will trigger schema sync. So schema is fresh here. +## 65536 will overflow when decoding using out-of-date schema (Int16). +#=> select col_2 from default.test +#Received exception from server (version {#WORD}): +#Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow when decoding data 65536 of column col_2 with type Nullable(Int16). +# Read with specified bigger schema version will trigger schema sync. +=> select col_2 from default.test " --schema_version "10000000 +β”Œβ”€col_2─┐ +β”‚ 1 β”‚ +β”‚ 2 β”‚ +β”‚ -128 β”‚ +β”‚ 128 β”‚ +β”‚ 65536 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ + +# Sync add column and type change together by checking value overflow in CH when flushing. +=> DBGInvoke __try_flush_region(4) +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_3 UInt8') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_3 UInt64') +=> DBGInvoke __raft_insert_row(default, test, 4, 55, 0, 256) +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_4 Nullable(UInt8)') + +# For engine DeltaTree, each write will trigger schema sync. So schema is fresh here. +## 256 will overflow when decoding using out-of-date schema (UInt8). +#=> select col_3 from default.test +#Received exception from server (version {#WORD}): +#Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow when decoding data 256 of column col_3 with type UInt8. +# Read with specified bigger schema version will trigger schema sync. +=> select col_3, col_4 from default.test " --schema_version "10000000 +β”Œβ”€col_3─┬─col_4─┐ +β”‚ 0 β”‚ \N β”‚ +β”‚ 0 β”‚ \N β”‚ +β”‚ 0 β”‚ \N β”‚ +β”‚ 0 β”‚ \N β”‚ +β”‚ 0 β”‚ \N β”‚ +β”‚ 256 β”‚ \N β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜ + +# Edge values not overflowing. +=> DBGInvoke __try_flush_region(4) +=> DBGInvoke __raft_insert_row(default, test, 4, 56, -9223372036854775807, 18446744073709551615, 1) +=> DBGInvoke __raft_insert_row(default, test, 4, 57, 9223372036854775807, 18446744073709551615, 1) +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_3) +# Here engine DeltaTree don't trigger schema sync too +=> select col_2, col_3, col_4 from default.test +β”Œβ”€col_2─┬─col_3─┬─col_4─┐ +β”‚ 1 β”‚ 0 β”‚ \N β”‚ +β”‚ 2 β”‚ 0 β”‚ \N β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜ +β”Œβ”€col_2─┬─col_3─┬─col_4─┐ +β”‚ -128 β”‚ 0 β”‚ \N β”‚ +β”‚ 128 β”‚ 0 β”‚ \N β”‚ +β”‚ 65536 β”‚ 0 β”‚ \N β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€col_2─┬────────────────col_3─┬─col_4─┐ +β”‚ 0 β”‚ 256 β”‚ \N β”‚ +β”‚ -9223372036854775807 β”‚ 18446744073709551615 β”‚ 1 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€col_2─┬────────────────col_3─┬─col_4─┐ +β”‚ 9223372036854775807 β”‚ 18446744073709551615 β”‚ 1 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜ +=> select col_3, col_4 from default.test " --schema_version "100000000 +Received exception from server (version {#WORD}): +Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_3. + +# Sync drop column and type change together by checking value overflow in CH when flushing. +=> DBGInvoke __try_flush_region(4) +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_4 Nullable(UInt64)') +=> DBGInvoke __raft_insert_row(default, test, 4, 58, 0, 256) +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_2) +# For engine DeltaTree, each write will trigger schema sync. So schema is fresh here. +## -256 will overflow when decoding using out-of-date schema (UInt8). +#=> select col_4 from default.test +#Received exception from server (version {#WORD}): +#Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow when decoding data 256 of column col_4 with type Nullable(UInt8). +# Read with specified bigger schema version will trigger schema sync. +=> select col_4 from default.test " --schema_version "100000000 +β”Œβ”€col_4─┐ +β”‚ \N β”‚ +β”‚ \N β”‚ +β”‚ \N β”‚ +β”‚ \N β”‚ +β”‚ \N β”‚ +β”‚ \N β”‚ +β”‚ 1 β”‚ +β”‚ 1 β”‚ +β”‚ 256 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ +=> select col_2 from default.test +Received exception from server (version {#WORD}): +Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_2. +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_2 Nullable(Int8)') +=> DBGInvoke __refresh_schemas() +=> select col_2 from default.test +β”Œβ”€col_2─┐ +β”‚ \N β”‚ +β”‚ \N β”‚ +β”‚ \N β”‚ +β”‚ \N β”‚ +β”‚ \N β”‚ +β”‚ \N β”‚ +β”‚ \N β”‚ +β”‚ \N β”‚ +β”‚ \N β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ +=> DBGInvoke __reset_schemas() +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_2) + +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_2 Int8') + +=> DBGInvoke __refresh_schemas() + +=> select col_2 from default.test +β”Œβ”€col_2─┐ +β”‚ 0 β”‚ +β”‚ 0 β”‚ +β”‚ 0 β”‚ +β”‚ 0 β”‚ +β”‚ 0 β”‚ +β”‚ 0 β”‚ +β”‚ 0 β”‚ +β”‚ 0 β”‚ +β”‚ 0 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ +# Clean up. +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __enable_schema_sync_service('true') diff --git a/tests/delta-merge-test/raft/schema/alter_on_write.test b/tests/delta-merge-test/raft/schema/alter_on_write.test new file mode 100644 index 00000000000..02ccb8be935 --- /dev/null +++ b/tests/delta-merge-test/raft/schema/alter_on_write.test @@ -0,0 +1,132 @@ +# Preparation. +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +# Sync add column by checking missing column in CH when flushing. +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> select col_1 from default.test +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_2 Nullable(Int8)') +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test1', 1) +=> DBGInvoke __try_flush_region(4) +=> selraw nokvstore col_2 from default.test +β”Œβ”€col_2─┐ +β”‚ 1 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ + +# Sync drop column by checking redundant column in CH when flushing. +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_1) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 2) +=> DBGInvoke __try_flush_region(4) +=> selraw nokvstore col_2 from default.test +β”Œβ”€col_1─┐ +β”‚ 1 β”‚ +β”‚ 2 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ +=> selraw nokvstore col_1 from default.test +Received exception from server (version {#WORD}): +Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_1. + +# Sync type change by checking sign overflow in CH when flushing. +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(Int16)') +=> DBGInvoke __raft_insert_row(default, test, 4, 52, -128) +=> DBGInvoke __try_flush_region(4) +=> selraw nokvstore col_2 from default.test +β”Œβ”€col_2─┐ +β”‚ 1 β”‚ +β”‚ 2 β”‚ +β”‚ -128 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ +=> desc default.test +β”Œβ”€name────────┬─type───────────┬─default_type─┬─default_expression─┐ +β”‚ _tidb_rowid β”‚ Int64 β”‚ β”‚ β”‚ +β”‚ col_2 β”‚ Nullable(Int8) β”‚ β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +=> DBGInvoke __raft_insert_row(default, test, 4, 53, 128) +=> DBGInvoke __try_flush_region(4) +=> selraw nokvstore col_2 from default.test +β”Œβ”€col_2─┐ +β”‚ 1 β”‚ +β”‚ 2 β”‚ +β”‚ -128 β”‚ +β”‚ 128 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ + +# Sync type change by checking value overflow in CH when flushing. +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(Int64)') +=> DBGInvoke __raft_insert_row(default, test, 4, 54, 65536) +=> DBGInvoke __try_flush_region(4) +=> selraw nokvstore col_2 from default.test +β”Œβ”€col_2─┐ +β”‚ 1 β”‚ +β”‚ 2 β”‚ +β”‚ -128 β”‚ +β”‚ 128 β”‚ +β”‚ 65536 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ + +# Sync add column and type change together by checking value overflow in CH when flushing. +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_3 UInt8') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_3 UInt64') +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_4 Nullable(UInt8)') +# For DeltaTree, each write will trigger schema sync. +=> DBGInvoke __raft_insert_row(default, test, 4, 55, 0, 256, 0) +=> selraw nokvstore col_3, col_4 from default.test +β”Œβ”€col_3─┬─col_4─┐ +β”‚ 0 β”‚ \N β”‚ +β”‚ 0 β”‚ \N β”‚ +β”‚ 0 β”‚ \N β”‚ +β”‚ 0 β”‚ \N β”‚ +β”‚ 0 β”‚ \N β”‚ +β”‚ 256 β”‚ 0 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜ + +# Not sync drop column for edge values not overflowing. +=> DBGInvoke __raft_insert_row(default, test, 4, 56, -9223372036854775807, 18446744073709551615, 1) +=> DBGInvoke __raft_insert_row(default, test, 4, 57, 9223372036854775807, 18446744073709551615, 1) +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_3) +=> DBGInvoke __try_flush_region(4) +=> selraw nokvstore col_2, col_3, col_4 from default.test +β”Œβ”€col_2─┬─col_3─┬─col_4─┐ +β”‚ 1 β”‚ 0 β”‚ \N β”‚ +β”‚ 2 β”‚ 0 β”‚ \N β”‚ +β”‚ -128 β”‚ 0 β”‚ \N β”‚ +β”‚ 128 β”‚ 0 β”‚ \N β”‚ +β”‚ 65536 β”‚ 0 β”‚ \N β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜ +β”Œβ”€col_2─┬─col_3─┬─col_4─┐ +β”‚ 0 β”‚ 256 β”‚ 0 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€col_2─┬────────────────col_3─┬─col_4─┐ +β”‚ -9223372036854775807 β”‚ 18446744073709551615 β”‚ 1 β”‚ +β”‚ 9223372036854775807 β”‚ 18446744073709551615 β”‚ 1 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜ +=> DBGInvoke __refresh_schemas() +=> selraw nokvstore col_3 from default.test +Received exception from server (version {#WORD}): +Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_3. + +# Sync drop column and type change together by checking value overflow in CH when flushing. +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_4 Nullable(UInt64)') +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_2) +# For DeltaTree, each write will trigger schema sync. +=> DBGInvoke __raft_insert_row(default, test, 4, 58, 256) +=> desc default.test +β”Œβ”€name────────┬─type─────────────┬─default_type─┬─default_expression─┐ +β”‚ _tidb_rowid β”‚ Int64 β”‚ β”‚ β”‚ +β”‚ col_4 β”‚ Nullable(UInt64) β”‚ β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +=> selraw nokvstore col_2 from default.test +Received exception from server (version {#WORD}): +Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_2. + +# Clean up. +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __enable_schema_sync_service('true') diff --git a/tests/delta-merge-test/raft/schema/default_value.test b/tests/delta-merge-test/raft/schema/default_value.test new file mode 100644 index 00000000000..ab097ff5f69 --- /dev/null +++ b/tests/delta-merge-test/raft/schema/default_value.test @@ -0,0 +1,42 @@ +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String', '', 'dt') + +=> DBGInvoke __put_region(4, 0, 100, default, test) + +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 'test2') +=> DBGInvoke __raft_insert_row(default, test, 4, 52, 'test2') +=> DBGInvoke __raft_insert_row(default, test, 4, 53, 'test2') +=> DBGInvoke __try_flush_region(4) + +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_2 Nullable(Int64) DEFAULT 123') +# test default value of time type +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_3 DEFAULT \'asTiDBType|Nullable(Time)|11:11:11\'') +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_4 DEFAULT \'asTiDBType|Nullable(Time(3))|-11:11:11.111\'') +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_5 DEFAULT \'asTiDBType|Nullable(Time(3))|111:11:11.111\'') +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_6 DEFAULT \'asTiDBType|Nullable(Time)|-111:11:11\'') +# test default value of year type +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_7 DEFAULT \'asTiDBType|Nullable(year)|0\'') +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_8 DEFAULT \'asTiDBType|Nullable(year)|70\'') +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_9 DEFAULT \'asTiDBType|Nullable(year)|1986\'') +# test default value of set type +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_10 DEFAULT \'asTiDBType|Nullable(set(\\\\\'1\\\\\',\\\\\'4\\\\\',\\\\\'10\\\\\',\\\\\'20\\\\\'))|1,10\'') + +=> DBGInvoke __put_region(5, 100, 200, default, test) +=> DBGInvoke __raft_insert_row(default, test, 5, 154, 'test2', 321, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) +=> DBGInvoke __raft_insert_row(default, test, 5, 155, 'test2', 321, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) +=> DBGInvoke __try_flush_region(5) + +=> select * from default.test +β”Œβ”€col_1─┬─_tidb_rowid─┬─col_2─┬──────────col_3─┬───────────col_4─┬───────────col_5─┬────────────col_6─┬─col_7─┬─col_8─┬─col_9─┬─col_10─┐ +β”‚ test2 β”‚ 154 β”‚ 321 β”‚ \N β”‚ \N β”‚ \N β”‚ \N β”‚ \N β”‚ \N β”‚ \N β”‚ \N β”‚ +β”‚ test2 β”‚ 155 β”‚ 321 β”‚ \N β”‚ \N β”‚ \N β”‚ \N β”‚ \N β”‚ \N β”‚ \N β”‚ \N β”‚ +β”‚ test2 β”‚ 51 β”‚ 123 β”‚ 40271000000000 β”‚ -40271111000000 β”‚ 400271111000000 β”‚ -400271000000000 β”‚ 2000 β”‚ 1970 β”‚ 1986 β”‚ 5 β”‚ +β”‚ test2 β”‚ 52 β”‚ 123 β”‚ 40271000000000 β”‚ -40271111000000 β”‚ 400271111000000 β”‚ -400271000000000 β”‚ 2000 β”‚ 1970 β”‚ 1986 β”‚ 5 β”‚ +β”‚ test2 β”‚ 53 β”‚ 123 β”‚ 40271000000000 β”‚ -40271111000000 β”‚ 400271111000000 β”‚ -400271000000000 β”‚ 2000 β”‚ 1970 β”‚ 1986 β”‚ 5 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __enable_schema_sync_service('true') diff --git a/tests/delta-merge-test/raft/schema/drop_on_read.test b/tests/delta-merge-test/raft/schema/drop_on_read.test new file mode 100644 index 00000000000..2cb42140300 --- /dev/null +++ b/tests/delta-merge-test/raft/schema/drop_on_read.test @@ -0,0 +1,27 @@ +=> DBGInvoke __clean_up_region() +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __drop_tidb_table(default, test, 'false') +=> select * from default.test +=> select * from default.test " --schema_version "10000000 +Received exception from server (version {#WORD}): +Code: 60. DB::Exception: Received from {#WORD} DB::Exception: Table default.test doesn't exist.. +=> DBGInvoke __clean_up_region() + +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String, col_2 Nullable(Int8)', '', 'dt') +=> select * from default.test +Received exception from server (version {#WORD}): +Code: 60. DB::Exception: Received from {#WORD} DB::Exception: Table default.test doesn't exist.. +=> select * from default.test " --schema_version "10000000 + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __enable_schema_sync_service('true') diff --git a/tests/delta-merge-test/raft/schema/drop_on_restart.test b/tests/delta-merge-test/raft/schema/drop_on_restart.test new file mode 100644 index 00000000000..fe61a736a6b --- /dev/null +++ b/tests/delta-merge-test/raft/schema/drop_on_restart.test @@ -0,0 +1,54 @@ +=> DBGInvoke __clean_up_region() +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> DBGInvoke __drop_tidb_db(db3) +=> DBGInvoke __drop_tidb_db(db4) +=> drop table if exists default.test +=> DBGInvoke __refresh_schemas() + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +=> DBGInvoke __mock_tidb_db(db3) +=> DBGInvoke __mock_tidb_db(db4) +=> DBGInvoke __mock_tidb_table(db3, test, 'col_1 String', '', 'dt') +=> DBGInvoke __mock_tidb_table(db4, test, 'col_1 String', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> show databases +β”Œβ”€name────┐ +β”‚ db3 β”‚ +β”‚ db4 β”‚ +β”‚ default β”‚ +β”‚ system β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +=> DBGInvoke __reset_schemas() +=> DBGInvoke __drop_tidb_db(db3); +=> DBGInvoke __drop_tidb_table(db4, test); +=> DBGInvoke __refresh_schemas() +=> show databases +β”Œβ”€name────┐ +β”‚ db4 β”‚ +β”‚ default β”‚ +β”‚ system β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +=> select * from db4.test +Received exception from server (version {#WORD}): +Code: 60. DB::Exception: Received from {#WORD} DB::Exception: Table db4.test doesn't exist.. +=> DBGInvoke __mock_tidb_db(db3) +=> DBGInvoke __refresh_schemas() +=> show databases +β”Œβ”€name────┐ +β”‚ db3 β”‚ +β”‚ db4 β”‚ +β”‚ default β”‚ +β”‚ system β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +=> DBGInvoke __reset_schemas() +=> DBGInvoke __drop_tidb_db(db3); +=> DBGInvoke __drop_tidb_db(db4); +=> DBGInvoke __refresh_schemas() +=> show databases +β”Œβ”€name────┐ +β”‚ default β”‚ +β”‚ system β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ diff --git a/tests/delta-merge-test/raft/schema/drop_on_write.test b/tests/delta-merge-test/raft/schema/drop_on_write.test new file mode 100644 index 00000000000..fa8ff9585a4 --- /dev/null +++ b/tests/delta-merge-test/raft/schema/drop_on_write.test @@ -0,0 +1,67 @@ +#TODO: For DeltaTree, each write will trigger schema sync. So we can not mock this situation, this test is useless +#RETURN + +=> DBGInvoke __clean_up_region() +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> select col_1 from default.test +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_2 Nullable(Int8)') +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test1', 1) +=> DBGInvoke __drop_tidb_table(default, test, 'false') +=> DBGInvoke __try_flush_region(4) +=> select * from default.test +Received exception from server (version {#WORD}): +Code: 60. DB::Exception: Received from {#WORD} DB::Exception: Table default.test doesn't exist.. + +=> DBGInvoke __clean_up_region() +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String, col_2 Nullable(Int8)', '', 'dt') +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __refresh_schemas() +=> select col_1, col_2 from default.test +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_2) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 'test2') +=> DBGInvoke __drop_tidb_table(default, test, 'false') +=> DBGInvoke __try_flush_region(4) +=> select * from default.test +β”Œβ”€col_1─┬─col_2─┬─_tidb_rowid─┐ +β”‚ test2 β”‚ \N β”‚ 51 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +=> DBGInvoke __clean_up_region() +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String, col_2 Int8', '', 'dt') +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __refresh_schemas() +=> select col_1, col_2 from default.test +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_2) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 'test2') +=> DBGInvoke __drop_tidb_table(default, test, 'false') +=> DBGInvoke __try_flush_region(4) +=> select * from default.test +Received exception from server (version {#WORD}): +Code: 60. DB::Exception: Received from {#WORD} DB::Exception: Table default.test doesn't exist.. + +=> DBGInvoke __clean_up_region() +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String, col_2 Nullable(Int8)', '', 'dt') +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __refresh_schemas() +=> select col_1, col_2 from default.test +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(Int64)') +=> DBGInvoke __raft_insert_row(default, test, 4, 52, 'test2', 256) +=> DBGInvoke __drop_tidb_table(default, test, 'false') +=> DBGInvoke __try_flush_region(4) +=> select * from default.test +Received exception from server (version {#WORD}): +Code: 60. DB::Exception: Received from {#WORD} DB::Exception: Table default.test doesn't exist.. + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __enable_schema_sync_service('true') +=> DBGInvoke __clean_up_region() diff --git a/tests/delta-merge-test/raft/schema/mydate.test b/tests/delta-merge-test/raft/schema/mydate.test new file mode 100644 index 00000000000..b8fd45b8237 --- /dev/null +++ b/tests/delta-merge-test/raft/schema/mydate.test @@ -0,0 +1,73 @@ + +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 MyDateTime(1)', '', 'dt') + +=> DBGInvoke __put_region(4, 0, 100, default, test) + +=> DBGInvoke __raft_insert_row(default, test, 4, 51, '1991-11-12 11:12:13.234') +=> DBGInvoke __raft_insert_row(default, test, 4, 52, '1991-00-14 11:00:01') +=> DBGInvoke __raft_insert_row(default, test, 4, 53, '2001-12-13 11:11:11') +=> DBGInvoke __try_flush_region(4) +=> select * from default.test + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€col_1───┬─_tidb_rowid─┐ +β”‚ 1991-11-12 11:12:13.2 β”‚ 51 β”‚ +β”‚ 1991-00-14 11:00:01.0 β”‚ 52 β”‚ +β”‚ 2001-12-13 11:11:11.0 β”‚ 53 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 MyDate', '', 'dt') + +=> DBGInvoke __put_region(4, 0, 100, default, test) + +=> DBGInvoke __raft_insert_row(default, test, 4, 51, '1991-11-12') +=> DBGInvoke __raft_insert_row(default, test, 4, 52, '1991-00-14') +=> DBGInvoke __raft_insert_row(default, test, 4, 53, '2001-12-13') +=> DBGInvoke __try_flush_region(4) +=> select * from default.test + +β”Œβ”€β”€β”€β”€β”€β”€col_1─┬─_tidb_rowid─┐ +β”‚ 1991-11-12 β”‚ 51 β”‚ +β”‚ 1991-00-14 β”‚ 52 β”‚ +β”‚ 2001-12-13 β”‚ 53 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 MyDateTime(1)', '', 'dt') + +=> DBGInvoke __put_region(4, 0, 100, default, test) + +=> DBGInvoke __raft_insert_row(default, test, 4, 51, '1991-11-12 11:12:13.234') +=> DBGInvoke __raft_insert_row(default, test, 4, 52, '1991-01-14 11:00:01') +=> DBGInvoke __raft_insert_row(default, test, 4, 53, '2001-12-13 11:11:11') +=> DBGInvoke __raft_insert_row(default, test, 4, 54, '1991-09-05 11:00:01') +=> DBGInvoke __try_flush_region(4) +=> select * from default.test + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€col_1───┬─_tidb_rowid─┐ +β”‚ 1991-11-12 11:12:13.2 β”‚ 51 β”‚ +β”‚ 1991-01-14 11:00:01.0 β”‚ 52 β”‚ +β”‚ 2001-12-13 11:11:11.0 β”‚ 53 β”‚ +β”‚ 1991-09-05 11:00:01.0 β”‚ 54 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +=> select ConvertTimeZoneFromUTC(col_1, 'Asia/Shanghai') as col_1 , _tidb_rowid from default.test +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€col_1───┬─_tidb_rowid─┐ +β”‚ 1991-11-12 19:12:13.2 β”‚ 51 β”‚ +β”‚ 1991-01-14 19:00:01.0 β”‚ 52 β”‚ +β”‚ 2001-12-13 19:11:11.0 β”‚ 53 β”‚ +β”‚ 1991-09-05 20:00:01.0 β”‚ 54 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __enable_schema_sync_service('true') diff --git a/tests/delta-merge-test/raft/schema/partition_table_restart.test b/tests/delta-merge-test/raft/schema/partition_table_restart.test new file mode 100644 index 00000000000..26be87b70e8 --- /dev/null +++ b/tests/delta-merge-test/raft/schema/partition_table_restart.test @@ -0,0 +1,43 @@ +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> DBGInvoke __drop_tidb_table(default, test1) + +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String, col_2 Int64', '', 'dt') +=> DBGInvoke __mock_tidb_partition(default, test, 9999) +=> DBGInvoke __mock_tidb_partition(default, test, 9998) +=> DBGInvoke __mock_tidb_partition(default, test, 9997) +=> DBGInvoke __refresh_schemas() +=> drop table test_9997 +=> DBGInvoke __reset_schemas() + +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_3 Nullable(Int8)') +=> DBGInvoke __rename_tidb_table(default, test, test1) +=> DBGInvoke __refresh_schemas() + +=> show tables +β”Œβ”€name───────┐ +β”‚ test1 β”‚ +β”‚ test1_9997 β”‚ +β”‚ test1_9998 β”‚ +β”‚ test1_9999 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +=> select col_2 from default.test1_9997 +=> select * from default.test_9997 +Received exception from server (version {#WORD}): +Code: 60. DB::Exception: Received from {#WORD} DB::Exception: Table default.test_9997 doesn't exist.. +=> select * from default.test_9998 +Received exception from server (version {#WORD}): +Code: 60. DB::Exception: Received from {#WORD} DB::Exception: Table default.test_9998 doesn't exist.. + +=> DBGInvoke __reset_schemas() +=> DBGInvoke __drop_tidb_partition(default, test1, 9997) +=> DBGInvoke __refresh_schemas() +=> select * from default.test1_9997 +Received exception from server (version {#WORD}): +Code: 60. DB::Exception: Received from {#WORD} DB::Exception: Table default.test1_9997 doesn't exist.. +=> select * from default.test1_9999 +=> DBGInvoke __drop_tidb_table(default, test1) +=> DBGInvoke __refresh_schemas() + +=> DBGInvoke __enable_schema_sync_service('true') diff --git a/tests/delta-merge-test/raft/schema/rename_column copy.test b/tests/delta-merge-test/raft/schema/rename_column copy.test new file mode 100644 index 00000000000..961245ee6c4 --- /dev/null +++ b/tests/delta-merge-test/raft/schema/rename_column copy.test @@ -0,0 +1,216 @@ + +# Preparation. +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __refresh_schemas() + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +# create table and insert some rows +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String, col_2 Int8', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test', 1) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 'test', 2) +=> DBGInvoke __try_flush_region(4) +=> select * from default.test +β”Œβ”€col_1─┬─col_2─┬─_tidb_rowid─┐ +β”‚ test β”‚ 1 β”‚ 50 β”‚ +β”‚ test β”‚ 2 β”‚ 51 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +# test cyclic rename +=> DBGInvoke __rename_column_in_tidb_table(default, test, col_1, col_3) +=> DBGInvoke __rename_column_in_tidb_table(default, test, col_2, col_1) +=> DBGInvoke __rename_column_in_tidb_table(default, test, col_3, col_2) +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_1 Nullable(Int32)') +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(String)') +=> DBGInvoke __refresh_schemas() +# test doesn't check column name, so we select columns one by one. +=> select col_2 from default.test +β”Œβ”€col_2─┐ +β”‚ test β”‚ +β”‚ test β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ +=> select col_1 from default.test +β”Œβ”€col_1─┐ +β”‚ 1 β”‚ +β”‚ 2 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __refresh_schemas() + +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String, col_2 Int8', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test', 1) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 'test', 2) +=> DBGInvoke __try_flush_region(4) +=> select * from default.test +β”Œβ”€col_1─┬─col_2─┬─_tidb_rowid─┐ +β”‚ test β”‚ 1 β”‚ 50 β”‚ +β”‚ test β”‚ 2 β”‚ 51 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +# test change nullable firstly, then rename cyclic. + +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_1 Nullable(String)') +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(Int32)') +=> DBGInvoke __refresh_schemas() +=> select * from default.test +β”Œβ”€col_1─┬─col_2─┬─_tidb_rowid─┐ +β”‚ test β”‚ 1 β”‚ 50 β”‚ +β”‚ test β”‚ 2 β”‚ 51 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +=> DBGInvoke __rename_column_in_tidb_table(default, test, col_1, col_3) +=> DBGInvoke __rename_column_in_tidb_table(default, test, col_2, col_1) +=> DBGInvoke __rename_column_in_tidb_table(default, test, col_3, col_2) +=> DBGInvoke __refresh_schemas() +=> select col_2 from default.test +β”Œβ”€col_2─┐ +β”‚ test β”‚ +β”‚ test β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ +=> select col_1 from default.test +β”Œβ”€col_1─┐ +β”‚ 1 β”‚ +β”‚ 2 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __refresh_schemas() + + +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String, col_2 Int8', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test', 1) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 'test', 2) +=> DBGInvoke __try_flush_region(4) +=> select * from default.test +β”Œβ”€col_1─┬─col_2─┬─_tidb_rowid─┐ +β”‚ test β”‚ 1 β”‚ 50 β”‚ +β”‚ test β”‚ 2 β”‚ 51 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +# test drop and then rename and then create + +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_1) +=> DBGInvoke __rename_column_in_tidb_table(default, test, col_2, col_1) +=> DBGInvoke __refresh_schemas() +=> select col_1 from default.test +β”Œβ”€col_1─┐ +β”‚ 1 β”‚ +β”‚ 2 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_2 Nullable(Int8) default 0') +=> DBGInvoke __refresh_schemas() +=> select col_2 from default.test +β”Œβ”€col_2─┐ +β”‚ 0 β”‚ +β”‚ 0 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __refresh_schemas() + +# do a lot of change and sync at once. + +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String, col_2 Int8', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test', 1) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 'test', 2) +=> DBGInvoke __try_flush_region(4) +=> select * from default.test +β”Œβ”€col_1─┬─col_2─┬─_tidb_rowid─┐ +β”‚ test β”‚ 1 β”‚ 50 β”‚ +β”‚ test β”‚ 2 β”‚ 51 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +# test drop and then rename and then create + +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_1) +=> DBGInvoke __rename_column_in_tidb_table(default, test, col_2, col_1) +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_2 Nullable(Int8) default 0') +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_1 Nullable(Int32)') +=> DBGInvoke __reset_schemas() +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(5, 101, 200, default, test) +=> DBGInvoke __raft_insert_row(default, test, 5, 150, NULL, NULL) +=> DBGInvoke __raft_insert_row(default, test, 5, 151, NULL, NULL) +=> DBGInvoke __try_flush_region(5) +=> select * from default.test +β”Œβ”€col_1─┬─_tidb_rowid─┬─col_2─┐ +β”‚ 1 β”‚ 50 β”‚ 0 β”‚ +β”‚ 2 β”‚ 51 β”‚ 0 β”‚ +β”‚ \N β”‚ 150 β”‚ \N β”‚ +β”‚ \N β”‚ 151 β”‚ \N β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜ + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __refresh_schemas() + +## test for partial-linked rename +=> DBGInvoke __mock_tidb_table(default, test, 'a String, b Int8', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test', 1) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 'test', 2) +=> DBGInvoke __try_flush_region(4) +=> select a, b from default.test order by _tidb_rowid +β”Œβ”€a────┬─b─┐ +β”‚ test β”‚ 1 β”‚ +β”‚ test β”‚ 2 β”‚ +β””β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”˜ + +# rename a -> c, and b -> a +=> DBGInvoke __rename_column_in_tidb_table(default, test, a, c) +=> DBGInvoke __rename_column_in_tidb_table(default, test, b, a) +=> DBGInvoke __refresh_schemas() +=> select a, c from default.test order by _tidb_rowid +β”Œβ”€a─┬─c────┐ +β”‚ 1 β”‚ test β”‚ +β”‚ 2 β”‚ test β”‚ +β””β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”˜ + +# rename after add column +=> DBGInvoke __add_column_to_tidb_table(default, test, 'k Nullable(Int8) default 0') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __rename_column_in_tidb_table(default, test, k, g) +=> DBGInvoke __refresh_schemas() +=> select a, g from default.test order by _tidb_rowid +β”Œβ”€a─┬─g─┐ +β”‚ 1 β”‚ 0 β”‚ +β”‚ 2 β”‚ 0 β”‚ +β””β”€β”€β”€β”΄β”€β”€β”€β”˜ + +=> DBGInvoke __rename_column_in_tidb_table(default, test, a, d) +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __rename_column_in_tidb_table(default, test, g, k) +=> DBGInvoke __refresh_schemas() +=> select d, k from default.test order by _tidb_rowid +β”Œβ”€d─┬─k─┐ +β”‚ 1 β”‚ 0 β”‚ +β”‚ 2 β”‚ 0 β”‚ +β””β”€β”€β”€β”΄β”€β”€β”€β”˜ + +=> DBGInvoke __rename_tidb_table(default, test, test1) +=> DBGInvoke __add_column_to_tidb_table(default, test1, 'z Nullable(Int8) default 0') +=> DBGInvoke __refresh_schemas() +=> select d, z from default.test1 order by _tidb_rowid +β”Œβ”€d─┬─z─┐ +β”‚ 1 β”‚ 0 β”‚ +β”‚ 2 β”‚ 0 β”‚ +β””β”€β”€β”€β”΄β”€β”€β”€β”˜ + +=> DBGInvoke __drop_tidb_table(default, test1) +=> drop table if exists default.test +=> DBGInvoke __refresh_schemas() diff --git a/tests/delta-merge-test/raft/schema/rename_column.test b/tests/delta-merge-test/raft/schema/rename_column.test index 0e4029e27fe..28b01b5ba7f 100644 --- a/tests/delta-merge-test/raft/schema/rename_column.test +++ b/tests/delta-merge-test/raft/schema/rename_column.test @@ -1,6 +1,3 @@ -#TODO: enable ddl tests for DeltaMerge -#RETURN - # Preparation. => DBGInvoke __enable_schema_sync_service('false') diff --git a/tests/delta-merge-test/raft/schema/rename_on_read.test b/tests/delta-merge-test/raft/schema/rename_on_read.test new file mode 100644 index 00000000000..7d077bf6d3b --- /dev/null +++ b/tests/delta-merge-test/raft/schema/rename_on_read.test @@ -0,0 +1,24 @@ +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __drop_tidb_table(default, test1) +=> drop table if exists default.test1 + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> select * from default.test +=> DBGInvoke __rename_tidb_table(default, test, test1) +=> select * from default.test +=> select * from default.test " --schema_version "1000000 +Received exception from server (version {#WORD}): +Code: 60. DB::Exception: Received from {#WORD} DB::Exception: Table default.test doesn't exist.. +=> select * from default.test1 +=> select * from default.test1 " --schema_version "1000000 + +=> DBGInvoke __drop_tidb_table(default, test1) +=> drop table if exists default.test1 +=> DBGInvoke __enable_schema_sync_service('true') diff --git a/tests/delta-merge-test/raft/schema/rename_on_write.test b/tests/delta-merge-test/raft/schema/rename_on_write.test new file mode 100644 index 00000000000..522e9ad3702 --- /dev/null +++ b/tests/delta-merge-test/raft/schema/rename_on_write.test @@ -0,0 +1,32 @@ +#TODO: We can not mock this situation, ignore for now +#RETURN + +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __drop_tidb_table(default, test1) +=> drop table if exists default.test1 + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> select col_1 from default.test +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_2 Nullable(Int8)') +=> DBGInvoke __rename_tidb_table(default, test, test1) +#For DeltaTree, each write will trigger schema sync. +=> DBGInvoke __raft_insert_row(default, test1, 4, 50, 'test1', 1) +=> select * from default.test +Received exception from server (version {#WORD}): +Code: 60. DB::Exception: Received from {#WORD} DB::Exception: Table default.test doesn't exist.. +=> select * from default.test1 +β”Œβ”€col_1─┬─_tidb_rowid─┬─col_2─┐ +β”‚ test1 β”‚ 50 β”‚ 1 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜ + +=> DBGInvoke __drop_tidb_table(default, test1) +=> drop table if exists default.test1 +=> DBGInvoke __enable_schema_sync_service('true') diff --git a/tests/delta-merge-test/raft/schema/truncate_on_read.test b/tests/delta-merge-test/raft/schema/truncate_on_read.test new file mode 100644 index 00000000000..09c71658fad --- /dev/null +++ b/tests/delta-merge-test/raft/schema/truncate_on_read.test @@ -0,0 +1,27 @@ +=> DBGInvoke __clean_up_region() +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test1') +=> select * from default.test +β”Œβ”€col_1─┬─_tidb_rowid─┐ +β”‚ test1 β”‚ 50 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +=> DBGInvoke __truncate_tidb_table(default, test) +=> select * from default.test +β”Œβ”€col_1─┬─_tidb_rowid─┐ +β”‚ test1 β”‚ 50 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +=> select * from default.test " --schema_version "10000000 + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __enable_schema_sync_service('true') +=> DBGInvoke __clean_up_region() diff --git a/tests/delta-merge-test/raft/schema/truncate_on_write.test b/tests/delta-merge-test/raft/schema/truncate_on_write.test new file mode 100644 index 00000000000..c6dc4e1e2c1 --- /dev/null +++ b/tests/delta-merge-test/raft/schema/truncate_on_write.test @@ -0,0 +1,29 @@ +#TODO: We can not mock this situation, ignore for now +#RETURN + +=> DBGInvoke __clean_up_region() +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String', '', 'dt') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test1') +=> select col_1 from default.test +β”Œβ”€col_1─┐ +β”‚ test1 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_2 Nullable(Int8)') +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 'test1', 1) +=> DBGInvoke __truncate_tidb_table(default, test) +=> DBGInvoke __try_flush_region(4) +=> select * from default.test + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __enable_schema_sync_service('true') +=> DBGInvoke __clean_up_region() diff --git a/tests/docker/config/tics_dt.toml b/tests/docker/config/tics_dt.toml new file mode 100644 index 00000000000..ebe665ea166 --- /dev/null +++ b/tests/docker/config/tics_dt.toml @@ -0,0 +1,23 @@ +tmp_path = "/tmp/tiflash/data/tmp" +display_name = "TiFlash" +default_profile = "default" +users_config = "users.toml" +# specify paths used for store data, multiple path should be seperated by comma +path = "/tmp/tiflash/data/db" +mark_cache_size = 5368709120 +minmax_index_cache_size = 5368709120 +listen_host = "0.0.0.0" +tcp_port = 9000 +http_port = 8123 +interserver_http_port = 9009 +[logger] + count = 10 + errorlog = "/tmp/tiflash/log/error.log" + size = "1000M" + log = "/tmp/tiflash/log/server.log" + level = "trace" +[application] + runAsDaemon = true +[raft] + # specify which storage engine we use. tmt or dt + storage_engine = "dt" diff --git a/tests/docker/config/config.toml b/tests/docker/config/tics_tmt.toml similarity index 100% rename from tests/docker/config/config.toml rename to tests/docker/config/tics_tmt.toml diff --git a/tests/docker/mock-test.yaml b/tests/docker/mock-test-dt.yaml similarity index 77% rename from tests/docker/mock-test.yaml rename to tests/docker/mock-test-dt.yaml index 0660e2a9bb7..22ef3e82337 100644 --- a/tests/docker/mock-test.yaml +++ b/tests/docker/mock-test-dt.yaml @@ -10,10 +10,10 @@ services: - "8123:8123" - "9009:9009" volumes: - - ./config/config.toml:/config.toml:ro + - ./config/tics_dt.toml:/config.toml:ro - ./config/users.toml:/users.toml:ro - - ./data/tics0:/tmp/tiflash/data - - ./log/tics0:/tmp/tiflash/log + - ./data/tics_dt:/tmp/tiflash/data + - ./log/tics_dt:/tmp/tiflash/log - ..:/tests - ./_env.sh:/tests/_env.sh command: diff --git a/tests/docker/mock-test-tmt.yaml b/tests/docker/mock-test-tmt.yaml new file mode 100644 index 00000000000..cf443c9887e --- /dev/null +++ b/tests/docker/mock-test-tmt.yaml @@ -0,0 +1,21 @@ +version: '2.3' + +services: + # tics0 container is for tests under mutable-test && delta-merge-test directory + tics0: + image: hub.pingcap.net/tiflash/tics:${TAG:-master} + ports: + - "3930:3930" + - "9000:9000" + - "8123:8123" + - "9009:9009" + volumes: + - ./config/tics_tmt.toml:/config.toml:ro + - ./config/users.toml:/users.toml:ro + - ./data/tics_tmt:/tmp/tiflash/data + - ./log/tics_tmt:/tmp/tiflash/log + - ..:/tests + - ./_env.sh:/tests/_env.sh + command: + - --config-file + - /config.toml diff --git a/tests/docker/run.sh b/tests/docker/run.sh index ee36ab1cba3..a1be3fea672 100644 --- a/tests/docker/run.sh +++ b/tests/docker/run.sh @@ -4,7 +4,7 @@ set -xe # Stop all docker instances if exist. # tiflash-dt && tiflash-tmt share the same name "tiflash0", we just need one here -docker-compose -f gtest.yaml -f cluster.yaml -f tiflash-dt.yaml -f mock-test.yaml down +docker-compose -f gtest.yaml -f cluster.yaml -f tiflash-dt.yaml -f mock-test-dt.yaml down rm -rf ./data ./log # run gtest cases. (only tics-gtest up) @@ -19,14 +19,22 @@ docker-compose -f cluster.yaml -f tiflash-dt.yaml up -d sleep 60 docker-compose -f cluster.yaml -f tiflash-dt.yaml up -d --build sleep 10 -# TODO: Enable fullstack-test/ddl for engine DeltaTree -docker-compose -f cluster.yaml -f tiflash-dt.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh fullstack-test/dml true' -docker-compose -f cluster.yaml -f tiflash-dt.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh fullstack-test/expr true' -#docker-compose -f cluster.yaml -f tiflash-dt.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh fullstack-test/fault-inject true' -docker-compose -f cluster.yaml -f tiflash-dt.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh fullstack-test/sample.test true' +docker-compose -f cluster.yaml -f tiflash-dt.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh fullstack-test true' docker-compose -f cluster.yaml -f tiflash-dt.yaml down +# We need to separate mock-test for dt and tmt, since this behavior +# is different in some tests +# * "tmt" engine ONLY support disable_bg_flush = false. +# * "dt" engine ONLY support disable_bg_flush = true. +rm -rf ./data ./log +# (only tics0 up) (for engine DetlaTree) +docker-compose -f mock-test-dt.yaml up -d +docker-compose -f mock-test-dt.yaml exec -T tics0 bash -c 'cd /tests ; ./run-test.sh delta-merge-test' +docker-compose -f mock-test-dt.yaml down + + + rm -rf ./data ./log # run fullstack-tests (for engine TxnMergeTree) docker-compose -f cluster.yaml -f tiflash-tmt.yaml up -d @@ -38,7 +46,7 @@ docker-compose -f cluster.yaml -f tiflash-tmt.yaml down rm -rf ./data ./log -# (only tics0 up) -docker-compose -f mock-test.yaml up -d -docker-compose -f mock-test.yaml exec -T tics0 bash -c 'cd /tests ; ./run-test.sh delta-merge-test && ./run-test.sh mutable-test' -docker-compose -f mock-test.yaml down +# (only tics0 up) (for engine TxnMergeTree) +docker-compose -f mock-test-tmt.yaml up -d +docker-compose -f mock-test-tmt.yaml exec -T tics0 bash -c 'cd /tests ; ./run-test.sh mutable-test' +docker-compose -f mock-test-tmt.yaml down diff --git a/tests/fullstack-test/ddl/alter_datetime_default_value.test b/tests/fullstack-test/ddl/alter_datetime_default_value.test new file mode 100644 index 00000000000..229aae6081b --- /dev/null +++ b/tests/fullstack-test/ddl/alter_datetime_default_value.test @@ -0,0 +1,90 @@ +mysql> drop table if exists test.t +mysql> create table test.t(a int) +mysql> alter table test.t set tiflash replica 1 location labels 'rack', 'host', 'abc' + +SLEEP 15 + +mysql> insert into test.t values (1); +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t; ++---+ +| a | ++---+ +| 1 | ++---+ + +mysql> alter table test.t add date_0 DATE NULL DEFAULT '1000-01-01' +mysql> alter table test.t add date_1 DATE NULL DEFAULT '9999-12-31' +mysql> alter table test.t add date_a DATE NOT NULL + +mysql> alter table test.t add time_0 TIME NULL DEFAULT '59' +mysql> alter table test.t add time_1 TIME(6) NULL DEFAULT '-838:59:59.000000' +mysql> alter table test.t add time_2 TIME(6) NULL DEFAULT '838:59:59.000000' +mysql> alter table test.t add time_3 TIME(6) NULL DEFAULT '0' +mysql> alter table test.t add time_a TIME NOT NULL +mysql> alter table test.t add time_b TIME(6) NOT NULL + +mysql> alter table test.t add datetime_0 DATETIME(6) NULL DEFAULT '1000-01-01 00:00:00.000000' +mysql> alter table test.t add datetime_1 DATETIME(6) NULL DEFAULT '9999-12-31 23:59:59.000000' +mysql> alter table test.t add datetime_a DATETIME NOT NULL +mysql> alter table test.t add datetime_b DATETIME(6) NOT NULL + +SLEEP 15 + +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t; ++------+------------+------------+------------+----------+-------------------+------------------+-----------------+----------+-----------------+----------------------------+----------------------------+---------------------+----------------------------+ +| a | date_0 | date_1 | date_a | time_0 | time_1 | time_2 | time_3 | time_a | time_b | datetime_0 | datetime_1 | datetime_a | datetime_b | ++------+------------+------------+------------+----------+-------------------+------------------+-----------------+----------+-----------------+----------------------------+----------------------------+---------------------+----------------------------+ +| 1 | 1000-01-01 | 9999-12-31 | 0000-00-00 | 00:00:59 | -838:59:59.000000 | 838:59:59.000000 | 00:00:00.000000 | 00:00:00 | 00:00:00.000000 | 1000-01-01 00:00:00.000000 | 9999-12-31 23:59:59.000000 | 0000-00-00 00:00:00 | 0000-00-00 00:00:00.000000 | ++------+------------+------------+------------+----------+-------------------+------------------+-----------------+----------+-----------------+----------------------------+----------------------------+---------------------+----------------------------+ + + +### tests for timestamp +mysql> drop table if exists test.t +mysql> create table test.t(a int) +mysql> alter table test.t set tiflash replica 1 location labels 'rack', 'host', 'abc' + +SLEEP 15 + +mysql> insert into test.t values (1); + +# The min, max, 0 is related to time zone of the machine +#mysql> alter table test.t add timestamp_0 TIMESTAMP(6) NULL DEFAULT '1970-01-01 00:00:01.000000' +#mysql> alter table test.t add timestamp_1 TIMESTAMP(6) NULL DEFAULT '2038-01-19 03:14:07.999999' +#mysql> alter table test.t add timestamp_2 TIMESTAMP(6) NULL DEFAULT '0000-00-00 00:00:00.000000' +#mysql> alter table test.t add timestamp_0 TIMESTAMP(6) NULL DEFAULT '2000-01-20 03:14:07.999999' +mysql> alter table test.t add timestamp_a TIMESTAMP NOT NULL +mysql> alter table test.t add timestamp_b TIMESTAMP(6) NOT NULL + +SLEEP 15 + +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t; ++------+---------------------+----------------------------+ +| a | timestamp_a | timestamp_b | ++------+---------------------+----------------------------+ +| 1 | 0000-00-00 00:00:00 | 0000-00-00 00:00:00.000000 | ++------+---------------------+----------------------------+ + +mysql> drop table if exists test.t + +### tests for year +mysql> drop table if exists test.t +mysql> create table test.t(a int) +mysql> alter table test.t set tiflash replica 1 location labels 'rack', 'host', 'abc' + +SLEEP 15 + +mysql> insert into test.t values (1); +mysql> alter table test.t add year_0 YEAR NULL DEFAULT '1901' +mysql> alter table test.t add year_1 YEAR NULL DEFAULT '2155' +mysql> alter table test.t add year_2 YEAR NULL DEFAULT '0000' +mysql> alter table test.t add year_a YEAR NOT NULL + +#TODO: fix zero value for year +#SLEEP 15 +#mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t; +#+------+--------+--------+--------+--------+ +#| a | year_0 | year_1 | year_2 | year_a | +#+------+--------+--------+--------+--------+ +#| 1 | 1901 | 2155 | 0000 | 2000 | +#+------+--------+--------+--------+--------+ + diff --git a/tests/fullstack-test/ddl/alter_decimal_default_value.test b/tests/fullstack-test/ddl/alter_decimal_default_value.test new file mode 100644 index 00000000000..f83669c2328 --- /dev/null +++ b/tests/fullstack-test/ddl/alter_decimal_default_value.test @@ -0,0 +1,40 @@ +mysql> drop table if exists test.t +mysql> create table test.t(a int) +mysql> alter table test.t set tiflash replica 1 location labels 'rack', 'host', 'abc' + +SLEEP 15 + +mysql> insert into test.t values (1); + +# Decimal32 precision:[1,9] +mysql> alter table test.t ADD COLUMN dec32_0 DECIMAL(9,0) NULL DEFAULT '999999999' +mysql> alter table test.t ADD COLUMN dec32_1 DECIMAL(9,9) NULL DEFAULT '.999999999' +mysql> alter table test.t ADD COLUMN dec32_3 DECIMAL(9,0) NULL DEFAULT '9' +mysql> alter table test.t ADD COLUMN dec32_4 DECIMAL(9,9) NULL DEFAULT '.9' +# Decimal64 precision:(9,18] +mysql> alter table test.t ADD COLUMN dec64_0 DECIMAL(18,0) NULL DEFAULT '999999999999999999' +mysql> alter table test.t ADD COLUMN dec64_1 DECIMAL(18,18) NULL DEFAULT '.999999999999999999' +mysql> alter table test.t ADD COLUMN dec64_3 DECIMAL(18,0) NULL DEFAULT '9' +mysql> alter table test.t ADD COLUMN dec64_4 DECIMAL(18,18) NULL DEFAULT '.9' +# Decimal128 precision:(18,38] +mysql> alter table test.t ADD COLUMN dec128_0 DECIMAL(38,0) NULL DEFAULT '99999999999999999999999999999999999999' +mysql> alter table test.t ADD COLUMN dec128_1 DECIMAL(38,30) NULL DEFAULT '99999999.999999999999999999999999999999' +mysql> alter table test.t ADD COLUMN dec128_3 DECIMAL(38,0) NULL DEFAULT '9' +mysql> alter table test.t ADD COLUMN dec128_4 DECIMAL(38,30) NULL DEFAULT '.9' +# Decimal256 precision:(38,65] +mysql> alter table test.t ADD COLUMN dec256_0 DECIMAL(65,0) NULL DEFAULT '99999999999999999999999999999999999999999999999999999999999999999' +mysql> alter table test.t ADD COLUMN dec256_1 DECIMAL(65,30) NULL DEFAULT '99999999999999999999999999999999999.999999999999999999999999999999' +mysql> alter table test.t ADD COLUMN dec256_3 DECIMAL(65,0) NULL DEFAULT '9' +mysql> alter table test.t ADD COLUMN dec256_4 DECIMAL(65,30) NULL DEFAULT '.9' + +SLEEP 15 + +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t; ++---+-----------+-------------+---------+-------------+--------------------+----------------------+---------+----------------------+----------------------------------------+-----------------------------------------+----------+----------------------------------+-------------------------------------------------------------------+--------------------------------------------------------------------+----------+----------------------------------+ +| a | dec32_0 | dec32_1 | dec32_3 | dec32_4 | dec64_0 | dec64_1 | dec64_3 | dec64_4 | dec128_0 | dec128_1 | dec128_3 | dec128_4 | dec256_0 | dec256_1 | dec256_3 | dec256_4 | ++---+-----------+-------------+---------+-------------+--------------------+----------------------+---------+----------------------+----------------------------------------+-----------------------------------------+----------+----------------------------------+-------------------------------------------------------------------+--------------------------------------------------------------------+----------+----------------------------------+ +| 1 | 999999999 | 0.999999999 | 9 | 0.900000000 | 999999999999999999 | 0.999999999999999999 | 9 | 0.900000000000000000 | 99999999999999999999999999999999999999 | 99999999.999999999999999999999999999999 | 9 | 0.900000000000000000000000000000 | 99999999999999999999999999999999999999999999999999999999999999999 | 99999999999999999999999999999999999.999999999999999999999999999999 | 9 | 0.900000000000000000000000000000 | ++---+-----------+-------------+---------+-------------+--------------------+----------------------+---------+----------------------+----------------------------------------+-----------------------------------------+----------+----------------------------------+-------------------------------------------------------------------+--------------------------------------------------------------------+----------+----------------------------------+ + + +mysql> drop table if exists test.t diff --git a/tests/fullstack-test/ddl/alter_default_value.test b/tests/fullstack-test/ddl/alter_default_value.test new file mode 100644 index 00000000000..74c97122f1f --- /dev/null +++ b/tests/fullstack-test/ddl/alter_default_value.test @@ -0,0 +1,21 @@ +mysql> drop table if exists test.t +mysql> create table test.t(a int) +mysql> alter table test.t set tiflash replica 1 location labels 'rack', 'host', 'abc' + +SLEEP 15 + +mysql> insert into test.t values (1); +mysql> alter table test.t add s varchar(999) default 'sss' +mysql> alter table test.t add e enum('unknown', 'male','female') default 'unknown' + +SLEEP 15 + +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t; ++---+-----+---------+ +| a | s | e | ++---+-----+---------+ +| 1 | sss | unknown | ++---+-----+---------+ + + +mysql> drop table if exists test.t diff --git a/tests/fullstack-test/expr/single_ifnull_in_predicate.test b/tests/fullstack-test/expr/single_ifnull_in_predicate.test index a3b69ace12e..ca907730ba0 100644 --- a/tests/fullstack-test/expr/single_ifnull_in_predicate.test +++ b/tests/fullstack-test/expr/single_ifnull_in_predicate.test @@ -14,3 +14,5 @@ mysql> select /*+ read_from_storage(tiflash[t]) */ c2 from test.t where ifnull(c | 123 | | 234 | +------+ + +mysql> drop table if exists test.t diff --git a/tests/fullstack-test/fault-inject/alter-table.test b/tests/fullstack-test/fault-inject/alter-table.test index d9f2fa7d1dd..c4ee5bce4c2 100644 --- a/tests/fullstack-test/fault-inject/alter-table.test +++ b/tests/fullstack-test/fault-inject/alter-table.test @@ -10,15 +10,30 @@ mysql> insert into test.t values (1, 2) SLEEP 15 +# ensure table is sync to tiflash +mysql> select table_schema,table_name,replica_count,location_labels,available from information_schema.tiflash_replica where table_schema='test' and table_name='t'; ++--------------+------------+---------------+-----------------+-----------+ +| table_schema | table_name | replica_count | location_labels | available | ++--------------+------------+---------------+-----------------+-----------+ +| test | t | 1 | rack,host,abc | 1 | ++--------------+------------+---------------+-----------------+-----------+ +>> select database,name from system.tables where database='test' and name='t'; +β”Œβ”€database─┬─name─┐ +β”‚ test β”‚ t β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”˜ >> DBGInvoke __try_flush() >> DBGInvoke __init_fail_point() + +# Process crash when rename column data is done but not for meta. +# Note that this only affect TxnMergeTree. DeltaTree don't care this since we store +# column data according to its column-id in TiDB. >> DBGInvoke __enable_fail_point(exception_between_alter_data_and_meta) mysql> alter table test.t change a c int SLEEP 15 -mysql> select /*+ read_from_storage(tiflash[t]) */ * from test.t; +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t; +---+---+ | c | b | +---+---+ diff --git a/tests/fullstack-test/fault-inject/drop-table.test b/tests/fullstack-test/fault-inject/drop-table.test index 920c081390a..52a66ba3ba2 100644 --- a/tests/fullstack-test/fault-inject/drop-table.test +++ b/tests/fullstack-test/fault-inject/drop-table.test @@ -11,18 +11,36 @@ mysql> insert into test.t values (1, 2) SLEEP 15 +# ensure table is sync to tiflash +mysql> select table_schema,table_name,replica_count,location_labels,available from information_schema.tiflash_replica where table_schema='test' and table_name='t'; ++--------------+------------+---------------+-----------------+-----------+ +| table_schema | table_name | replica_count | location_labels | available | ++--------------+------------+---------------+-----------------+-----------+ +| test | t | 1 | rack,host,abc | 1 | ++--------------+------------+---------------+-----------------+-----------+ +>> select database,name from system.tables where database='test' and name='t'; +β”Œβ”€database─┬─name─┐ +β”‚ test β”‚ t β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”˜ >> DBGInvoke __try_flush() >> DBGInvoke __init_fail_point() + +# Process crash when drop table metadata is done but not for its data dir. >> DBGInvoke __enable_fail_point(exception_between_drop_meta_and_data) mysql> truncate table test.t ; -SLEEP 15 +# sleep to wait schema synced to tiflash +SLEEP 20 + +# After restart, test.t is truncated, it is empty +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t; + + mysql> insert into test.t values (1, 1) mysql> insert into test.t values (1, 2) -mysql> select /*+ read_from_storage(tiflash[t]) */ * from test.t; - +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t; +---+---+ | a | b | +---+---+ @@ -33,11 +51,14 @@ mysql> select /*+ read_from_storage(tiflash[t]) */ * from test.t; >> DBGInvoke __enable_fail_point(exception_drop_table_during_remove_meta) mysql> truncate table test.t ; -SLEEP 15 +SLEEP 20 + +# After restart, test.t is truncated, it is empty +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t; mysql> insert into test.t values (1, 1) mysql> insert into test.t values (1, 2) -mysql> select /*+ read_from_storage(tiflash[t]) */ * from test.t; +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t; +---+---+ | a | b | diff --git a/tests/fullstack-test/fault-inject/rename-table.test b/tests/fullstack-test/fault-inject/rename-table.test new file mode 100644 index 00000000000..9b66c5b76d8 --- /dev/null +++ b/tests/fullstack-test/fault-inject/rename-table.test @@ -0,0 +1,46 @@ + +#TODO: atomic rename table +#RETURN + +mysql> drop table if exists test.t; +mysql> create table test.t(a int not null, b int not null); +mysql> alter table test.t set tiflash replica 1 location labels 'rack', 'host', 'abc' + +SLEEP 15 + + +mysql> insert into test.t values (1, 1),(1,2) + +SLEEP 15 + +# ensure table is sync to tiflash +mysql> select table_schema,table_name,replica_count,location_labels,available from information_schema.tiflash_replica where table_schema='test' and table_name='t'; ++--------------+------------+---------------+-----------------+-----------+ +| table_schema | table_name | replica_count | location_labels | available | ++--------------+------------+---------------+-----------------+-----------+ +| test | t | 1 | rack,host,abc | 1 | ++--------------+------------+---------------+-----------------+-----------+ +>> select database,name from system.tables where database='test' and name='t'; +β”Œβ”€database─┬─name─┐ +β”‚ test β”‚ t β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”˜ +>> DBGInvoke __try_flush() +>> DBGInvoke __init_fail_point() + +# Process crash when rename table data is done but not for its metadata file. +>> DBGInvoke __enable_fail_point(exception_between_rename_table_metadata_and_data) + +mysql> drop table if exists test.t_2; +mysql> rename table test.t to test.t_2 ; + +SLEEP 15 + +# After restart, test.t is rename to test.t_2, we can still read data from tiflash +>> select database,name from system.tables where database='test' and name='t'; +>> select database,name from system.tables where database='test' and name='t_2'; +β”Œβ”€database─┬─name─┐ +β”‚ test β”‚ t_2 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”˜ +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t_2; + +mysql> drop table if exists test.t diff --git a/tests/fullstack-test/sample.test b/tests/fullstack-test/sample.test index 2ed33986731..8833c44d108 100644 --- a/tests/fullstack-test/sample.test +++ b/tests/fullstack-test/sample.test @@ -1,5 +1,6 @@ mysql> drop table if exists test.t -mysql> create table test.t(s varchar(256), i int) + +mysql> create table if not exists test.t(s varchar(256), i int) mysql> alter table test.t set tiflash replica 1 location labels 'rack', 'host', 'abc'; mysql> delete from test.t mysql> insert into test.t values('Hello world', 666) @@ -12,3 +13,5 @@ SLEEP 15 β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”˜ mysql> delete from test.t + +mysql> drop table if exists test.t diff --git a/tests/run-test.py b/tests/run-test.py index 9f8029fd56d..8cdc1696a87 100644 --- a/tests/run-test.py +++ b/tests/run-test.py @@ -160,6 +160,7 @@ def on_line(self, line): if line.startswith(SLEEP_PREFIX): time.sleep(float(line[len(SLEEP_PREFIX):])) elif line.startswith(CMD_PREFIX_TIDB): + if verbose: print 'running', line if self.outputs != None and ((not self.is_mysql and not matched(self.outputs, self.matches, self.fuzz)) or (self.is_mysql and not MySQLCompare.matched(self.outputs, self.matches))): return False self.is_mysql = True diff --git a/tests/run-test.sh b/tests/run-test.sh index d09f9b009f2..3e101b0a72e 100755 --- a/tests/run-test.sh +++ b/tests/run-test.sh @@ -13,7 +13,7 @@ function run_file() local ext=${path##*.} if [ "$ext" == "test" ]; then - python2 run-test.py "$dbc" "$path" "$fuzz" "$mysql_client" "$verbose" + python run-test.py "$dbc" "$path" "$fuzz" "$mysql_client" "$verbose" else if [ "$ext" == "visual" ]; then python run-test-gen-from-visual.py "$path" "$skip_raw_test" "$verbose"