From 635e2b0c32400fa55423f4a81df452937f221cac Mon Sep 17 00:00:00 2001 From: hehechen Date: Mon, 6 Dec 2021 20:55:55 +0800 Subject: [PATCH] add UT for schema update and segment stat (#3471) --- .../tests/gtest_dm_delta_merge_store.cpp | 150 ++++++++++++++++++ .../tests/gtest_dm_storage_delta_merge.cpp | 8 + 2 files changed, 158 insertions(+) diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp index 7d939cda423..df4250a6c73 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp @@ -2020,6 +2020,156 @@ try } CATCH +TEST_P(DeltaMergeStore_RWTest, DDLAddColumnInt8) +try +{ + const String col_name_to_add = "Int8"; + const ColId col_id_to_add = 2; + const DataTypePtr col_type_to_add = DataTypeFactory::instance().get("Int8"); + + // write some rows before DDL + size_t num_rows_write = 1; + { + Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); + store->write(*db_context, db_context->getSettingsRef(), std::move(block)); + } + + // DDL add column Int8 with default value + { + AlterCommands commands; + { + AlterCommand com; + com.type = AlterCommand::ADD_COLUMN; + com.data_type = col_type_to_add; + com.column_name = col_name_to_add; + + // mock default value + // actual ddl is like: ADD COLUMN `Int8` Int8 DEFAULT 1 + auto cast = std::make_shared(); + { + cast->name = "CAST"; + ASTPtr arg = std::make_shared(Field((Int64)1)); + cast->arguments = std::make_shared(); + cast->children.push_back(cast->arguments); + cast->arguments->children.push_back(arg); + cast->arguments->children.push_back(ASTPtr()); // dummy alias + } + com.default_expression = cast; + commands.emplace_back(std::move(com)); + } + ColumnID _col_to_add = col_id_to_add; + store->applyAlters(commands, std::nullopt, _col_to_add, *db_context); + } + + // try read + { + auto in = store->read(*db_context, + db_context->getSettingsRef(), + store->getTableColumns(), + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + /* expected_block_size= */ 1024)[0]; + + in->readPrefix(); + size_t num_rows_read = 0; + while (Block block = in->read()) + { + num_rows_read += block.rows(); + ASSERT_TRUE(block.has(col_name_to_add)); + const auto & col = block.getByName(col_name_to_add); + ASSERT_DATATYPE_EQ(col.type, col_type_to_add); + ASSERT_EQ(col.name, col_name_to_add); + for (size_t i = 0; i < block.rows(); ++i) + { + Field tmp; + col.column->get(i, tmp); + // There is some loss of precision during the convertion, so we just do a rough comparison + EXPECT_FLOAT_EQ(tmp.get(), 1); + } + } + in->readSuffix(); + ASSERT_EQ(num_rows_read, num_rows_write); + } +} +CATCH + +TEST_P(DeltaMergeStore_RWTest, DDLAddColumnUInt8) +try +{ + const String col_name_to_add = "UInt8"; + const ColId col_id_to_add = 2; + const DataTypePtr col_type_to_add = DataTypeFactory::instance().get("UInt8"); + + // write some rows before DDL + size_t num_rows_write = 1; + { + Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); + store->write(*db_context, db_context->getSettingsRef(), std::move(block)); + } + + // DDL add column UInt8 with default value + { + AlterCommands commands; + { + AlterCommand com; + com.type = AlterCommand::ADD_COLUMN; + com.data_type = col_type_to_add; + com.column_name = col_name_to_add; + + // mock default value + // actual ddl is like: ADD COLUMN `UInt8` UInt8 DEFAULT 1 + auto cast = std::make_shared(); + { + cast->name = "CAST"; + ASTPtr arg = std::make_shared(Field((UInt64)1)); + cast->arguments = std::make_shared(); + cast->children.push_back(cast->arguments); + cast->arguments->children.push_back(arg); + cast->arguments->children.push_back(ASTPtr()); // dummy alias + } + com.default_expression = cast; + commands.emplace_back(std::move(com)); + } + ColumnID _col_to_add = col_id_to_add; + store->applyAlters(commands, std::nullopt, _col_to_add, *db_context); + } + + // try read + { + auto in = store->read(*db_context, + db_context->getSettingsRef(), + store->getTableColumns(), + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + /* expected_block_size= */ 1024)[0]; + + in->readPrefix(); + size_t num_rows_read = 0; + while (Block block = in->read()) + { + num_rows_read += block.rows(); + ASSERT_TRUE(block.has(col_name_to_add)); + const auto & col = block.getByName(col_name_to_add); + ASSERT_DATATYPE_EQ(col.type, col_type_to_add); + ASSERT_EQ(col.name, col_name_to_add); + for (size_t i = 0; i < block.rows(); ++i) + { + Field tmp; + col.column->get(i, tmp); + // There is some loss of precision during the convertion, so we just do a rough comparison + EXPECT_FLOAT_EQ(tmp.get(), 1); + } + } + in->readSuffix(); + ASSERT_EQ(num_rows_read, num_rows_write); + } +} +CATCH + TEST_P(DeltaMergeStore_RWTest, DDLAddColumnDateTime) try { diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp index 8baf1ff2c98..464c7beed52 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp @@ -140,6 +140,14 @@ try EXPECT_EQ(col_value->getDataAt(i), String(DB::toString(num_rows_read))); } } + auto delta_store = storage->getStore(); + size_t total_segment_rows = 0; + auto segment_stats = delta_store->getSegmentStats(); + for (auto & stat : segment_stats) + { + total_segment_rows += stat.rows; + } + EXPECT_EQ(total_segment_rows, num_rows_read); storage->drop(); } CATCH