From 27eb2edf2de251bd057b9e7b5625e8a34bd4e7de Mon Sep 17 00:00:00 2001 From: Ti Chi Robot Date: Thu, 18 May 2023 20:39:35 +0800 Subject: [PATCH] fix query fail when there is extra table id column and late materialization (#7497) (#7505) close pingcap/tiflash#7496 --- dbms/src/Storages/StorageDeltaMerge.cpp | 9 ++- dbms/src/Storages/StorageDeltaMerge.h | 1 - ...materialization_extra_table_id_column.test | 59 +++++++++++++++++++ .../late_materialization_generate_column.test | 2 +- 4 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 tests/fullstack-test/mpp/late_materialization_extra_table_id_column.test diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index d05e7e314d5..0db81a0701a 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -755,7 +755,7 @@ DM::PushDownFilterPtr StorageDeltaMerge::buildPushDownFilter(const RSOperatorPtr columns_to_read_map.emplace(column.id, column); // The source_columns_of_analyzer should be the same as the size of table_scan_column_info - // The columns_to_read is a subset of table_scan_column_info, when there are generated columns. + // The columns_to_read is a subset of table_scan_column_info, when there are generated columns and extra table id column. NamesAndTypes source_columns_of_analyzer; source_columns_of_analyzer.reserve(table_scan_column_info.size()); for (size_t i = 0; i < table_scan_column_info.size(); ++i) @@ -769,6 +769,11 @@ DM::PushDownFilterPtr StorageDeltaMerge::buildPushDownFilter(const RSOperatorPtr source_columns_of_analyzer.emplace_back(col_name, data_type); continue; } + if (cid == EXTRA_TABLE_ID_COLUMN_ID) + { + source_columns_of_analyzer.emplace_back(EXTRA_TABLE_ID_COLUMN_NAME, EXTRA_TABLE_ID_COLUMN_TYPE); + continue; + } RUNTIME_CHECK_MSG(columns_to_read_map.contains(cid), "ColumnID({}) not found in columns_to_read_map", cid); source_columns_of_analyzer.emplace_back(columns_to_read_map.at(cid).name, columns_to_read_map.at(cid).type); } @@ -846,7 +851,7 @@ DM::PushDownFilterPtr StorageDeltaMerge::buildPushDownFilter(const RSOperatorPtr const auto & current_names_and_types = analyzer->getCurrentInputColumns(); for (size_t i = 0; i < table_scan_column_info.size(); ++i) { - if (table_scan_column_info[i].hasGeneratedColumnFlag()) + if (table_scan_column_info[i].hasGeneratedColumnFlag() || table_scan_column_info[i].id == EXTRA_TABLE_ID_COLUMN_ID) continue; auto col = columns_to_read_map.at(table_scan_column_info[i].id); RUNTIME_CHECK_MSG(col.name == current_names_and_types[i].name, "Column name mismatch, expect: {}, actual: {}", col.name, current_names_and_types[i].name); diff --git a/dbms/src/Storages/StorageDeltaMerge.h b/dbms/src/Storages/StorageDeltaMerge.h index d642e622c43..e22aafa14a9 100644 --- a/dbms/src/Storages/StorageDeltaMerge.h +++ b/dbms/src/Storages/StorageDeltaMerge.h @@ -70,7 +70,6 @@ class StorageDeltaMerge size_t max_block_size, unsigned num_streams) override; - SourceOps readSourceOps( PipelineExecutorStatus & exec_status_, const Names & column_names, diff --git a/tests/fullstack-test/mpp/late_materialization_extra_table_id_column.test b/tests/fullstack-test/mpp/late_materialization_extra_table_id_column.test new file mode 100644 index 00000000000..5dfcee3e753 --- /dev/null +++ b/tests/fullstack-test/mpp/late_materialization_extra_table_id_column.test @@ -0,0 +1,59 @@ +# Copyright 2023 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mysql> drop table if exists test.t; +mysql> create table test.t(id int, age int, t time, key(id)) partition by range(id) ( partition p0 values less than (100), partition p1 values less than (200), partition p2 values less than (300)); +mysql> insert into test.t values (1, 10, '700:11:11.1234'), (10, 11, '700:11:11.1234'); +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; + +mysql> alter table test.t set tiflash replica 1; + +func> wait_table test t + +mysql> select count(*) from test.t; ++----------+ +| count(*) | ++----------+ +| 16384 | ++----------+ + +mysql> analyze table test.t; + +mysql> begin; set @@session.tidb_isolation_read_engines='tiflash'; set @@session.tidb_partition_prune_mode='dynamic'; insert into test.t values (11, 10, '700:11:11.1234'), (12, 11, '710:11:11.1234'); select * from test.t where id > 10; select hour(t) as hour, sum(age) from test.t where id > 10 group by hour; commit; ++------+------+-----------+ +| id | age | t | ++------+------+-----------+ +| 11 | 10 | 700:11:11 | +| 12 | 11 | 710:11:11 | ++------+------+-----------+ ++------+----------+ +| hour | sum(age) | ++------+----------+ +| 710 | 11 | +| 700 | 10 | ++------+----------+ + +mysql> drop table test.t; diff --git a/tests/fullstack-test/mpp/late_materialization_generate_column.test b/tests/fullstack-test/mpp/late_materialization_generate_column.test index cb027f51c1f..d6b575bda09 100644 --- a/tests/fullstack-test/mpp/late_materialization_generate_column.test +++ b/tests/fullstack-test/mpp/late_materialization_generate_column.test @@ -1,4 +1,4 @@ -# Copyright 2022 PingCAP, Ltd. +# Copyright 2023 PingCAP, Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.