From 4b427cf13de06d0ba2655654736b7c3e126d3d6f Mon Sep 17 00:00:00 2001 From: Lloyd-Pottiger Date: Thu, 18 May 2023 15:34:59 +0800 Subject: [PATCH 1/3] fix query fail when there is extra table id column and late materialization is enabled Signed-off-by: Lloyd-Pottiger --- dbms/src/Storages/StorageDeltaMerge.cpp | 9 ++- dbms/src/Storages/StorageDeltaMerge.h | 1 - ...materialization_extra_table_id_column.test | 59 +++++++++++++++++++ 3 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 tests/fullstack-test/mpp/late_materialization_extra_table_id_column.test diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index d05e7e314d5..cad2187fe8a 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -755,7 +755,7 @@ DM::PushDownFilterPtr StorageDeltaMerge::buildPushDownFilter(const RSOperatorPtr columns_to_read_map.emplace(column.id, column); // The source_columns_of_analyzer should be the same as the size of table_scan_column_info - // The columns_to_read is a subset of table_scan_column_info, when there are generated columns. + // The columns_to_read is a subset of table_scan_column_info, when there are generated columns and extra table id column. NamesAndTypes source_columns_of_analyzer; source_columns_of_analyzer.reserve(table_scan_column_info.size()); for (size_t i = 0; i < table_scan_column_info.size(); ++i) @@ -769,6 +769,11 @@ DM::PushDownFilterPtr StorageDeltaMerge::buildPushDownFilter(const RSOperatorPtr source_columns_of_analyzer.emplace_back(col_name, data_type); continue; } + if (cid == EXTRA_TABLE_ID_COLUMN_ID) + { + source_columns_of_analyzer.emplace_back(EXTRA_TABLE_ID_COLUMN_NAME, MutableSupport::extra_table_id_column_type); + continue; + } RUNTIME_CHECK_MSG(columns_to_read_map.contains(cid), "ColumnID({}) not found in columns_to_read_map", cid); source_columns_of_analyzer.emplace_back(columns_to_read_map.at(cid).name, columns_to_read_map.at(cid).type); } @@ -846,7 +851,7 @@ DM::PushDownFilterPtr StorageDeltaMerge::buildPushDownFilter(const RSOperatorPtr const auto & current_names_and_types = analyzer->getCurrentInputColumns(); for (size_t i = 0; i < table_scan_column_info.size(); ++i) { - if (table_scan_column_info[i].hasGeneratedColumnFlag()) + if (table_scan_column_info[i].hasGeneratedColumnFlag() || table_scan_column_info[i].id == EXTRA_TABLE_ID_COLUMN_ID) continue; auto col = columns_to_read_map.at(table_scan_column_info[i].id); RUNTIME_CHECK_MSG(col.name == current_names_and_types[i].name, "Column name mismatch, expect: {}, actual: {}", col.name, current_names_and_types[i].name); diff --git a/dbms/src/Storages/StorageDeltaMerge.h b/dbms/src/Storages/StorageDeltaMerge.h index d642e622c43..e22aafa14a9 100644 --- a/dbms/src/Storages/StorageDeltaMerge.h +++ b/dbms/src/Storages/StorageDeltaMerge.h @@ -70,7 +70,6 @@ class StorageDeltaMerge size_t max_block_size, unsigned num_streams) override; - SourceOps readSourceOps( PipelineExecutorStatus & exec_status_, const Names & column_names, diff --git a/tests/fullstack-test/mpp/late_materialization_extra_table_id_column.test b/tests/fullstack-test/mpp/late_materialization_extra_table_id_column.test new file mode 100644 index 00000000000..c7776f0b5a4 --- /dev/null +++ b/tests/fullstack-test/mpp/late_materialization_extra_table_id_column.test @@ -0,0 +1,59 @@ +# Copyright 2022 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mysql> drop table if exists test.t; +mysql> create table test.t(id int, age int, t time, key(id)) partition by range(id) ( partition p0 values less than (100), partition p1 values less than (200), partition p2 values less than (300)); +mysql> insert into test.t values (1, 10, '700:11:11.1234'), (10, 11, '700:11:11.1234'); +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; +mysql> insert into test.t select * from test.t; + +mysql> alter table test.t set tiflash replica 1; + +func> wait_table test t + +mysql> select count(*) from test.t; ++----------+ +| count(*) | ++----------+ +| 16384 | ++----------+ + +mysql> analyze table test.t; + +mysql> begin; set @@session.tidb_isolation_read_engines='tiflash'; set @@session.tidb_partition_prune_mode='dynamic'; insert into test.t values (11, 10, '700:11:11.1234'), (12, 11, '710:11:11.1234'); select * from test.t where id > 10; select hour(t) as hour, sum(age) from test.t where id > 10 group by hour; commit; ++------+------+-----------+ +| id | age | t | ++------+------+-----------+ +| 11 | 10 | 700:11:11 | +| 12 | 11 | 710:11:11 | ++------+------+-----------+ ++------+----------+ +| hour | sum(age) | ++------+----------+ +| 710 | 11 | +| 700 | 10 | ++------+----------+ + +mysql> drop table test.t; From 7e3a752bcf1e5bd427dd1a16c3a550624b6308c5 Mon Sep 17 00:00:00 2001 From: Lloyd-Pottiger Date: Thu, 18 May 2023 15:43:47 +0800 Subject: [PATCH 2/3] 2022 -> 2023 Signed-off-by: Lloyd-Pottiger --- .../mpp/late_materialization_extra_table_id_column.test | 2 +- .../mpp/late_materialization_generate_column.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fullstack-test/mpp/late_materialization_extra_table_id_column.test b/tests/fullstack-test/mpp/late_materialization_extra_table_id_column.test index c7776f0b5a4..5dfcee3e753 100644 --- a/tests/fullstack-test/mpp/late_materialization_extra_table_id_column.test +++ b/tests/fullstack-test/mpp/late_materialization_extra_table_id_column.test @@ -1,4 +1,4 @@ -# Copyright 2022 PingCAP, Ltd. +# Copyright 2023 PingCAP, Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/fullstack-test/mpp/late_materialization_generate_column.test b/tests/fullstack-test/mpp/late_materialization_generate_column.test index cb027f51c1f..d6b575bda09 100644 --- a/tests/fullstack-test/mpp/late_materialization_generate_column.test +++ b/tests/fullstack-test/mpp/late_materialization_generate_column.test @@ -1,4 +1,4 @@ -# Copyright 2022 PingCAP, Ltd. +# Copyright 2023 PingCAP, Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 8fd172c391a2136e5807a2397c1cf8f808987ad6 Mon Sep 17 00:00:00 2001 From: Lloyd-Pottiger Date: Thu, 18 May 2023 17:14:42 +0800 Subject: [PATCH 3/3] refine Signed-off-by: Lloyd-Pottiger --- dbms/src/Storages/StorageDeltaMerge.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index cad2187fe8a..0db81a0701a 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -771,7 +771,7 @@ DM::PushDownFilterPtr StorageDeltaMerge::buildPushDownFilter(const RSOperatorPtr } if (cid == EXTRA_TABLE_ID_COLUMN_ID) { - source_columns_of_analyzer.emplace_back(EXTRA_TABLE_ID_COLUMN_NAME, MutableSupport::extra_table_id_column_type); + source_columns_of_analyzer.emplace_back(EXTRA_TABLE_ID_COLUMN_NAME, EXTRA_TABLE_ID_COLUMN_TYPE); continue; } RUNTIME_CHECK_MSG(columns_to_read_map.contains(cid), "ColumnID({}) not found in columns_to_read_map", cid);