fix query fail when there is extra table id column and late materialization (#7497)

dbms/src/Storages/StorageDeltaMerge.cpp

@@ -755,7 +755,7 @@ DM::PushDownFilterPtr StorageDeltaMerge::buildPushDownFilter(const RSOperatorPtr
             columns_to_read_map.emplace(column.id, column);
 
         // The source_columns_of_analyzer should be the same as the size of table_scan_column_info
-        // The columns_to_read is a subset of table_scan_column_info, when there are generated columns.
+        // The columns_to_read is a subset of table_scan_column_info, when there are generated columns and extra table id column.
         NamesAndTypes source_columns_of_analyzer;
         source_columns_of_analyzer.reserve(table_scan_column_info.size());
         for (size_t i = 0; i < table_scan_column_info.size(); ++i)
@@ -769,6 +769,11 @@ DM::PushDownFilterPtr StorageDeltaMerge::buildPushDownFilter(const RSOperatorPtr
                 source_columns_of_analyzer.emplace_back(col_name, data_type);
                 continue;
             }
+            if (cid == EXTRA_TABLE_ID_COLUMN_ID)
+            {
+                source_columns_of_analyzer.emplace_back(EXTRA_TABLE_ID_COLUMN_NAME, EXTRA_TABLE_ID_COLUMN_TYPE);
+                continue;
+            }
             RUNTIME_CHECK_MSG(columns_to_read_map.contains(cid), "ColumnID({}) not found in columns_to_read_map", cid);
             source_columns_of_analyzer.emplace_back(columns_to_read_map.at(cid).name, columns_to_read_map.at(cid).type);
         }
@@ -846,7 +851,7 @@ DM::PushDownFilterPtr StorageDeltaMerge::buildPushDownFilter(const RSOperatorPtr
             const auto & current_names_and_types = analyzer->getCurrentInputColumns();
             for (size_t i = 0; i < table_scan_column_info.size(); ++i)
             {
-                if (table_scan_column_info[i].hasGeneratedColumnFlag())
+                if (table_scan_column_info[i].hasGeneratedColumnFlag() || table_scan_column_info[i].id == EXTRA_TABLE_ID_COLUMN_ID)
                     continue;
                 auto col = columns_to_read_map.at(table_scan_column_info[i].id);
                 RUNTIME_CHECK_MSG(col.name == current_names_and_types[i].name, "Column name mismatch, expect: {}, actual: {}", col.name, current_names_and_types[i].name);

dbms/src/Storages/StorageDeltaMerge.h

@@ -70,7 +70,6 @@ class StorageDeltaMerge
         size_t max_block_size,
         unsigned num_streams) override;
 
-
     SourceOps readSourceOps(
         PipelineExecutorStatus & exec_status_,
         const Names & column_names,

tests/fullstack-test/mpp/late_materialization_extra_table_id_column.test

@@ -0,0 +1,59 @@
+# Copyright 2023 PingCAP, Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mysql> drop table if exists test.t;
+mysql> create table test.t(id int, age int, t time, key(id)) partition by range(id) ( partition p0 values less than (100), partition p1 values less than (200), partition p2 values less than (300));
+mysql> insert into test.t values (1, 10, '700:11:11.1234'), (10, 11, '700:11:11.1234');
+mysql> insert into test.t select * from test.t;
+mysql> insert into test.t select * from test.t;
+mysql> insert into test.t select * from test.t;
+mysql> insert into test.t select * from test.t;
+mysql> insert into test.t select * from test.t;
+mysql> insert into test.t select * from test.t;
+mysql> insert into test.t select * from test.t;
+mysql> insert into test.t select * from test.t;
+mysql> insert into test.t select * from test.t;
+mysql> insert into test.t select * from test.t;
+mysql> insert into test.t select * from test.t;
+mysql> insert into test.t select * from test.t;
+mysql> insert into test.t select * from test.t;
+
+mysql> alter table test.t set tiflash replica 1;
+
+func> wait_table test t
+
+mysql> select count(*) from test.t;
++----------+
+| count(*) |
++----------+
+|    16384 |
++----------+
+
+mysql> analyze table test.t;
+
+mysql> begin; set @@session.tidb_isolation_read_engines='tiflash'; set @@session.tidb_partition_prune_mode='dynamic'; insert into test.t values (11, 10, '700:11:11.1234'), (12, 11, '710:11:11.1234'); select * from test.t where id > 10; select hour(t) as hour, sum(age) from test.t where id > 10 group by hour; commit;
++------+------+-----------+
+| id   | age  | t         |
++------+------+-----------+
+|   11 |   10 | 700:11:11 |
+|   12 |   11 | 710:11:11 |
++------+------+-----------+
++------+----------+
+| hour | sum(age) |
++------+----------+
+|  710 |       11 |
+|  700 |       10 |
++------+----------+
+
+mysql> drop table test.t;

tests/fullstack-test/mpp/late_materialization_generate_column.test

@@ -1,4 +1,4 @@
-# Copyright 2022 PingCAP, Ltd.
+# Copyright 2023 PingCAP, Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.