From f0ba7ea952234cbceb30acbc6f78a5e558a61119 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Wed, 19 Jul 2023 16:30:34 +0200 Subject: [PATCH 1/5] Added support for new DDL actions for remove and add table partitioning --- dbms/src/TiDB/Schema/SchemaBuilder.cpp | 2 + dbms/src/TiDB/Schema/SchemaGetter.h | 7 +- .../ddl/alter_partition_by.test | 122 ++++++++++++++++++ .../ddl/remove_partitioning.test | 92 +++++++++++++ tests/run-test.sh | 12 +- 5 files changed, 231 insertions(+), 4 deletions(-) create mode 100644 tests/fullstack-test2/ddl/alter_partition_by.test create mode 100644 tests/fullstack-test2/ddl/remove_partitioning.test diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.cpp b/dbms/src/TiDB/Schema/SchemaBuilder.cpp index fd585fe3ba6..97bb972b7ec 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.cpp +++ b/dbms/src/TiDB/Schema/SchemaBuilder.cpp @@ -215,6 +215,8 @@ void SchemaBuilder::applyDiff(const SchemaDiff & diff) case SchemaActionType::DropTablePartition: case SchemaActionType::TruncateTablePartition: case SchemaActionType::ActionReorganizePartition: + case SchemaActionType::ActionRemovePartitioning: + case SchemaActionType::ActionAlterTablePartitioning: { applyPartitionDiff(diff.schema_id, diff.table_id); break; diff --git a/dbms/src/TiDB/Schema/SchemaGetter.h b/dbms/src/TiDB/Schema/SchemaGetter.h index 0a30a7b30d1..ea2b3784a49 100644 --- a/dbms/src/TiDB/Schema/SchemaGetter.h +++ b/dbms/src/TiDB/Schema/SchemaGetter.h @@ -92,12 +92,17 @@ enum class SchemaActionType : Int8 ActionReorganizePartition = 64, ActionAlterTTLInfo = 65, ActionAlterTTLRemove = 67, + ActionCreateResourceGroup = 68, + ActionAlterResourceGroup = 69, + ActionDropResourceGroup = 70, + ActionAlterTablePartitioning = 71, + ActionRemovePartitioning = 72, // If we support new type from TiDB. // MaxRecognizedType also needs to be changed. // It should always be equal to the maximum supported type + 1 - MaxRecognizedType = 68, + MaxRecognizedType = 73, }; struct AffectedOption diff --git a/tests/fullstack-test2/ddl/alter_partition_by.test b/tests/fullstack-test2/ddl/alter_partition_by.test new file mode 100644 index 00000000000..f8837ad6a72 --- /dev/null +++ b/tests/fullstack-test2/ddl/alter_partition_by.test @@ -0,0 +1,122 @@ +# Copyright 2023 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +## partition_table --> partition_table + +mysql> drop table if exists test.t; +mysql> create table test.t (a int primary key, b varchar(255), c int, key (b), key (c,b)) partition by range (a) (partition p0 values less than (1000000), partition p1M values less than (2000000)); +mysql> analyze table test.t; +mysql> alter table test.t set tiflash replica 1; + +mysql> insert into test.t values (1,"1",-1); +mysql> insert into test.t select a+1,a+1,-(a+1) from test.t; +mysql> insert into test.t select a+2,a+2,-(a+2) from test.t; +mysql> insert into test.t select a+500000,a+500000,-(a+500000) from test.t; +mysql> insert into test.t select a+1000000,a+1000000,-(a+1000000) from test.t; + +func> wait_table test t + +# check table info in tiflash +>> select tidb_database,tidb_name from system.tables where tidb_database = 'test' and tidb_name = 't' and is_tombstone = 0 +┌─tidb_database─┬─tidb_name─┐ +│ test │ t │ +│ test │ t │ +└───────────────┴───────────┘ + +mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p0); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> show warnings; +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p0); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> show warnings; +mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p1M); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p1M); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> show warnings; + +mysql> set @@global.tidb_enable_alter_partition_by=1; +mysql> alter table test.t partition by range (a) (partition p0 values less than (500000), partition p500k values less than (1000000), partition p1M values less than (2000000)); + +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p0); ++----------+ +| count(*) | ++----------+ +| 4 | ++----------+ + +mysql> show warnings; + +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p500k); ++----------+ +| count(*) | ++----------+ +| 4 | ++----------+ + +mysql> show warnings; + +mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p0); ++----------+ +| count(*) | ++----------+ +| 4 | ++----------+ + +mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p500k); ++----------+ +| count(*) | ++----------+ +| 4 | ++----------+ + +mysql> show warnings; + +mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p1M); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p1M); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> drop table test.t; + diff --git a/tests/fullstack-test2/ddl/remove_partitioning.test b/tests/fullstack-test2/ddl/remove_partitioning.test new file mode 100644 index 00000000000..f05cd58eaa4 --- /dev/null +++ b/tests/fullstack-test2/ddl/remove_partitioning.test @@ -0,0 +1,92 @@ +# Copyright 2023 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +## partition_table --> partition_table + +mysql> drop table if exists test.t; +mysql> create table test.t (a int primary key, b varchar(255), c int, key (b), key (c,b)) partition by range (a) (partition p0 values less than (1000000), partition p1M values less than (2000000)); +mysql> analyze table test.t; +mysql> alter table test.t set tiflash replica 1; + +mysql> insert into test.t values (1,"1",-1); +mysql> insert into test.t select a+1,a+1,-(a+1) from test.t; +mysql> insert into test.t select a+2,a+2,-(a+2) from test.t; +mysql> insert into test.t select a+500000,a+500000,-(a+500000) from test.t; +mysql> insert into test.t select a+1000000,a+1000000,-(a+1000000) from test.t; + +func> wait_table test t + +mysql> set @@global.tidb_allow_mpp = 0; +# check table info in tiflash +>> select tidb_database,tidb_name from system.tables where tidb_database = 'test' and tidb_name = 't' and is_tombstone = 0 +┌─tidb_database─┬─tidb_name─┐ +│ test │ t │ +│ test │ t │ +└───────────────┴───────────┘ + +mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p0); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> show warnings; +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p0); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> show warnings; +mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p1M); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p1M); ++----------+ +| count(*) | ++----------+ +| 8 | ++----------+ + +mysql> show warnings; + +mysql> set @@global.tidb_enable_remove_partitioning = 1; +mysql> alter table test.t remove partitioning; + +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t; ++----------+ +| count(*) | ++----------+ +| 16 | ++----------+ + +mysql> show warnings; + +mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t; ++----------+ +| count(*) | ++----------+ +| 16 | ++----------+ + +mysql> show warnings; + +mysql> drop table test.t; diff --git a/tests/run-test.sh b/tests/run-test.sh index 4e04d374ca0..bf3427dd365 100755 --- a/tests/run-test.sh +++ b/tests/run-test.sh @@ -35,9 +35,15 @@ function get_elapse_s() # Another way, the time part may start with 0, which means # it will be regarded as oct format, use "10#" to ensure # calculateing with decimal - if [ "$end_nanos" -lt "$start_nanos" ];then - end_s=$(( 10#$end_s - 1 )) - end_nanos=$(( 10#$end_nanos + 10**9 )) + if [ "$end_nanos" = "N" -a "N" = "$start_nanos" ];then + # MacOS does not support '%N' output_fmt in date... + end_nanos=0 + start_nanos=0 + else + if [ "$end_nanos" -lt "$start_nanos" ];then + end_s=$(( 10#$end_s - 1 )) + end_nanos=$(( 10#$end_nanos + 10**9 )) + fi fi elapse_s=$(( 10#$end_s - 10#$start_s )).`printf "%03d\n" $(( (10#$end_nanos - 10#$start_nanos)/10**6 ))` From 43d77782d1b64a6fa28fc308a92d8bbcc8838e9a Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Wed, 19 Jul 2023 18:16:05 +0200 Subject: [PATCH 2/5] Updated remove and add partitioning logic --- dbms/src/TiDB/Schema/SchemaBuilder.cpp | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.cpp b/dbms/src/TiDB/Schema/SchemaBuilder.cpp index 97bb972b7ec..35cd436202f 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.cpp +++ b/dbms/src/TiDB/Schema/SchemaBuilder.cpp @@ -215,12 +215,28 @@ void SchemaBuilder::applyDiff(const SchemaDiff & diff) case SchemaActionType::DropTablePartition: case SchemaActionType::TruncateTablePartition: case SchemaActionType::ActionReorganizePartition: - case SchemaActionType::ActionRemovePartitioning: - case SchemaActionType::ActionAlterTablePartitioning: { applyPartitionDiff(diff.schema_id, diff.table_id); break; } + case SchemaActionType::ActionAlterTablePartitioning: + { + if (diff.table_id == diff.old_table_id) + { + applyPartitionDiff(diff.schema_id, diff.table_id); + } + else + { + applyCreateTable(diff.schema_id, diff.table_id); + } + break; + } + case SchemaActionType::ActionRemovePartitioning: + { + /// The new non-partitioned table will have a new id + applyCreateTable(diff.schema_id, diff.table_id); + break; + } case SchemaActionType::ExchangeTablePartition: { applyExchangeTablePartition(diff); From 9ef312a89d5a51dfdcb2e00bb07a3d6636277c5a Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Thu, 3 Aug 2023 14:23:26 +0200 Subject: [PATCH 3/5] Fixed bug with non-cleaned table definitions --- dbms/src/TiDB/Schema/SchemaBuilder.cpp | 10 ++++------ tests/fullstack-test2/ddl/alter_partition_by.test | 3 --- tests/fullstack-test2/ddl/remove_partitioning.test | 2 -- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.cpp b/dbms/src/TiDB/Schema/SchemaBuilder.cpp index 35cd436202f..9abe30cbb98 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.cpp +++ b/dbms/src/TiDB/Schema/SchemaBuilder.cpp @@ -220,23 +220,21 @@ void SchemaBuilder::applyDiff(const SchemaDiff & diff) break; } case SchemaActionType::ActionAlterTablePartitioning: + case SchemaActionType::ActionRemovePartitioning: { if (diff.table_id == diff.old_table_id) { + /// Only internal additions of new partitions applyPartitionDiff(diff.schema_id, diff.table_id); } else { + /// The new non-partitioned table will have a new id + applyDropTable(diff.schema_id, diff.old_table_id); applyCreateTable(diff.schema_id, diff.table_id); } break; } - case SchemaActionType::ActionRemovePartitioning: - { - /// The new non-partitioned table will have a new id - applyCreateTable(diff.schema_id, diff.table_id); - break; - } case SchemaActionType::ExchangeTablePartition: { applyExchangeTablePartition(diff); diff --git a/tests/fullstack-test2/ddl/alter_partition_by.test b/tests/fullstack-test2/ddl/alter_partition_by.test index f8837ad6a72..37c1586cd08 100644 --- a/tests/fullstack-test2/ddl/alter_partition_by.test +++ b/tests/fullstack-test2/ddl/alter_partition_by.test @@ -32,7 +32,6 @@ func> wait_table test t >> select tidb_database,tidb_name from system.tables where tidb_database = 'test' and tidb_name = 't' and is_tombstone = 0 ┌─tidb_database─┬─tidb_name─┐ │ test │ t │ -│ test │ t │ └───────────────┴───────────┘ mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p0); @@ -67,7 +66,6 @@ mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partitio mysql> show warnings; -mysql> set @@global.tidb_enable_alter_partition_by=1; mysql> alter table test.t partition by range (a) (partition p0 values less than (500000), partition p500k values less than (1000000), partition p1M values less than (2000000)); mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p0); @@ -119,4 +117,3 @@ mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partitio +----------+ mysql> drop table test.t; - diff --git a/tests/fullstack-test2/ddl/remove_partitioning.test b/tests/fullstack-test2/ddl/remove_partitioning.test index f05cd58eaa4..b293de90ea4 100644 --- a/tests/fullstack-test2/ddl/remove_partitioning.test +++ b/tests/fullstack-test2/ddl/remove_partitioning.test @@ -33,7 +33,6 @@ mysql> set @@global.tidb_allow_mpp = 0; >> select tidb_database,tidb_name from system.tables where tidb_database = 'test' and tidb_name = 't' and is_tombstone = 0 ┌─tidb_database─┬─tidb_name─┐ │ test │ t │ -│ test │ t │ └───────────────┴───────────┘ mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p0); @@ -68,7 +67,6 @@ mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partitio mysql> show warnings; -mysql> set @@global.tidb_enable_remove_partitioning = 1; mysql> alter table test.t remove partitioning; mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t; From a7e381ed349f1e4f4403502ea204bd5ca600f8ba Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Tue, 29 Aug 2023 11:48:55 +0200 Subject: [PATCH 4/5] Added test for non-partitioned -> partitioned --- .../ddl/alter_partition_by.test | 77 ++++++++++++++++++- .../ddl/remove_partitioning.test | 2 +- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/tests/fullstack-test2/ddl/alter_partition_by.test b/tests/fullstack-test2/ddl/alter_partition_by.test index 37c1586cd08..3a53104d19c 100644 --- a/tests/fullstack-test2/ddl/alter_partition_by.test +++ b/tests/fullstack-test2/ddl/alter_partition_by.test @@ -15,7 +15,7 @@ ## partition_table --> partition_table -mysql> drop table if exists test.t; +mysql> drop table if exists test.t, test.t2; mysql> create table test.t (a int primary key, b varchar(255), c int, key (b), key (c,b)) partition by range (a) (partition p0 values less than (1000000), partition p1M values less than (2000000)); mysql> analyze table test.t; mysql> alter table test.t set tiflash replica 1; @@ -116,4 +116,79 @@ mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partitio | 8 | +----------+ +## non-partitioned table --> partitioned table +mysql> create table test.t2 (a int primary key, b varchar(255), c int, key (b), key (c,b)); +mysql> alter table test.t2 set tiflash replica 1; +mysql> insert into test.t2 select * from test.t; +func> wait_table test t2 + mysql> drop table test.t; + +>> select tidb_database,tidb_name from system.tables where tidb_database = 'test' and tidb_name = 't2' and is_tombstone = 0 +┌─tidb_database─┬─tidb_name─┐ +│ test │ t2 │ +└───────────────┴───────────┘ + +mysql> alter table test.t2 partition by hash (a) partitions 3; +mysql> analyze table test.t2; + +mysql> explain format='brief' select /*+ READ_FROM_STORAGE(TIFLASH[t2]) */ count(*) from test.t2 partition (p0); +id estRows task access object operator info +StreamAgg 1.00 root funcs:count(Column#16)->Column#4 +└─IndexReader 1.00 root partition:p0 index:StreamAgg +└─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#16 +└─IndexFullScan 16.00 cop[tikv] table:t2, index:b(b) keep order:false + +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t2]) */ count(*) from test.t2 partition (p0); ++----------+ +| count(*) | ++----------+ +| 5 | ++----------+ + +mysql> show warnings; + +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t2]) */ count(*) from test.t2 partition (p1); ++----------+ +| count(*) | ++----------+ +| 6 | ++----------+ + +mysql> show warnings; + +mysql> select /*+ READ_FROM_STORAGE(TIKV[t2]) */ count(*) from test.t2 partition (p0); ++----------+ +| count(*) | ++----------+ +| 5 | ++----------+ + +mysql> select /*+ READ_FROM_STORAGE(TIKV[t2]) */ count(*) from test.t2 partition (p1); ++----------+ +| count(*) | ++----------+ +| 6 | ++----------+ + +mysql> show warnings; + + +mysql> select /*+ READ_FROM_STORAGE(TIKV[t2]) */ count(*) from test.t2 partition (p2); ++----------+ +| count(*) | ++----------+ +| 5 | ++----------+ + +mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t2]) */ count(*) from test.t2 partition (p2); ++----------+ +| count(*) | ++----------+ +| 5 | ++----------+ + +mysql> show warnings; + +mysql> drop table test.t2; + diff --git a/tests/fullstack-test2/ddl/remove_partitioning.test b/tests/fullstack-test2/ddl/remove_partitioning.test index b293de90ea4..3ded3b9e4f5 100644 --- a/tests/fullstack-test2/ddl/remove_partitioning.test +++ b/tests/fullstack-test2/ddl/remove_partitioning.test @@ -13,7 +13,7 @@ # limitations under the License. -## partition_table --> partition_table +## partition_table --> non-partitioned table mysql> drop table if exists test.t; mysql> create table test.t (a int primary key, b varchar(255), c int, key (b), key (c,b)) partition by range (a) (partition p0 values less than (1000000), partition p1M values less than (2000000)); From 4c7ae948d11b2c09c314478218ab43e8b3c46902 Mon Sep 17 00:00:00 2001 From: JaySon Date: Mon, 11 Sep 2023 15:53:19 +0800 Subject: [PATCH 5/5] Remove tidb global settings --- tests/fullstack-test2/ddl/remove_partitioning.test | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/fullstack-test2/ddl/remove_partitioning.test b/tests/fullstack-test2/ddl/remove_partitioning.test index 3ded3b9e4f5..2f8f9574be8 100644 --- a/tests/fullstack-test2/ddl/remove_partitioning.test +++ b/tests/fullstack-test2/ddl/remove_partitioning.test @@ -28,7 +28,6 @@ mysql> insert into test.t select a+1000000,a+1000000,-(a+1000000) from test.t; func> wait_table test t -mysql> set @@global.tidb_allow_mpp = 0; # check table info in tiflash >> select tidb_database,tidb_name from system.tables where tidb_database = 'test' and tidb_name = 't' and is_tombstone = 0 ┌─tidb_database─┬─tidb_name─┐