From 49f46024857043d442bc0e552b319f523f525dcb Mon Sep 17 00:00:00 2001 From: meiyi Date: Thu, 28 Nov 2024 18:45:35 +0800 Subject: [PATCH] r --- .../test_schema_change_and_compaction.groovy | 63 ++++++++ ...schema_change_and_compaction_docker.groovy | 141 ++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction.groovy create mode 100644 regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction_docker.groovy diff --git a/regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction.groovy b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction.groovy new file mode 100644 index 000000000000000..88f159987e186be --- /dev/null +++ b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction.groovy @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The cases is copied from https://github.com/trinodb/trino/tree/master +// /testing/trino-product-tests/src/main/resources/sql-tests/testcases +// and modified by Doris. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.NodeType + +suite("test_schema_change_and_compaction", "nonConcurrent") { + def getAlterTableState = { dbName, show_sql -> + def retry = 0 + sql "use ${dbName};" + while (true) { + sleep(2000) + def state = sql_return_maparray show_sql + logger.info("alter table state: ${state}") + if (state.size() > 0 && state[0].State == "FINISHED") { + return true + } + retry++ + if (retry >= 20) { + return false + } + } + return false + } + + def block_convert_historical_rowsets = { + if (isCloudMode()) { + GetDebugPoint().enableDebugPointForAllBEs("CloudSchemaChangeJob::_convert_historical_rowsets.block") + } else { + GetDebugPoint().enableDebugPointForAllBEs("SchemaChangeJob::_convert_historical_rowsets.block") + } + } + + def unblock = { + if (isCloudMode()) { + GetDebugPoint().disableDebugPointForAllBEs("CloudSchemaChangeJob::_convert_historical_rowsets.block") + } else { + GetDebugPoint().disableDebugPointForAllBEs("SchemaChangeJob::_convert_historical_rowsets.block") + } + } + + sql 'CREATE TABLE tbl_1 (k1 INT, k2 INT, v1 INT) unique key(k1, k2) cluster by(k2) DISTRIBUTED BY HASH(k1) BUCKETS 1 PROPERTIES ()' + sql 'INSERT INTO tbl_1 VALUES (10, 20, 30), (11, 21, 31)' + order_qt_select_1 'SELECT * FROM tbl_1' +} diff --git a/regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction_docker.groovy b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction_docker.groovy new file mode 100644 index 000000000000000..88dfc9a80174d49 --- /dev/null +++ b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction_docker.groovy @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The cases is copied from https://github.com/trinodb/trino/tree/master +// /testing/trino-product-tests/src/main/resources/sql-tests/testcases +// and modified by Doris. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.NodeType + +suite("test_schema_change_and_compaction", 'docker') { + def getAlterTableState = { dbName, show_sql -> + def retry = 0 + sql "use ${dbName};" + while (true) { + sleep(2000) + def state = sql_return_maparray show_sql + logger.info("alter table state: ${state}") + if (state.size() > 0 && state[0].State == "FINISHED") { + return true + } + retry++ + if (retry >= 20) { + return false + } + } + return false + } + + def block_convert_historical_rowsets = { + if (isCloudMode()) { + GetDebugPoint().enableDebugPointForAllBEs("CloudSchemaChangeJob::_convert_historical_rowsets.block") + } else { + GetDebugPoint().enableDebugPointForAllBEs("SchemaChangeJob::_convert_historical_rowsets.block") + } + } + + def unblock = { + if (isCloudMode()) { + GetDebugPoint().disableDebugPointForAllBEs("CloudSchemaChangeJob::_convert_historical_rowsets.block") + } else { + GetDebugPoint().disableDebugPointForAllBEs("SchemaChangeJob::_convert_historical_rowsets.block") + } + } + + def options = new ClusterOptions() + options.setFeNum(1) + options.enableDebugPoints() + // options.feConfigs.add('publish_wait_time_second=-1') + // options.feConfigs.add('commit_timeout_second=10') + // options.feConfigs.add('sys_log_verbose_modules=org.apache.doris') + // options.beConfigs.add('sys_log_verbose_modules=*') + options.beConfigs.add('enable_java_support=false') + docker(options) { + // ---------- test restart fe ---------- + def result = sql 'SELECT DATABASE()' + def dbName = result[0][0] + + sql 'CREATE TABLE tbl_1 (k1 INT, k2 INT, v1 INT) unique key(k1, k2) cluster by(k2) DISTRIBUTED BY HASH(k1) BUCKETS 1 PROPERTIES ()' + sql 'INSERT INTO tbl_1 VALUES (10, 20, 30), (11, 21, 31)' + order_qt_select_1 'SELECT * FROM tbl_1' + + sql 'CREATE TABLE tbl_2 (k1 INT, k2 INT) DISTRIBUTED BY HASH(k1) BUCKETS 2 PROPERTIES ( "light_schema_change" = "false")' + sql 'CREATE TABLE tbl_3 (k1 INT, k2 INT) DISTRIBUTED BY HASH(k1) BUCKETS 2 PROPERTIES ( "light_schema_change" = "false")' + sql 'CREATE TABLE tbl_4 (k1 INT, k2 INT, v INT SUM) AGGREGATE KEY (k1, k2) DISTRIBUTED BY HASH(k1) BUCKETS 2' + sql 'CREATE TABLE tbl_5 (k1 INT, k2 INT) DISTRIBUTED BY HASH(k1) BUCKETS 2 PROPERTIES ( "light_schema_change" = "false")' + + // stop publish, insert succ, txn is commit but not visible + cluster.injectDebugPoints(NodeType.FE, ['PublishVersionDaemon.stop_publish':null]) + + for (int i : [2, 3, 4, 5]) { + sql 'begin' + if (i != 4) { + sql """ INSERT INTO tbl_${i} SELECT * FROM tbl_1 """ + sql """ INSERT INTO tbl_${i} SELECT * FROM tbl_1 """ + } else { + sql """ INSERT INTO tbl_${i} SELECT k1, k2, 0 FROM tbl_1 """ + sql """ INSERT INTO tbl_${i} SELECT k1, k2, 1 FROM tbl_1 """ + } + sql 'commit' + order_qt_select_10 """ SELECT * FROM tbl_${i} """ + } + + result = sql_return_maparray 'SHOW PROC "/transactions"' + runningTxn = result.find { it.DbName.indexOf(dbName) >= 0 }.RunningTransactionNum as int + assertEquals(4, runningTxn) + + sql "ALTER TABLE tbl_2 ADD COLUMN k3 INT DEFAULT '-1'" + sql 'CREATE MATERIALIZED VIEW tbl_3_mv AS SELECT k1, k1 + k2 FROM tbl_3' + sql 'ALTER TABLE tbl_4 ADD ROLLUP tbl_3_r1(k1, v)' + sql 'ALTER TABLE tbl_5 ORDER BY (k2, k1)' + + sleep(5000) + def jobs = sql_return_maparray "SHOW ALTER TABLE COLUMN WHERE TableName = 'tbl_2'" + assertEquals(1, jobs.size()) + assertEquals('WAITING_TXN', jobs[0].State) + + jobs = sql_return_maparray "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName = 'tbl_3'" + assertEquals(1, jobs.size()) + assertEquals('WAITING_TXN', jobs[0].State) + + jobs = sql_return_maparray "SHOW ALTER TABLE ROLLUP WHERE TableName = 'tbl_4'" + assertEquals(1, jobs.size()) + assertEquals('WAITING_TXN', jobs[0].State) + + jobs = sql_return_maparray "SHOW ALTER TABLE COLUMN WHERE TableName = 'tbl_5'" + assertEquals(1, jobs.size()) + assertEquals('WAITING_TXN', jobs[0].State) + + cluster.restartFrontends() + sleep(30000) + context.reconnectFe() + + // should publish visible + order_qt_select_2 'SELECT k1, k2 FROM tbl_2' + order_qt_select_3 'SELECT * FROM tbl_3' + order_qt_select_3m 'SELECT k1, k1 + k2 FROM tbl_3' + order_qt_select_4 'SELECT * FROM tbl_4' + order_qt_select_4r 'SELECT k1, v FROM tbl_4' + order_qt_select_5 'SELECT k1, k2 FROM tbl_5' + + assertTrue(getAlterTableState(dbName, "SHOW ALTER TABLE COLUMN WHERE TableName = 'tbl_2'")) + assertTrue(getAlterTableState(dbName, "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName = 'tbl_3'")) + assertTrue(getAlterTableState(dbName, "SHOW ALTER TABLE ROLLUP WHERE TableName = 'tbl_4'")) + assertTrue(getAlterTableState(dbName, "SHOW ALTER TABLE COLUMN WHERE TableName = 'tbl_5'")) + } +}