From 210ac782e3654594277eb9da54f3e54c87eb114f Mon Sep 17 00:00:00 2001 From: w41ter Date: Tue, 31 Oct 2023 17:07:36 +0800 Subject: [PATCH] [cases](regression-test) Add backup restore operation test 1. restore overwrites an exists table 2. backup & restore with exclude table 3. restore to a new table 4. restore mix exists and new tables 5. restore with alias --- .../test_backup_restore_alias.out | 41 +++++++ .../test_backup_restore_exclude.out | 29 +++++ ..._backup_restore_multi_tables_overwrite.out | 27 +++++ .../test_backup_restore_alias.groovy | 88 +++++++++++++++ .../test_backup_restore_exclude.groovy | 103 ++++++++++++++++++ ...ckup_restore_multi_tables_overwrite.groovy | 101 +++++++++++++++++ ...st_restore_mix_exists_and_new_table.groovy | 99 +++++++++++++++++ .../test_restore_to_new_table.groovy | 86 +++++++++++++++ 8 files changed, 574 insertions(+) create mode 100644 regression-test/data/backup_restore/test_backup_restore_alias.out create mode 100644 regression-test/data/backup_restore/test_backup_restore_exclude.out create mode 100644 regression-test/data/backup_restore/test_backup_restore_multi_tables_overwrite.out create mode 100644 regression-test/suites/backup_restore/test_backup_restore_alias.groovy create mode 100644 regression-test/suites/backup_restore/test_backup_restore_exclude.groovy create mode 100644 regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy create mode 100644 regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy create mode 100644 regression-test/suites/backup_restore/test_restore_to_new_table.groovy diff --git a/regression-test/data/backup_restore/test_backup_restore_alias.out b/regression-test/data/backup_restore/test_backup_restore_alias.out new file mode 100644 index 00000000000000..a8927690586daa --- /dev/null +++ b/regression-test/data/backup_restore/test_backup_restore_alias.out @@ -0,0 +1,41 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +20 21 +123 341 + +-- !select -- +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +20 21 +123 341 + +-- !select -- +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 + diff --git a/regression-test/data/backup_restore/test_backup_restore_exclude.out b/regression-test/data/backup_restore/test_backup_restore_exclude.out new file mode 100644 index 00000000000000..1c307a75a8c83b --- /dev/null +++ b/regression-test/data/backup_restore/test_backup_restore_exclude.out @@ -0,0 +1,29 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +20 20 +21 21 + +-- !select -- +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +20 20 +21 21 + diff --git a/regression-test/data/backup_restore/test_backup_restore_multi_tables_overwrite.out b/regression-test/data/backup_restore/test_backup_restore_multi_tables_overwrite.out new file mode 100644 index 00000000000000..6ee3e7f7eae5c6 --- /dev/null +++ b/regression-test/data/backup_restore/test_backup_restore_multi_tables_overwrite.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +20 20 +21 21 + +-- !select -- +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 + diff --git a/regression-test/suites/backup_restore/test_backup_restore_alias.groovy b/regression-test/suites/backup_restore/test_backup_restore_alias.groovy new file mode 100644 index 00000000000000..532d04ed13d07c --- /dev/null +++ b/regression-test/suites/backup_restore/test_backup_restore_alias.groovy @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_restore_alias", "backup_restore") { + String repoName = "test_backup_restore_alias_repo" + String dbName = "backup_restore_alias_db" + String tableName = "test_backup_restore_alias_table" + String aliasName = "test_backup_restore_alias_table_alias" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `count` LARGEINT SUM DEFAULT "0") + AGGREGATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES + ( + "replication_num" = "1" + ) + """ + + List values = [] + for (int i = 1; i <= 10; ++i) { + values.add("(${i}, ${i})") + } + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + String snapshotName = "test_backup_restore_snapshot" + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${tableName}) + """ + + while (!syncer.checkSnapshotFinish(dbName)) { + Thread.sleep(3000) + } + + snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + sql "INSERT INTO ${dbName}.${tableName} VALUES (20, 21), (123, 341)" + qt_select "SELECT * FROM ${dbName}.${tableName} ORDER BY id" + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}` AS `${aliasName}` ) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "replication_num" = "1" + ) + """ + + while (!syncer.checkAllRestoreFinish(dbName)) { + Thread.sleep(3000) + } + + qt_select "SELECT * FROM ${dbName}.${tableName} ORDER BY id" + qt_select "SELECT * FROM ${dbName}.${aliasName} ORDER BY id" + + sql "DROP TABLE ${dbName}.${tableName} FORCE" + sql "DROP DATABASE ${dbName} FORCE" + sql "DROP REPOSITORY `${repoName}`" +} + diff --git a/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy b/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy new file mode 100644 index 00000000000000..6d248bb47ff049 --- /dev/null +++ b/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_restore_exclude", "backup_restore") { + String dbName = "backup_restore_exclude" + String suiteName = "test_backup_restore_exclude" + String repoName = "${suiteName}_repo" + String snapshotName = "${suiteName}_snapshot" + String tableNamePrefix = "${suiteName}_tables" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + + int numTables = 10; + int numRows = 10; + List tables = [] + for (int i = 0; i < numTables; ++i) { + String tableName = "${tableNamePrefix}_${i}" + tables.add(tableName) + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `count` LARGEINT SUM DEFAULT "0" + ) + AGGREGATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES + ( + "replication_num" = "1" + ) + """ + List values = [] + for (int j = 1; j <= numRows; ++j) { + values.add("(${j}, ${j})") + } + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), numRows); + } + + def backupExcludeTable = tables.removeLast(); + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + EXCLUDE (${backupExcludeTable}) + """ + + while (!syncer.checkSnapshotFinish(dbName)) { + Thread.sleep(3000) + } + + snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + // Overwrite exists table. + sql "INSERT INTO ${dbName}.${backupExcludeTable} VALUES (20, 20), (21, 21)" + qt_select "SELECT * FROM ${dbName}.${backupExcludeTable} ORDER BY id" + + def restoreExcludeTable = tables.removeLast() + sql "DROP TABLE ${dbName}.${restoreExcludeTable} FORCE" + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + EXCLUDE (${restoreExcludeTable}) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "replication_num" = "1" + ) + """ + + while (!syncer.checkAllRestoreFinish(dbName)) { + Thread.sleep(3000) + } + + qt_select "SELECT * FROM ${dbName}.${backupExcludeTable} ORDER BY id" + for (def tableName in tables) { + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), numRows); + sql "DROP TABLE ${dbName}.${tableName} FORCE" + } + + sql "DROP DATABASE ${dbName} FORCE" + sql "DROP REPOSITORY `${repoName}`" +} + diff --git a/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy b/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy new file mode 100644 index 00000000000000..e7e2a7f3fe4d4c --- /dev/null +++ b/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_restore_multi_tables_overwrite", "backup_restore") { + String dbName = "backup_restore_multi_tables_overwrite_db" + String suiteName = "test_backup_restore_multi_tables_overwrite" + String repoName = "${suiteName}_repo" + String snapshotName = "${suiteName}_snapshot" + String tableNamePrefix = "${suiteName}_tables" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + + int numTables = 10; + int numRows = 10; + List tables = [] + for (int i = 0; i < numTables; ++i) { + String tableName = "${tableNamePrefix}_${i}" + tables.add(tableName) + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `count` LARGEINT SUM DEFAULT "0" + ) + AGGREGATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES + ( + "replication_num" = "1" + ) + """ + List values = [] + for (int j = 1; j <= numRows; ++j) { + values.add("(${j}, ${j})") + } + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), numRows); + } + + def backupTables = tables[0..5] + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${backupTables.join(",")}) + """ + + while (!syncer.checkSnapshotFinish(dbName)) { + Thread.sleep(3000) + } + + snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + // Overwrite exists table. + def firstTableName = backupTables[0] + sql "INSERT INTO ${dbName}.${firstTableName} VALUES (20, 20), (21, 21)" + qt_select "SELECT * FROM ${dbName}.${firstTableName} ORDER BY id" + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON (${backupTables.join(",")}) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "replication_num" = "1" + ) + """ + + while (!syncer.checkAllRestoreFinish(dbName)) { + Thread.sleep(3000) + } + + qt_select "SELECT * FROM ${dbName}.${firstTableName} ORDER BY id" + for (def tableName in tables) { + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), numRows); + sql "DROP TABLE ${dbName}.${tableName} FORCE" + } + + sql "DROP DATABASE ${dbName} FORCE" + sql "DROP REPOSITORY `${repoName}`" +} + diff --git a/regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy b/regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy new file mode 100644 index 00000000000000..2c9b0b31aedab8 --- /dev/null +++ b/regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_restore_mix_exists_and_new_table", "backup_restore") { + String dbName = "restore_mix_exists_and_new_table" + String suiteName = "test_restore_mix_exists_and_new_table" + String repoName = "${suiteName}_repo" + String snapshotName = "${suiteName}_snapshot" + String tableNamePrefix = "${suiteName}_tables" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + + int numTables = 10; + int numRows = 10; + List tables = [] + for (int i = 0; i < numTables; ++i) { + String tableName = "${tableNamePrefix}_${i}" + tables.add(tableName) + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `count` LARGEINT SUM DEFAULT "0" + ) + AGGREGATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES + ( + "replication_num" = "1" + ) + """ + List values = [] + for (int j = 1; j <= numRows; ++j) { + values.add("(${j}, ${j})") + } + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), numRows); + } + + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${tables.join(",")}) + """ + + while (!syncer.checkSnapshotFinish(dbName)) { + Thread.sleep(3000) + } + + snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + def dropTables = tables[0..5] + for (def tableName in dropTables) { + sql "DROP TABLE ${dbName}.${tableName} FORCE" + } + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON (${tables.join(",")}) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "replication_num" = "1" + ) + """ + + while (!syncer.checkAllRestoreFinish(dbName)) { + Thread.sleep(3000) + } + + for (def tableName in tables) { + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), numRows); + sql "DROP TABLE ${dbName}.${tableName} FORCE" + } + + sql "DROP DATABASE ${dbName} FORCE" + sql "DROP REPOSITORY `${repoName}`" +} + diff --git a/regression-test/suites/backup_restore/test_restore_to_new_table.groovy b/regression-test/suites/backup_restore/test_restore_to_new_table.groovy new file mode 100644 index 00000000000000..74d2ec08bb1672 --- /dev/null +++ b/regression-test/suites/backup_restore/test_restore_to_new_table.groovy @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_restore_to_new_table", "backup_restore") { + String repoName = "test_restore_to_new_table_repo" + String dbName = "restore_to_new_table_db" + String tableName = "test_restore_to_new_table_table" + String snapshotName = "test_backup_restore_snapshot" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `count` LARGEINT SUM DEFAULT "0") + AGGREGATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES + ( + "replication_num" = "1" + ) + """ + + List values = [] + for (int i = 1; i <= 10; ++i) { + values.add("(${i}, ${i})") + } + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${tableName}) + """ + + while (!syncer.checkSnapshotFinish(dbName)) { + Thread.sleep(3000) + } + + snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + sql "DROP TABLE ${dbName}.${tableName} FORCE" + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "replication_num" = "1" + ) + """ + + while (!syncer.checkAllRestoreFinish(dbName)) { + Thread.sleep(3000) + } + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + sql "DROP TABLE ${dbName}.${tableName} FORCE" + sql "DROP DATABASE ${dbName} FORCE" + sql "DROP REPOSITORY `${repoName}`" +} +