From ae0ec38396b2805c9230f74ce999946fdb5112c3 Mon Sep 17 00:00:00 2001 From: zhangyifan27 Date: Fri, 23 Apr 2021 15:32:39 +0800 Subject: [PATCH 1/2] feat(test): add one-time backup function tests --- rdsn | 2 +- src/test/function_test/run.sh | 2 + .../function_test/test_backup_and_restore.cpp | 247 ++++++++++++++++++ 3 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 src/test/function_test/test_backup_and_restore.cpp diff --git a/rdsn b/rdsn index 4ee361b3fc..1ff27d55d8 160000 --- a/rdsn +++ b/rdsn @@ -1 +1 @@ -Subproject commit 4ee361b3fc5de7073b95fce265c3870b926ac841 +Subproject commit 1ff27d55d8e3773a0efd911ad2b9789154a93d1a diff --git a/src/test/function_test/run.sh b/src/test/function_test/run.sh index 334b590351..f0deb5af9c 100755 --- a/src/test/function_test/run.sh +++ b/src/test/function_test/run.sh @@ -73,4 +73,6 @@ if [ $on_travis == "NO" ]; then exit_if_fail $? "run test bulk load failed: $test_case $config_file $table_name" GTEST_OUTPUT="xml:$REPORT_DIR/test_detect_hotspot.xml" GTEST_FILTER="test_detect_hotspot.*" ./$test_case $config_file $table_name exit_if_fail $? "run test test_detect_hotspot load failed: $test_case $config_file $table_name" + GTEST_OUTPUT="xml:$REPORT_DIR/backup_restore_test.xml" GTEST_FILTER="backup_restore_test.*" ./$test_case $config_file $table_name + exit_if_fail $? "run test backup_restore_test load failed: $test_case $config_file $table_name" fi diff --git a/src/test/function_test/test_backup_and_restore.cpp b/src/test/function_test/test_backup_and_restore.cpp new file mode 100644 index 0000000000..49256f48f0 --- /dev/null +++ b/src/test/function_test/test_backup_and_restore.cpp @@ -0,0 +1,247 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include + +#include "base/pegasus_const.h" +#include "global_env.h" + +using namespace ::dsn; +using namespace ::dsn::replication; +using namespace pegasus; + +class backup_restore_test : public testing::Test +{ +public: + backup_restore_test() + : _ddl_client(nullptr), + _num_of_rows(1000), + _cluster_name("onebox"), + _old_app_name("test_app"), + _new_app_name("new_app"), + _provider("local_service") + { + } + + void SetUp() override + { + // initialize root dirs + _pegasus_root_dir = global_env::instance()._pegasus_root; + _working_root_dir = global_env::instance()._working_dir; + + // modify the config to enable backup, and restart onebox + chdir(_pegasus_root_dir.c_str()); + system("./run.sh clear_onebox"); + system("cp src/server/config.min.ini config.test_backup_restore.ini"); + system("sed -i \"/^\\s*cold_backup_disabled/c cold_backup_disabled = false\" " + "config.test_backup_restore.ini"); + system("sed -i \"/^\\s*cold_backup_checkpoint_reserve_minutes/c " + "cold_backup_checkpoint_reserve_minutes = 0\" " + "config.test_backup_restore.ini"); + std::string cmd = "sed -i \"/^\\s*cold_backup_root/c cold_backup_root = " + _cluster_name; + cmd = cmd + std::string("\" config.test_backup_restore.ini"); + system(cmd.c_str()); + system("./run.sh start_onebox --config_path config.test_backup_restore.ini"); + std::this_thread::sleep_for(std::chrono::seconds(3)); + + // initialize ddl_client + std::vector meta_list; + replica_helper::load_meta_servers( + meta_list, PEGASUS_CLUSTER_SECTION_NAME.c_str(), _cluster_name.c_str()); + _ddl_client = std::make_shared(meta_list); + + // initialize _old_app_id + int32_t partition_count; + std::vector partitions; + _ddl_client->list_app(_old_app_name, _old_app_id, partition_count, partitions); + } + + void TearDown() override + { + chdir(_pegasus_root_dir.c_str()); + system("./run.sh clear_onebox"); + system("./run.sh start_onebox -w"); + chdir(_working_root_dir.c_str()); + } + + bool write_data() + { + pegasus::pegasus_client *client = pegasus::pegasus_client_factory::get_client( + _cluster_name.c_str(), _old_app_name.c_str()); + if (client == nullptr) { + std::cout << "get pegasus client failed" << std::endl; + return false; + } + + for (int i = 0; i < _num_of_rows; ++i) { + int ret = client->set("hashkey_" + std::to_string(i), + "sortkey_" + std::to_string(i), + "value_" + std::to_string(i)); + if (ret != pegasus::PERR_OK) { + std::cout << "write data failed. " << std::endl; + return false; + } + } + return true; + } + + bool verify_data(const std::string &app_name) + { + pegasus::pegasus_client *client = + pegasus::pegasus_client_factory::get_client(_cluster_name.c_str(), app_name.c_str()); + if (client == nullptr) { + std::cout << "get pegasus client failed" << std::endl; + return false; + } + + for (int i = 0; i < _num_of_rows; ++i) { + const std::string &expected_value = "value_" + std::to_string(i); + std::string value; + int ret = + client->get("hashkey_" + std::to_string(i), "sortkey_" + std::to_string(i), value); + if (ret != pegasus::PERR_OK) { + return false; + } + if (value != expected_value) { + return false; + } + } + return true; + } + + start_backup_app_response start_backup(const std::string &user_specified_path = "") + { + auto err_resp = _ddl_client->backup_app(_old_app_id, _provider, user_specified_path); + return err_resp.get_value(); + } + + query_backup_status_response query_backup(int64_t backup_id) + { + auto err_resp = _ddl_client->query_backup(_old_app_id, backup_id); + return err_resp.get_value(); + } + + error_code start_restore(int64_t backup_id, const std::string &user_specified_path = "") + { + return _ddl_client->do_restore(_provider, + _cluster_name, + /*policy_name=*/"", + backup_id, + _old_app_name, + _old_app_id, + _new_app_name, + /*skip_bad_partition=*/false, + user_specified_path); + } + + bool wait_backup_complete(int64_t backup_id, int max_sleep_seconds) + { + int sleep_sec = 0; + bool is_backup_complete = false; + while (!is_backup_complete && sleep_sec <= max_sleep_seconds) { + std::cout << "sleep 10s to wait backup complete." << std::endl; + sleep(10); + sleep_sec += 10; + + auto resp = query_backup(backup_id); + if (resp.err != ERR_OK) { + return false; + } + // we got only one backup_item for a certain app_id and backup_id. + auto item = resp.backup_items[0]; + is_backup_complete = (item.end_time_ms > 0); + } + return is_backup_complete; + } + + bool wait_app_become_healthy(const std::string &app_name, int max_sleep_seconds) + { + int sleep_sec = 0; + bool is_app_healthy = false; + while (!is_app_healthy && sleep_sec <= max_sleep_seconds) { + std::cout << "sleep 10s to wait app become healthy." << std::endl; + sleep(10); + sleep_sec += 10; + + int32_t new_app_id; + int32_t partition_count; + std::vector partitions; + auto err = _ddl_client->list_app(app_name, _old_app_id, partition_count, partitions); + if (err != ERR_OK) { + std::cout << "list app " + app_name + " failed" << std::endl; + return false; + } + int32_t healthy_partition_count = 0; + for (const auto &partition : partitions) { + if (partition.primary.is_invalid()) { + break; + } + if (partition.secondaries.size() + 1 < partition.max_replica_count) { + break; + } + healthy_partition_count++; + } + is_app_healthy = (healthy_partition_count == partition_count); + } + return is_app_healthy; + } + + void test_backup_and_restore(const std::string &user_specified_path = "") + { + error_code err = _ddl_client->create_app(_old_app_name, "pegasus", 4, 3, {}, false); + ASSERT_EQ(ERR_OK, err); + ASSERT_TRUE(wait_app_become_healthy(_old_app_name, 180)); + + ASSERT_TRUE(write_data()); + ASSERT_TRUE(verify_data(_old_app_name)); + + auto resp = start_backup(user_specified_path); + ASSERT_EQ(ERR_OK, resp.err); + int64_t backup_id = resp.backup_id; + ASSERT_TRUE(wait_backup_complete(backup_id, 180)); + err = start_restore(backup_id, user_specified_path); + ASSERT_EQ(ERR_OK, err); + ASSERT_TRUE(wait_app_become_healthy(_new_app_name, 180)); + + ASSERT_TRUE(verify_data(_new_app_name)); + } + +private: + std::shared_ptr _ddl_client; + + const int _num_of_rows; + const std::string _cluster_name; + const std::string _old_app_name; + const std::string _new_app_name; + const std::string _provider; + + std::string _pegasus_root_dir; + std::string _working_root_dir; + int32_t _old_app_id; +}; + +TEST_F(backup_restore_test, test_backup_and_restore) { test_backup_and_restore(); } + +TEST_F(backup_restore_test, test_backup_and_restore_with_user_specified_path) +{ + test_backup_and_restore("test/path"); +} From 7054b339d41d1c8abae94ee5c554c50087d335b9 Mon Sep 17 00:00:00 2001 From: zhangyifan27 Date: Mon, 26 Apr 2021 20:10:29 +0800 Subject: [PATCH 2/2] fix --- rdsn | 2 +- .../function_test/test_backup_and_restore.cpp | 34 ++++++++++--------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/rdsn b/rdsn index 1ff27d55d8..b4b9a4ecc4 160000 --- a/rdsn +++ b/rdsn @@ -1 +1 @@ -Subproject commit 1ff27d55d8e3773a0efd911ad2b9789154a93d1a +Subproject commit b4b9a4ecc499289fe651b2bd1f3ab114baa88604 diff --git a/src/test/function_test/test_backup_and_restore.cpp b/src/test/function_test/test_backup_and_restore.cpp index 49256f48f0..247b4cfd4c 100644 --- a/src/test/function_test/test_backup_and_restore.cpp +++ b/src/test/function_test/test_backup_and_restore.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -25,8 +26,8 @@ #include "base/pegasus_const.h" #include "global_env.h" -using namespace ::dsn; -using namespace ::dsn::replication; +using namespace dsn; +using namespace dsn::replication; using namespace pegasus; class backup_restore_test : public testing::Test @@ -35,6 +36,7 @@ class backup_restore_test : public testing::Test backup_restore_test() : _ddl_client(nullptr), _num_of_rows(1000), + _check_interval_sec(10), _cluster_name("onebox"), _old_app_name("test_app"), _new_app_name("new_app"), @@ -57,8 +59,9 @@ class backup_restore_test : public testing::Test system("sed -i \"/^\\s*cold_backup_checkpoint_reserve_minutes/c " "cold_backup_checkpoint_reserve_minutes = 0\" " "config.test_backup_restore.ini"); - std::string cmd = "sed -i \"/^\\s*cold_backup_root/c cold_backup_root = " + _cluster_name; - cmd = cmd + std::string("\" config.test_backup_restore.ini"); + std::string cmd = fmt::format("sed -i \"/^\\s*cold_backup_root/c cold_backup_root = {}\" " + "config.test_backup_restore.ini", + _cluster_name); system(cmd.c_str()); system("./run.sh start_onebox --config_path config.test_backup_restore.ini"); std::this_thread::sleep_for(std::chrono::seconds(3)); @@ -130,14 +133,12 @@ class backup_restore_test : public testing::Test start_backup_app_response start_backup(const std::string &user_specified_path = "") { - auto err_resp = _ddl_client->backup_app(_old_app_id, _provider, user_specified_path); - return err_resp.get_value(); + return _ddl_client->backup_app(_old_app_id, _provider, user_specified_path).get_value(); } query_backup_status_response query_backup(int64_t backup_id) { - auto err_resp = _ddl_client->query_backup(_old_app_id, backup_id); - return err_resp.get_value(); + return _ddl_client->query_backup(_old_app_id, backup_id).get_value(); } error_code start_restore(int64_t backup_id, const std::string &user_specified_path = "") @@ -158,9 +159,9 @@ class backup_restore_test : public testing::Test int sleep_sec = 0; bool is_backup_complete = false; while (!is_backup_complete && sleep_sec <= max_sleep_seconds) { - std::cout << "sleep 10s to wait backup complete." << std::endl; - sleep(10); - sleep_sec += 10; + std::cout << "sleep a while to wait backup complete." << std::endl; + sleep(_check_interval_sec); + sleep_sec += _check_interval_sec; auto resp = query_backup(backup_id); if (resp.err != ERR_OK) { @@ -173,14 +174,14 @@ class backup_restore_test : public testing::Test return is_backup_complete; } - bool wait_app_become_healthy(const std::string &app_name, int max_sleep_seconds) + bool wait_app_become_healthy(const std::string &app_name, uint32_t max_sleep_seconds) { int sleep_sec = 0; bool is_app_healthy = false; while (!is_app_healthy && sleep_sec <= max_sleep_seconds) { - std::cout << "sleep 10s to wait app become healthy." << std::endl; - sleep(10); - sleep_sec += 10; + std::cout << "sleep a while to wait app become healthy." << std::endl; + sleep(_check_interval_sec); + sleep_sec += _check_interval_sec; int32_t new_app_id; int32_t partition_count; @@ -228,7 +229,8 @@ class backup_restore_test : public testing::Test private: std::shared_ptr _ddl_client; - const int _num_of_rows; + const uint32_t _num_of_rows; + const uint8_t _check_interval_sec; const std::string _cluster_name; const std::string _old_app_name; const std::string _new_app_name;