From 59421fb1f76e0caa8b7d1fe0a175572d529653c3 Mon Sep 17 00:00:00 2001 From: zhaoliwei Date: Thu, 28 May 2020 19:54:53 +0800 Subject: [PATCH 1/7] fix --- src/test/function_test/test_restore.cpp | 43 +++++++++++++++++-------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/src/test/function_test/test_restore.cpp b/src/test/function_test/test_restore.cpp index eb752805be..94d97992bf 100644 --- a/src/test/function_test/test_restore.cpp +++ b/src/test/function_test/test_restore.cpp @@ -22,7 +22,6 @@ class restore_test : public testing::Test working_root_dir = global_env::instance()._working_dir; chdir(pegasus_root_dir.c_str()); - cluster_name = dsn::utils::filesystem::path_combine(pegasus_root_dir, backup_data_dir); system("pwd"); // modify the config to enable backup, and restart onebox @@ -39,9 +38,17 @@ class restore_test : public testing::Test system("./run.sh start_onebox --config_path config-server-test-restore.ini"); std::this_thread::sleep_for(std::chrono::seconds(3)); + cmd = "grep -A 5 block_service." + backup_provider_name + + " config-server-test-restore.ini | grep args | cut -f2,3 -d'/'"; + std::stringstream ss; + assert(dsn::utils::pipe_execute(cmd.c_str(), ss) == 0); + std::string provider_dir = ss.str().substr(0, ss.str().length() - 1); + policy_dir = "onebox/" + provider_dir + '/' + + dsn::utils::filesystem::path_combine(cluster_name, policy_name); + std::vector meta_list; replica_helper::load_meta_servers( - meta_list, PEGASUS_CLUSTER_SECTION_NAME.c_str(), "mycluster"); + meta_list, PEGASUS_CLUSTER_SECTION_NAME.c_str(), cluster_name.c_str()); ddl_client = std::make_shared(meta_list); error_code err = @@ -56,7 +63,8 @@ class restore_test : public testing::Test old_app_id = app_id; } ASSERT_GE(app_id, 0); - pg_client = pegasus::pegasus_client_factory::get_client("mycluster", app_name.c_str()); + pg_client = + pegasus::pegasus_client_factory::get_client(cluster_name.c_str(), app_name.c_str()); ASSERT_NE(pg_client, nullptr); write_data(); @@ -105,9 +113,16 @@ class restore_test : public testing::Test bool verify_data() { - int err = PERR_OK; std::cout << "start to get " << kv_pair_cnt << " key-value pairs, using get()..." << std::endl; + new_pg_client = + pegasus::pegasus_client_factory::get_client(cluster_name.c_str(), new_app_name.c_str()); + if (nullptr == new_pg_client) { + std::cout << "error to create client for " << new_app_name << std::endl; + return false; + } + + int err = PERR_OK; int64_t start = dsn_now_ms(); for (int i = 1; i <= kv_pair_cnt; i++) { std::string index = std::to_string(i); @@ -115,10 +130,10 @@ class restore_test : public testing::Test std::string s_key = sort_key_prefix + "_" + index; std::string value = value_prefix + "_" + index; std::string value_new; - err = pg_client->get(h_key, s_key, value_new); + err = new_pg_client->get(h_key, s_key, value_new); if (err != PERR_OK) { std::cout << "get <" << h_key << ">, <" << s_key - << "> failed， with err = " << pg_client->get_error_string(err) + << "> failed， with err = " << new_pg_client->get_error_string(err) << std::endl; return false; } @@ -136,8 +151,6 @@ class restore_test : public testing::Test bool restore() { - system("./run.sh clear_onebox"); - system("./run.sh start_onebox"); std::this_thread::sleep_for(std::chrono::seconds(3)); time_stamp = get_first_backup_timestamp(); std::cout << "first backup_timestamp = " << time_stamp << std::endl; @@ -147,7 +160,7 @@ class restore_test : public testing::Test time_stamp, app_name, old_app_id, - app_name, + new_app_name, false); if (err != ERR_OK) { std::cout << "restore failed, err = " << err.to_string() << std::endl; @@ -178,9 +191,9 @@ class restore_test : public testing::Test p_confs.clear(); app_id = 0, partition_cnt = 0; - err = ddl_client->list_app(app_name, app_id, partition_cnt, p_confs); + err = ddl_client->list_app(new_app_name, app_id, partition_cnt, p_confs); if (err != ERR_OK) { - std::cout << "list app failed, app_name = " << app_name + std::cout << "list app failed, app_name = " << new_app_name << ", with err = " << err.to_string() << std::endl; continue; } @@ -232,7 +245,6 @@ class restore_test : public testing::Test int64_t get_first_backup_timestamp() { - std::string policy_dir = dsn::utils::filesystem::path_combine(cluster_name, policy_name); std::string cmd = "cd " + policy_dir + "; " "ls -c > restore_app_from_backup_test_tmp; " "tail -n 1 restore_app_from_backup_test_tmp; " @@ -256,7 +268,6 @@ class restore_test : public testing::Test bool find_second_backup_timestamp() { - std::string policy_dir = dsn::utils::filesystem::path_combine(cluster_name, policy_name); std::vector dirs; ::dsn::utils::filesystem::get_subdirectories(policy_dir, dirs, false); return (dirs.size() >= 2); @@ -264,11 +275,13 @@ class restore_test : public testing::Test public: pegasus_client *pg_client; + pegasus_client *new_pg_client; std::shared_ptr ddl_client; std::string pegasus_root_dir; std::string working_root_dir; + std::string policy_dir; - std::string cluster_name; + std::string cluster_name = "mycluster"; int32_t old_app_id; int64_t time_stamp; @@ -280,6 +293,7 @@ class restore_test : public testing::Test static const std::string backup_data_dir; static const std::string app_name; + static const std::string new_app_name; static const std::string hash_key_prefix; static const std::string sort_key_prefix; @@ -301,6 +315,7 @@ const std::string restore_test::start_time = "24:0"; const std::string restore_test::backup_data_dir = "backup_data"; const std::string restore_test::app_name = "backup_test"; +const std::string restore_test::new_app_name = "backup_test_new"; const std::string restore_test::hash_key_prefix = "hash_key"; const std::string restore_test::sort_key_prefix = "sort_key"; From 9cd4b95b0bf0bce4ea94ea47c559d2d03f8dedc5 Mon Sep 17 00:00:00 2001 From: zhaoliwei Date: Fri, 29 May 2020 09:58:01 +0800 Subject: [PATCH 2/7] fix --- src/test/function_test/test_restore.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/function_test/test_restore.cpp b/src/test/function_test/test_restore.cpp index 94d97992bf..784ce4cfa6 100644 --- a/src/test/function_test/test_restore.cpp +++ b/src/test/function_test/test_restore.cpp @@ -113,6 +113,7 @@ class restore_test : public testing::Test bool verify_data() { + int err = PERR_OK; std::cout << "start to get " << kv_pair_cnt << " key-value pairs, using get()..." << std::endl; new_pg_client = @@ -122,7 +123,6 @@ class restore_test : public testing::Test return false; } - int err = PERR_OK; int64_t start = dsn_now_ms(); for (int i = 1; i <= kv_pair_cnt; i++) { std::string index = std::to_string(i); From 7514647790e617518358b91a4ffb1d35740de199 Mon Sep 17 00:00:00 2001 From: zhaoliwei Date: Fri, 29 May 2020 10:13:09 +0800 Subject: [PATCH 3/7] fix --- src/test/function_test/test_restore.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/test/function_test/test_restore.cpp b/src/test/function_test/test_restore.cpp index 784ce4cfa6..960ba67063 100644 --- a/src/test/function_test/test_restore.cpp +++ b/src/test/function_test/test_restore.cpp @@ -19,7 +19,6 @@ class restore_test : public testing::Test virtual void SetUp() override { pegasus_root_dir = global_env::instance()._pegasus_root; - working_root_dir = global_env::instance()._working_dir; chdir(pegasus_root_dir.c_str()); system("pwd"); @@ -278,10 +277,10 @@ class restore_test : public testing::Test pegasus_client *new_pg_client; std::shared_ptr ddl_client; std::string pegasus_root_dir; - std::string working_root_dir; std::string policy_dir; - std::string cluster_name = "mycluster"; + const std::string cluster_name = "mycluster"; + const std::string new_app_name = "backup_test_new"; int32_t old_app_id; int64_t time_stamp; @@ -293,7 +292,6 @@ class restore_test : public testing::Test static const std::string backup_data_dir; static const std::string app_name; - static const std::string new_app_name; static const std::string hash_key_prefix; static const std::string sort_key_prefix; @@ -315,7 +313,6 @@ const std::string restore_test::start_time = "24:0"; const std::string restore_test::backup_data_dir = "backup_data"; const std::string restore_test::app_name = "backup_test"; -const std::string restore_test::new_app_name = "backup_test_new"; const std::string restore_test::hash_key_prefix = "hash_key"; const std::string restore_test::sort_key_prefix = "sort_key"; From 32fa632cb37aa54561c21d0bfdb44f1bc1cfaa57 Mon Sep 17 00:00:00 2001 From: zhaoliwei Date: Fri, 29 May 2020 10:27:15 +0800 Subject: [PATCH 4/7] fix --- rdsn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdsn b/rdsn index 3036256082..88dd590d94 160000 --- a/rdsn +++ b/rdsn @@ -1 +1 @@ -Subproject commit 30362560829fcc819bd37d9b0f437edb6c8fdf76 +Subproject commit 88dd590d94f6417b9f42cd322f70ab5f2add837d From 790cb2cd47b4d20e083f31e3858191e34a3fe67a Mon Sep 17 00:00:00 2001 From: zhaoliwei Date: Fri, 29 May 2020 15:53:09 +0800 Subject: [PATCH 5/7] fix --- src/test/function_test/test_restore.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/function_test/test_restore.cpp b/src/test/function_test/test_restore.cpp index 960ba67063..e89f26371e 100644 --- a/src/test/function_test/test_restore.cpp +++ b/src/test/function_test/test_restore.cpp @@ -85,7 +85,6 @@ class restore_test : public testing::Test chdir(global_env::instance()._pegasus_root.c_str()); system("./run.sh clear_onebox"); system("./run.sh start_onebox -w"); - std::string cmd = "rm -rf " + backup_data_dir; system(cmd.c_str()); chdir(global_env::instance()._working_dir.c_str()); } @@ -289,7 +288,6 @@ class restore_test : public testing::Test static const int backup_interval_seconds; static const int backup_history_count_to_keep; static const std::string start_time; - static const std::string backup_data_dir; static const std::string app_name; @@ -310,7 +308,6 @@ const std::string restore_test::backup_provider_name = "local_service"; const int restore_test::backup_interval_seconds = 1; const int restore_test::backup_history_count_to_keep = 6; const std::string restore_test::start_time = "24:0"; -const std::string restore_test::backup_data_dir = "backup_data"; const std::string restore_test::app_name = "backup_test"; From ea2091351b170ad4808feb361d5b1b2018558107 Mon Sep 17 00:00:00 2001 From: zhaoliwei Date: Fri, 29 May 2020 15:54:08 +0800 Subject: [PATCH 6/7] fix --- src/test/function_test/test_restore.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/function_test/test_restore.cpp b/src/test/function_test/test_restore.cpp index e89f26371e..4d5982a824 100644 --- a/src/test/function_test/test_restore.cpp +++ b/src/test/function_test/test_restore.cpp @@ -85,7 +85,6 @@ class restore_test : public testing::Test chdir(global_env::instance()._pegasus_root.c_str()); system("./run.sh clear_onebox"); system("./run.sh start_onebox -w"); - system(cmd.c_str()); chdir(global_env::instance()._working_dir.c_str()); } From 585c8c9192408543360ae3aa8f6c8f49c10745d6 Mon Sep 17 00:00:00 2001 From: zhaoliwei Date: Fri, 29 May 2020 16:02:30 +0800 Subject: [PATCH 7/7] fix --- src/test/function_test/test_restore.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/function_test/test_restore.cpp b/src/test/function_test/test_restore.cpp index 4d5982a824..51478419f6 100644 --- a/src/test/function_test/test_restore.cpp +++ b/src/test/function_test/test_restore.cpp @@ -37,6 +37,11 @@ class restore_test : public testing::Test system("./run.sh start_onebox --config_path config-server-test-restore.ini"); std::this_thread::sleep_for(std::chrono::seconds(3)); + // First of all, we are in the path of pegasus root, for example: /home/mi/pegasus. + // And we can get the provider_dir which actually is `block_service/local_service`, + // from config-server-test-restore.ini. + // With cluster_name = mycluster and policy_name = policy_1, we can get the absolute + // path of policy: /home/mi/pegasus/onebox/block_service/local_service/mycluster/policy_1 cmd = "grep -A 5 block_service." + backup_provider_name + " config-server-test-restore.ini | grep args | cut -f2,3 -d'/'"; std::stringstream ss;