Skip to content

Commit

Permalink
BREAKING CHANGE(backup): remove 'policy_name' from backup files' path (…
Browse files Browse the repository at this point in the history 
…#716)
  • Loading branch information
@zhangyifan27
zhangyifan27 authored Dec 30, 2020
1 parent 4f5fcb9 commit bcc0e6a
Show file tree
Hide file tree
Showing 8 changed files with 89 additions and 151 deletions.
81 changes: 22 additions & 59 deletions src/common/backup_utils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,123 +29,86 @@ const int32_t cold_backup_constant::PROGRESS_FINISHED = 1000;

namespace cold_backup {

std::string get_policy_path(const std::string &root, const std::string &policy_name)
std::string get_backup_path(const std::string &root, int64_t backup_id)
{
std::stringstream ss;
ss << root << "/" << policy_name;
return ss.str();
return root + "/" + std::to_string(backup_id);
}

std::string
get_backup_path(const std::string &root, const std::string &policy_name, int64_t backup_id)
std::string get_backup_info_file(const std::string &root, int64_t backup_id)
{
std::stringstream ss;
ss << get_policy_path(root, policy_name) << "/" << backup_id;
return ss.str();
}

std::string get_app_backup_path(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
int32_t app_id,
int64_t backup_id)
{
std::stringstream ss;
ss << get_backup_path(root, policy_name, backup_id) << "/" << app_name << "_" << app_id;
return ss.str();
return get_backup_path(root, backup_id) + "/" + cold_backup_constant::BACKUP_INFO;
}

std::string get_replica_backup_path(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
gpid pid,
int64_t backup_id)
{
std::stringstream ss;
ss << get_policy_path(root, policy_name) << "/" << backup_id << "/" << app_name << "_"
<< pid.get_app_id() << "/" << pid.get_partition_index();
return ss.str();
std::string str_app = app_name + "_" + std::to_string(pid.get_app_id());
return get_backup_path(root, backup_id) + "/" + str_app + "/" +
std::to_string(pid.get_partition_index());
}

std::string get_app_meta_backup_path(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
int32_t app_id,
int64_t backup_id)
{
std::stringstream ss;
ss << get_policy_path(root, policy_name) << "/" << backup_id << "/" << app_name << "_" << app_id
<< "/meta";
return ss.str();
std::string str_app = app_name + "_" + std::to_string(app_id);
return get_backup_path(root, backup_id) + "/" + str_app + "/meta";
}

std::string get_app_metadata_file(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
int32_t app_id,
int64_t backup_id)
{
std::stringstream ss;
ss << get_app_meta_backup_path(root, policy_name, app_name, app_id, backup_id) << "/"
<< cold_backup_constant::APP_METADATA;
return ss.str();
return get_app_meta_backup_path(root, app_name, app_id, backup_id) + "/" +
cold_backup_constant::APP_METADATA;
}

std::string get_app_backup_status_file(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
int32_t app_id,
int64_t backup_id)
{
std::stringstream ss;
ss << get_app_meta_backup_path(root, policy_name, app_name, app_id, backup_id) << "/"
<< cold_backup_constant::APP_BACKUP_STATUS;
return ss.str();
return get_app_meta_backup_path(root, app_name, app_id, backup_id) + "/" +
cold_backup_constant::APP_BACKUP_STATUS;
}

std::string get_current_chkpt_file(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
gpid pid,
int64_t backup_id)
{
std::stringstream ss;
ss << get_replica_backup_path(root, policy_name, app_name, pid, backup_id) << "/"
<< cold_backup_constant::CURRENT_CHECKPOINT;
return ss.str();
return get_replica_backup_path(root, app_name, pid, backup_id) + "/" +
cold_backup_constant::CURRENT_CHECKPOINT;
}

std::string get_remote_chkpt_dirname()
{
// here using server address as suffix of remote_chkpt_dirname
rpc_address local_address = dsn_primary_address();
std::stringstream ss;
ss << "chkpt_" << local_address.ipv4_str() << "_" << local_address.port();
return ss.str();
std::string local_address = dsn_primary_address().ipv4_str();
std::string port = std::to_string(dsn_primary_address().port());
return "chkpt_" + local_address + "_" + port;
}

std::string get_remote_chkpt_dir(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
gpid pid,
int64_t backup_id)
{
std::stringstream ss;
ss << get_replica_backup_path(root, policy_name, app_name, pid, backup_id) << "/"
<< get_remote_chkpt_dirname();
return ss.str();
return get_replica_backup_path(root, app_name, pid, backup_id) + "/" +
get_remote_chkpt_dirname();
}

std::string get_remote_chkpt_meta_file(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
gpid pid,
int64_t backup_id)
{
std::stringstream ss;
ss << get_remote_chkpt_dir(root, policy_name, app_name, pid, backup_id) << "/"
<< cold_backup_constant::BACKUP_METADATA;
return ss.str();
return get_remote_chkpt_dir(root, app_name, pid, backup_id) + "/" +
cold_backup_constant::BACKUP_METADATA;
}

} // namespace cold_backup
Expand Down
71 changes: 24 additions & 47 deletions src/common/backup_utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,19 +45,19 @@ namespace cold_backup {

// The directory structure on block service
//
// <root>/<policy_name>/<backup_id>/<appname_appid>/meta/app_metadata
// /meta/app_backup_status
// /partition_1/checkpoint@ip:port/***.sst
// /partition_1/checkpoint@ip:port/CURRENT
// /partition_1/checkpoint@ip:port/backup_metadata
// /partition_1/current_checkpoint
// <root>/<policy_name>/<backup_id>/<appname_appid>/meta/app_metadata
// /meta/app_backup_status
// /partition_1/checkpoint@ip:port/***.sst
// /partition_1/checkpoint@ip:port/CURRENT
// /partition_1/checkpoint@ip:port/backup_metadata
// /partition_1/current_checkpoint
// <root>/<policy_name>/<backup_id>/backup_info
// <root>/<backup_id>/<appname_appid>/meta/app_metadata
// /meta/app_backup_status
// /partition_1/checkpoint@ip:port/***.sst
// /partition_1/checkpoint@ip:port/CURRENT
// /partition_1/checkpoint@ip:port/backup_metadata
// /partition_1/current_checkpoint
// <root>/<backup_id>/<appname_appid>/meta/app_metadata
// /meta/app_backup_status
// /partition_1/checkpoint@ip:port/***.sst
// /partition_1/checkpoint@ip:port/CURRENT
// /partition_1/checkpoint@ip:port/backup_metadata
// /partition_1/current_checkpoint
// <root>/<backup_id>/backup_info
//

//
Expand All @@ -72,39 +72,22 @@ namespace cold_backup {
// 5, backup_info : recording the information of this backup
//

// compose the path for policy on block service
// input:
// -- root: the prefix of path
// return:
// the path: <root>/<policy_name>
std::string get_policy_path(const std::string &root, const std::string &policy_name);

// compose the path for app on block service
// input:
// -- root: the prefix of path
// return:
// the path: <root>/<policy_name>/<backup_id>
std::string
get_backup_path(const std::string &root, const std::string &policy_name, int64_t backup_id);
// the path: <root>/<backup_id>
std::string get_backup_path(const std::string &root, int64_t backup_id);

// compose the path for app on block service
// input:
// -- root: the prefix of path
// return:
// the path: <root>/<policy_name>/<backup_id>/<appname_appid>
std::string get_app_backup_path(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
int32_t app_id,
int64_t backup_id);
// return: <root>/<backup_id>/backup_info
std::string get_backup_info_file(const std::string &root, int64_t backup_id);

// compose the path for replica on block service
// input:
// -- root: the prefix of the path
// return:
// the path: <root>/<policy_name>/<backup_id>/<appname_appid>/<partition_index>
// the path: <root>/<backup_id>/<appname_appid>/<partition_index>
std::string get_replica_backup_path(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
gpid pid,
int64_t backup_id);
Expand All @@ -113,9 +96,8 @@ std::string get_replica_backup_path(const std::string &root,
// input:
// -- root: the prefix of the path
// return:
// the path: <root>/<policy_name>/<backup_id>/<appname_appid>/meta
// the path: <root>/<backup_id>/<appname_appid>/meta
std::string get_app_meta_backup_path(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
int32_t app_id,
int64_t backup_id);
Expand All @@ -125,9 +107,8 @@ std::string get_app_meta_backup_path(const std::string &root,
// -- prefix: the prefix of AP
// return:
// the AP of app meta data file:
// <root>/<policy_name>/<backup_id>/<appname_appid>/meta/app_metadata
// <root>/<backup_id>/<appname_appid>/meta/app_metadata
std::string get_app_metadata_file(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
int32_t app_id,
int64_t backup_id);
Expand All @@ -137,9 +118,8 @@ std::string get_app_metadata_file(const std::string &root,
// -- prefix: the prefix of AP
// return:
// the AP of flag-file, which represent whether the app have finished backup:
// <root>/<policy_name>/<backup_id>/<appname_appid>/meta/app_backup_status
// <root>/<backup_id>/<appname_appid>/meta/app_backup_status
std::string get_app_backup_status_file(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
int32_t app_id,
int64_t backup_id);
Expand All @@ -150,9 +130,8 @@ std::string get_app_backup_status_file(const std::string &root,
// -- pid: gpid of replica
// return:
// the AP of current checkpoint file:
// <root>/<policy_name>/<backup_id>/<appname_appid>/<partition_index>/current_checkpoint
// <root>/<backup_id>/<appname_appid>/<partition_index>/current_checkpoint
std::string get_current_chkpt_file(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
gpid pid,
int64_t backup_id);
Expand All @@ -168,9 +147,8 @@ std::string get_remote_chkpt_dirname();
// -- pid: gpid of replcia
// return:
// the AP of the checkpoint dir:
// <root>/<policy_name>/<backup_id>/<appname_appid>/<partition_index>/checkpoint@<ip:port>
// <root>/<backup_id>/<appname_appid>/<partition_index>/checkpoint@<ip:port>
std::string get_remote_chkpt_dir(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
gpid pid,
int64_t backup_id);
Expand All @@ -181,9 +159,8 @@ std::string get_remote_chkpt_dir(const std::string &root,
// -- pid: gpid of replcia
// return:
// the AP of the checkpoint file metadata:
// <root>/<policy_name>/<backup_id>/<appname_appid>/<partition_index>/checkpoint@<ip:port>/backup_metadata
// <root>/<backup_id>/<appname_appid>/<partition_index>/checkpoint@<ip:port>/backup_metadata
std::string get_remote_chkpt_meta_file(const std::string &root,
const std::string &policy_name,
const std::string &app_name,
gpid pid,
int64_t backup_id);
Expand Down
12 changes: 4 additions & 8 deletions src/meta/meta_backup_service.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@ void policy_context::start_backup_app_meta_unlocked(int32_t app_id)
dist::block_service::create_file_request create_file_req;
create_file_req.ignore_metadata = true;
create_file_req.file_name = cold_backup::get_app_metadata_file(_backup_service->backup_root(),
_policy.policy_name,
_policy.app_names.at(app_id),
app_id,
_cur_backup.backup_id);
Expand Down Expand Up @@ -183,7 +182,6 @@ void policy_context::write_backup_app_finish_flag_unlocked(int32_t app_id,
create_file_req.ignore_metadata = true;
create_file_req.file_name =
cold_backup::get_app_backup_status_file(_backup_service->backup_root(),
_policy.policy_name,
_policy.app_names.at(app_id),
app_id,
_cur_backup.backup_id);
Expand Down Expand Up @@ -295,10 +293,8 @@ void policy_context::write_backup_info_unlocked(const backup_info &b_info,

dist::block_service::create_file_request create_file_req;
create_file_req.ignore_metadata = true;
create_file_req.file_name = utils::filesystem::path_combine(
cold_backup::get_backup_path(
_backup_service->backup_root(), _policy.policy_name, b_info.backup_id),
cold_backup_constant::BACKUP_INFO);
create_file_req.file_name =
cold_backup::get_backup_info_file(_backup_service->backup_root(), b_info.backup_id);
// here we can use synchronous way coz create_file with ignored metadata is very fast
_block_service
->create_file(create_file_req,
Expand Down Expand Up @@ -903,8 +899,8 @@ void policy_context::gc_backup_info_unlocked(const backup_info &info_to_gc)
dsn::task_ptr sync_callback =
::dsn::tasking::create_task(LPC_DEFAULT_CALLBACK, &_tracker, [this, info_to_gc]() {
dist::block_service::remove_path_request req;
req.path = cold_backup::get_backup_path(
_backup_service->backup_root(), _policy.policy_name, info_to_gc.backup_id);
req.path =
cold_backup::get_backup_path(_backup_service->backup_root(), info_to_gc.backup_id);
req.recursive = true;
_block_service->remove_path(
req,
Expand Down
11 changes: 6 additions & 5 deletions src/meta/server_state_restore.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,12 @@ void server_state::sync_app_from_backup_media(
return;
}

std::string app_metadata = cold_backup::get_app_metadata_file(request.cluster_name,
request.policy_name,
request.app_name,
request.app_id,
request.time_stamp);
std::string cluster_root = request.cluster_name;
if (!request.policy_name.empty()) {
cluster_root += ("/" + request.policy_name);
}
std::string app_metadata = cold_backup::get_app_metadata_file(
cluster_root, request.app_name, request.app_id, request.time_stamp);

error_code err = ERR_OK;
block_file_ptr file_handle = nullptr;
Expand Down
Loading

0 comments on commit bcc0e6a

Please sign in to comment.