refactor(backup): move common utils out to backup_utils (#623)

src/common/backup_utils.cpp

@@ -0,0 +1,153 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "backup_utils.h"
+#include "replica/backup/cold_backup_context.h"
+
+namespace dsn {
+namespace replication {
+const std::string cold_backup_constant::APP_METADATA("app_metadata");
+const std::string cold_backup_constant::APP_BACKUP_STATUS("app_backup_status");
+const std::string cold_backup_constant::CURRENT_CHECKPOINT("current_checkpoint");
+const std::string cold_backup_constant::BACKUP_METADATA("backup_metadata");
+const std::string cold_backup_constant::BACKUP_INFO("backup_info");
+const int32_t cold_backup_constant::PROGRESS_FINISHED = 1000;
+
+namespace cold_backup {
+
+std::string get_policy_path(const std::string &root, const std::string &policy_name)
+{
+    std::stringstream ss;
+    ss << root << "/" << policy_name;
+    return ss.str();
+}
+
+std::string
+get_backup_path(const std::string &root, const std::string &policy_name, int64_t backup_id)
+{
+    std::stringstream ss;
+    ss << get_policy_path(root, policy_name) << "/" << backup_id;
+    return ss.str();
+}
+
+std::string get_app_backup_path(const std::string &root,
+                                const std::string &policy_name,
+                                const std::string &app_name,
+                                int32_t app_id,
+                                int64_t backup_id)
+{
+    std::stringstream ss;
+    ss << get_backup_path(root, policy_name, backup_id) << "/" << app_name << "_" << app_id;
+    return ss.str();
+}
+
+std::string get_replica_backup_path(const std::string &root,
+                                    const std::string &policy_name,
+                                    const std::string &app_name,
+                                    gpid pid,
+                                    int64_t backup_id)
+{
+    std::stringstream ss;
+    ss << get_policy_path(root, policy_name) << "/" << backup_id << "/" << app_name << "_"
+       << pid.get_app_id() << "/" << pid.get_partition_index();
+    return ss.str();
+}
+
+std::string get_app_meta_backup_path(const std::string &root,
+                                     const std::string &policy_name,
+                                     const std::string &app_name,
+                                     int32_t app_id,
+                                     int64_t backup_id)
+{
+    std::stringstream ss;
+    ss << get_policy_path(root, policy_name) << "/" << backup_id << "/" << app_name << "_" << app_id
+       << "/meta";
+    return ss.str();
+}
+
+std::string get_app_metadata_file(const std::string &root,
+                                  const std::string &policy_name,
+                                  const std::string &app_name,
+                                  int32_t app_id,
+                                  int64_t backup_id)
+{
+    std::stringstream ss;
+    ss << get_app_meta_backup_path(root, policy_name, app_name, app_id, backup_id) << "/"
+       << cold_backup_constant::APP_METADATA;
+    return ss.str();
+}
+
+std::string get_app_backup_status_file(const std::string &root,
+                                       const std::string &policy_name,
+                                       const std::string &app_name,
+                                       int32_t app_id,
+                                       int64_t backup_id)
+{
+    std::stringstream ss;
+    ss << get_app_meta_backup_path(root, policy_name, app_name, app_id, backup_id) << "/"
+       << cold_backup_constant::APP_BACKUP_STATUS;
+    return ss.str();
+}
+
+std::string get_current_chkpt_file(const std::string &root,
+                                   const std::string &policy_name,
+                                   const std::string &app_name,
+                                   gpid pid,
+                                   int64_t backup_id)
+{
+    std::stringstream ss;
+    ss << get_replica_backup_path(root, policy_name, app_name, pid, backup_id) << "/"
+       << cold_backup_constant::CURRENT_CHECKPOINT;
+    return ss.str();
+}
+
+std::string get_remote_chkpt_dirname()
+{
+    // here using server address as suffix of remote_chkpt_dirname
+    rpc_address local_address = dsn_primary_address();
+    std::stringstream ss;
+    ss << "chkpt_" << local_address.ipv4_str() << "_" << local_address.port();
+    return ss.str();
+}
+
+std::string get_remote_chkpt_dir(const std::string &root,
+                                 const std::string &policy_name,
+                                 const std::string &app_name,
+                                 gpid pid,
+                                 int64_t backup_id)
+{
+    std::stringstream ss;
+    ss << get_replica_backup_path(root, policy_name, app_name, pid, backup_id) << "/"
+       << get_remote_chkpt_dirname();
+    return ss.str();
+}
+
+std::string get_remote_chkpt_meta_file(const std::string &root,
+                                       const std::string &policy_name,
+                                       const std::string &app_name,
+                                       gpid pid,
+                                       int64_t backup_id)
+{
+    std::stringstream ss;
+    ss << get_remote_chkpt_dir(root, policy_name, app_name, pid, backup_id) << "/"
+       << cold_backup_constant::BACKUP_METADATA;
+    return ss.str();
+}
+
+} // namespace cold_backup
+} // namespace replication
+} // namespace dsn

src/common/backup_utils.h

@@ -0,0 +1,193 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+#include <dsn/tool-api/gpid.h>
+
+namespace dsn {
+namespace replication {
+
+class cold_backup_constant
+{
+public:
+    static const std::string APP_METADATA;
+    static const std::string APP_BACKUP_STATUS;
+    static const std::string CURRENT_CHECKPOINT;
+    static const std::string BACKUP_METADATA;
+    static const std::string BACKUP_INFO;
+    static const int32_t PROGRESS_FINISHED;
+};
+
+namespace cold_backup {
+
+//
+//  Attention: when compose the path on block service, we use appname_appid, because appname_appid
+//              can identify the case below:
+//     -- case: you create one app with name A and it's appid is 1, then after backup a time later,
+//              you drop the table, then create a new app with name A and with appid 3
+//              using appname_appid, can idenfity the backup data is belong to which app
+
+// The directory structure on block service
+//
+//      <root>/<policy_name>/<backup_id>/<appname_appid>/meta/app_metadata
+//                                                      /meta/app_backup_status
+//                                                      /partition_1/checkpoint@ip:port/***.sst
+//                                                      /partition_1/checkpoint@ip:port/CURRENT
+//                                                      /partition_1/checkpoint@ip:port/backup_metadata
+//                                                      /partition_1/current_checkpoint
+//      <root>/<policy_name>/<backup_id>/<appname_appid>/meta/app_metadata
+//                                                      /meta/app_backup_status
+//                                                      /partition_1/checkpoint@ip:port/***.sst
+//                                                      /partition_1/checkpoint@ip:port/CURRENT
+//                                                      /partition_1/checkpoint@ip:port/backup_metadata
+//                                                      /partition_1/current_checkpoint
+//      <root>/<policy_name>/<backup_id>/backup_info
+//
+
+//
+// the purpose of some file:
+//      1, app_metadata : the metadata of the app, the same with the app's app_info
+//      2, app_backup_status: the flag file, represent whether the app have finish backup, if this
+//         file exist on block filesystem, backup is finished, otherwise, app haven't finished
+//         backup, we ignore its context
+//      3, backup_metadata : the file to statistic the information of a checkpoint, include all the
+//         file's name, size and md5
+//      4, current_checkpoint : specifing which checkpoint directory is valid
+//      5, backup_info : recording the information of this backup
+//
+
+// compose the path for policy on block service
+// input:
+//  -- root: the prefix of path
+// return:
+//      the path: <root>/<policy_name>
+std::string get_policy_path(const std::string &root, const std::string &policy_name);
+
+// compose the path for app on block service
+// input:
+//  -- root:  the prefix of path
+// return:
+//      the path: <root>/<policy_name>/<backup_id>
+std::string
+get_backup_path(const std::string &root, const std::string &policy_name, int64_t backup_id);
+
+// compose the path for app on block service
+// input:
+//  -- root:  the prefix of path
+// return:
+//      the path: <root>/<policy_name>/<backup_id>/<appname_appid>
+std::string get_app_backup_path(const std::string &root,
+                                const std::string &policy_name,
+                                const std::string &app_name,
+                                int32_t app_id,
+                                int64_t backup_id);
+
+// compose the path for replica on block service
+// input:
+//  -- root:  the prefix of the path
+// return:
+//      the path: <root>/<policy_name>/<backup_id>/<appname_appid>/<partition_index>
+std::string get_replica_backup_path(const std::string &root,
+                                    const std::string &policy_name,
+                                    const std::string &app_name,
+                                    gpid pid,
+                                    int64_t backup_id);
+
+// compose the path for meta on block service
+// input:
+//  -- root:  the prefix of the path
+// return:
+//      the path: <root>/<policy_name>/<backup_id>/<appname_appid>/meta
+std::string get_app_meta_backup_path(const std::string &root,
+                                     const std::string &policy_name,
+                                     const std::string &app_name,
+                                     int32_t app_id,
+                                     int64_t backup_id);
+
+// compose the absolute path(AP) of app_metadata_file on block service
+// input:
+//  -- prefix:      the prefix of AP
+// return:
+//      the AP of app meta data file:
+//      <root>/<policy_name>/<backup_id>/<appname_appid>/meta/app_metadata
+std::string get_app_metadata_file(const std::string &root,
+                                  const std::string &policy_name,
+                                  const std::string &app_name,
+                                  int32_t app_id,
+                                  int64_t backup_id);
+
+// compose the absolute path(AP) of app_backup_status file on block service
+// input:
+//  -- prefix:      the prefix of AP
+// return:
+//      the AP of flag-file, which represent whether the app have finished backup:
+//      <root>/<policy_name>/<backup_id>/<appname_appid>/meta/app_backup_status
+std::string get_app_backup_status_file(const std::string &root,
+                                       const std::string &policy_name,
+                                       const std::string &app_name,
+                                       int32_t app_id,
+                                       int64_t backup_id);
+
+// compose the absolute path(AP) of current chekpoint file on block service
+// input:
+//  -- root:      the prefix of AP on block service
+//  -- pid:         gpid of replica
+// return:
+//      the AP of current checkpoint file:
+//      <root>/<policy_name>/<backup_id>/<appname_appid>/<partition_index>/current_checkpoint
+std::string get_current_chkpt_file(const std::string &root,
+                                   const std::string &policy_name,
+                                   const std::string &app_name,
+                                   gpid pid,
+                                   int64_t backup_id);
+
+// compose the checkpoint directory name on block service
+// return:
+//      checkpoint directory name: checkpoint@<ip:port>
+std::string get_remote_chkpt_dirname();
+
+// compose the absolute path(AP) of checkpoint dir for replica on block service
+// input:
+//  -- root:       the prefix of the AP
+//  -- pid:          gpid of replcia
+// return:
+//      the AP of the checkpoint dir:
+//      <root>/<policy_name>/<backup_id>/<appname_appid>/<partition_index>/checkpoint@<ip:port>
+std::string get_remote_chkpt_dir(const std::string &root,
+                                 const std::string &policy_name,
+                                 const std::string &app_name,
+                                 gpid pid,
+                                 int64_t backup_id);
+
+// compose the absolute path(AP) of checkpoint meta for replica on block service
+// input:
+//  -- root:       the prefix of the AP
+//  -- pid:          gpid of replcia
+// return:
+//      the AP of the checkpoint file metadata:
+//      <root>/<policy_name>/<backup_id>/<appname_appid>/<partition_index>/checkpoint@<ip:port>/backup_metadata
+std::string get_remote_chkpt_meta_file(const std::string &root,
+                                       const std::string &policy_name,
+                                       const std::string &app_name,
+                                       gpid pid,
+                                       int64_t backup_id);
+
+} // namespace cold_backup
+} // namespace replication
+} // namespace dsn