feat(bulk_load): add bulk load manager for meta (#986)

XiaoMi · Dec 24, 2021 · 982c2bd · 982c2bd
1 parent 24bf381
commit 982c2bd
Show file tree

Hide file tree

Showing 7 changed files with 202 additions and 67 deletions.
diff --git a/include/dsn/cpp/json_helper.h b/include/dsn/cpp/json_helper.h
@@ -249,6 +249,18 @@ inline bool json_decode(const JsonObject &in, std::string &str)
     return true;
 }
 
+inline void json_encode(JsonWriter &out, const error_code &err)
+{
+    const char *str = err.to_string();
+    out.String(str, strlen(str), true);
+}
+inline bool json_decode(const JsonObject &in, error_code &err)
+{
+    dverify(in.IsString());
+    err = error_code(in.GetString());
+    return true;
+}
+
 // json serialization for bool types.
 // for compatibility, we treat bool as integers, which is not this case in json standard
 inline void json_encode(JsonWriter &out, bool t) { out.Int(t ? 1 : 0); }

diff --git a/include/dsn/utility/error_code.h b/include/dsn/utility/error_code.h
@@ -130,4 +130,5 @@ DEFINE_ERR_CODE(ERR_SPLITTING)
 DEFINE_ERR_CODE(ERR_PARENT_PARTITION_MISUSED)
 DEFINE_ERR_CODE(ERR_CHILD_NOT_READY)
 DEFINE_ERR_CODE(ERR_DISK_INSUFFICIENT)
+DEFINE_ERR_CODE(ERR_RETRY_EXHAUSTED)
 } // namespace dsn
diff --git a/src/common/bulk_load.thrift b/src/common/bulk_load.thrift
@@ -195,9 +195,14 @@ struct query_bulk_load_request
 struct query_bulk_load_response
 {
     // Possible error:
+    // - ERR_OK
     // - ERR_APP_NOT_EXIST: app not exist
     // - ERR_APP_DROPPED: app has been dropped
-    // - ERR_INVALID_STATE: app is not executing bulk load
+    // - ERR_FILE_OPERATION_FAILED: local file system error
+    // - ERR_FS_INTERNAL: remote file system error
+    // - ERR_CORRUPTION: file not exist or damaged
+    // - ERR_INGESTION_FAILED: ingest failed
+    // - ERR_RETRY_EXHAUSTED: retry too many times
     1:dsn.error_code                                        err;
     2:string                                                app_name;
     3:bulk_load_status                                      app_status;
@@ -206,4 +211,5 @@ struct query_bulk_load_response
     // detailed bulk load state for each replica
     6:list<map<dsn.rpc_address, partition_bulk_load_state>> bulk_load_states;
     7:optional string                                       hint_msg;
+    8:bool                                                  is_bulk_loading;
 }