Skip to content

Commit

Permalink
feat(bulk_load): add remote_file_root for start_bulk_load_request (#660)
Browse files Browse the repository at this point in the history
  • Loading branch information
hycdong authored Dec 21, 2020

Unverified

The key that signed this is expired.
1 parent df8f41e commit 3f6f95c
Showing 6 changed files with 17 additions and 7 deletions.
2 changes: 1 addition & 1 deletion rdsn
1 change: 0 additions & 1 deletion src/server/config.ini
Original file line number Diff line number Diff line change
@@ -266,7 +266,6 @@
cold_backup_root = %{cluster.name}
max_concurrent_uploading_file_count = 10

bulk_load_provider_root = bulk_load_root
max_concurrent_bulk_load_downloading_count = 5

[pegasus.server]
1 change: 0 additions & 1 deletion src/server/config.min.ini
Original file line number Diff line number Diff line change
@@ -117,7 +117,6 @@
duplication_disabled = true
cluster_name = onebox
cold_backup_checkpoint_reserve_minutes = 10
bulk_load_provider_root = bulk_load_root

[meta_server.apps.@APP_NAME@]
app_name = @APP_NAME@
15 changes: 13 additions & 2 deletions src/shell/commands/bulk_load.cpp
Original file line number Diff line number Diff line change
@@ -24,16 +24,18 @@ bool start_bulk_load(command_executor *e, shell_context *sc, arguments args)
static struct option long_options[] = {{"app_name", required_argument, 0, 'a'},
{"cluster_name", required_argument, 0, 'c'},
{"file_provider_type", required_argument, 0, 'p'},
{"root_path", required_argument, 0, 'r'},
{0, 0, 0, 0}};
std::string app_name;
std::string cluster_name;
std::string file_provider_type;
std::string remote_root_path;

optind = 0;
while (true) {
int option_index = 0;
int c;
c = getopt_long(args.argc, args.argv, "a:c:p:", long_options, &option_index);
c = getopt_long(args.argc, args.argv, "a:c:p:r:", long_options, &option_index);
if (c == -1)
break;
switch (c) {
@@ -46,6 +48,9 @@ bool start_bulk_load(command_executor *e, shell_context *sc, arguments args)
case 'p':
file_provider_type = optarg;
break;
case 'r':
remote_root_path = optarg;
break;
default:
return false;
}
@@ -65,7 +70,13 @@ bool start_bulk_load(command_executor *e, shell_context *sc, arguments args)
return false;
}

auto err_resp = sc->ddl_client->start_bulk_load(app_name, cluster_name, file_provider_type);
if (remote_root_path.empty()) {
fprintf(stderr, "remote_root_path should not be empty\n");
return false;
}

auto err_resp = sc->ddl_client->start_bulk_load(
app_name, cluster_name, file_provider_type, remote_root_path);
dsn::error_s err = err_resp.get_error();
std::string hint_msg;
if (err.is_ok()) {
3 changes: 2 additions & 1 deletion src/shell/main.cpp
Original file line number Diff line number Diff line change
@@ -460,7 +460,8 @@ static command_executor commands[] = {
{
"start_bulk_load",
"start app bulk load",
"<-a --app_name str> <-c --cluster_name str> <-p --file_provider_type str>",
"<-a --app_name str> <-c --cluster_name str> <-p --file_provider_type str> <-r "
"--root_path>",
start_bulk_load,
},
{
2 changes: 1 addition & 1 deletion src/test/function_test/test_bulk_load.cpp
Original file line number Diff line number Diff line change
@@ -103,7 +103,7 @@ class bulk_load_test : public testing::Test

error_code start_bulk_load()
{
auto err_resp = ddl_client->start_bulk_load(APP_NAME, CLUSTER, PROVIDER);
auto err_resp = ddl_client->start_bulk_load(APP_NAME, CLUSTER, PROVIDER, LOCAL_ROOT);
return err_resp.get_value().err;
}

0 comments on commit 3f6f95c

Please sign in to comment.