feat(GCS+gRPC): timeout downloads in benchmark (#7112)

googleapis · Aug 4, 2021 · ae0fbe6 · ae0fbe6
1 parent c2e8c7a
commit ae0fbe6
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 7 deletions.
diff --git a/google/cloud/storage/benchmarks/aggregate_throughput_benchmark.cc b/google/cloud/storage/benchmarks/aggregate_throughput_benchmark.cc
@@ -121,8 +121,9 @@ std::string FormatBandwidthGbPerSecond(
 
 gcs::Client MakeClient(AggregateThroughputOptions const& options) {
   auto client_options =
-      google::cloud::Options{}.set<gcs_experimental::HttpVersionOption>(
-          options.rest_http_version);
+      google::cloud::Options{}
+          .set<gcs_experimental::HttpVersionOption>(options.rest_http_version)
+          .set<gcs::DownloadStallTimeoutOption>(options.download_stall_timeout);
 #if GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
   if (options.api == ApiName::kApiGrpc) {
     auto channels = options.grpc_channel_count;
@@ -226,11 +227,11 @@ int main(int argc, char* argv[]) {
   // header because sometimes we interrupt the benchmark and these tools
   // require a header even for empty files.
   std::cout << "Labels,Iteration,ObjectCount,DatasetSize,ThreadCount"
-               ",RepeatsPerIteration,ReadSize,ReadBufferSize,Api"
-               ",GrpcChannelCount,GrpcPluginConfig,StatusCode,Peer"
-               ",BytesDownloaded,ElapsedMicroseconds,IterationBytes"
-               ",IterationElapsedMicroseconds,IterationCpuMicroseconds"
-            << std::endl;
+            << ",RepeatsPerIteration,ReadSize,ReadBufferSize,Api"
+            << ",GrpcChannelCount,GrpcPluginConfig,ClientPerThread"
+            << ",StatusCode,Peer,BytesDownloaded,ElapsedMicroseconds"
+            << ",IterationBytes,IterationElapsedMicroseconds"
+            << ",IterationCpuMicroseconds" << std::endl;
   for (int i = 0; i != options->iteration_count; ++i) {
     auto timer = Timer::PerProcess();
     std::vector<std::future<TaskResult>> tasks(configs.size());
@@ -255,6 +256,8 @@ int main(int argc, char* argv[]) {
     auto const labels = clean_csv_field(options->labels);
     auto const grpc_plugin_config =
         clean_csv_field(options->grpc_plugin_config);
+    auto const* client_per_thread =
+        options->client_per_thread ? "true" : "false";
     // Print the results after each iteration. Makes it possible to interrupt
     // the benchmark in the middle and still get some data.
     for (auto const& r : iteration_results) {
@@ -272,6 +275,7 @@ int main(int argc, char* argv[]) {
                   << ',' << ToString(options->api)          //
                   << ',' << options->grpc_channel_count     //
                   << ',' << grpc_plugin_config              //
+                  << ',' << client_per_thread               //
                   << ',' << d.status.code()                 //
                   << ',' << d.peer                          //
                   << ',' << d.bytes_downloaded              //

diff --git a/google/cloud/storage/benchmarks/aggregate_throughput_options.cc b/google/cloud/storage/benchmarks/aggregate_throughput_options.cc
@@ -83,6 +83,11 @@ ParseAggregateThroughputOptions(std::vector<std::string> const& argv,
          options.client_per_thread =
              testing_util::ParseBoolean(val).value_or("true");
        }},
+      {"--download-stall-timeout",
+       "abort downloads stalled for more than this time",
+       [&options](std::string const& val) {
+         options.download_stall_timeout = testing_util::ParseDuration(val);
+       }},
   };
   auto usage = BuildUsage(desc, argv[0]);
 

diff --git a/google/cloud/storage/benchmarks/aggregate_throughput_options.h b/google/cloud/storage/benchmarks/aggregate_throughput_options.h
@@ -38,6 +38,7 @@ struct AggregateThroughputOptions {
   std::string grpc_plugin_config;
   std::string rest_http_version;
   bool client_per_thread = false;
+  std::chrono::seconds download_stall_timeout = std::chrono::seconds(60);
   bool exit_after_parse = false;
 };
 

diff --git a/google/cloud/storage/benchmarks/aggregate_throughput_options_test.cc b/google/cloud/storage/benchmarks/aggregate_throughput_options_test.cc
@@ -39,6 +39,7 @@ TEST(AggregateThroughputOptions, Basic) {
           "--grpc-plugin-config=default",
           "--rest-http-version=1.1",
           "--client-per-thread",
+          "--download-stall-timeout=120s",
       },
       "");
   ASSERT_STATUS_OK(options);
@@ -56,6 +57,7 @@ TEST(AggregateThroughputOptions, Basic) {
   EXPECT_EQ("default", options->grpc_plugin_config);
   EXPECT_EQ("1.1", options->rest_http_version);
   EXPECT_EQ(true, options->client_per_thread);
+  EXPECT_EQ(std::chrono::seconds(120), options->download_stall_timeout);
 }
 
 TEST(AggregateThroughputOptions, Description) {