add s3.useVirtualAddressing config, default value: false

opencurve · Apr 12, 2022 · 2a84e86 · 2a84e86
1 parent 56b2ac2
commit 2a84e86
Show file tree

Hide file tree

Showing 7 changed files with 10 additions and 32 deletions.
diff --git a/conf/s3.conf b/conf/s3.conf
@@ -27,3 +27,4 @@ s3.throttle.iopsWriteLimit=5000
 s3.throttle.bpsTotalMB=1280
 s3.throttle.bpsReadMB=1280
 s3.throttle.bpsWriteMB=1280
+s3.useVirtualAddressing=false
diff --git a/curvefs/conf/client.conf b/curvefs/conf/client.conf
@@ -146,6 +146,7 @@ s3.throttle.iopsWriteLimit=0
 s3.throttle.bpsTotalMB=0
 s3.throttle.bpsReadMB=0
 s3.throttle.bpsWriteMB=0
+s3.useVirtualAddressing=false
 
 # TODO(hongsong): limit bytes、iops/bps
 #### disk cache options

diff --git a/curvefs/conf/mds.conf b/curvefs/conf/mds.conf
@@ -128,3 +128,4 @@ s3.throttle.iopsWriteLimit=0
 s3.throttle.bpsTotalMB=0
 s3.throttle.bpsReadMB=0
 s3.throttle.bpsWriteMB=0
+s3.useVirtualAddressing=false
diff --git a/curvefs/conf/metaserver.conf b/curvefs/conf/metaserver.conf
@@ -26,6 +26,7 @@ s3.throttle.iopsWriteLimit=0
 s3.throttle.bpsTotalMB=0
 s3.throttle.bpsReadMB=0
 s3.throttle.bpsWriteMB=0
+s3.useVirtualAddressing=false
 # s3 workqueue
 s3compactwq.enable=True
 s3compactwq.thread_num=2

diff --git a/curvefs/conf/tools.conf b/curvefs/conf/tools.conf
@@ -29,5 +29,6 @@ s3.endpoint=endpoint
 s3.bucket_name=bucket
 s3.blocksize=4194304
 s3.chunksize=67108864
+s3.useVirtualAddressing=false
 # statistic info in xattr, hardlink will not be supported when enable
 enableSumInDir=true
diff --git a/src/common/s3_adapter.cpp b/src/common/s3_adapter.cpp
@@ -69,41 +69,11 @@ Aws::String GetObjectRequestRange(uint64_t offset, uint64_t len) {
 }  // namespace
 
 void InitS3AdaptorOption(Configuration* conf, S3AdapterOption* s3Opt) {
-    LOG_IF(FATAL, !conf->GetIntValue("s3.loglevel", &s3Opt->loglevel));
-    LOG_IF(FATAL, !conf->GetStringValue("s3.logPrefix", &s3Opt->logPrefix));
+    InitS3AdaptorOptionExceptS3InfoOption(conf, s3Opt);
     LOG_IF(FATAL, !conf->GetStringValue("s3.endpoint", &s3Opt->s3Address));
     LOG_IF(FATAL, !conf->GetStringValue("s3.ak", &s3Opt->ak));
     LOG_IF(FATAL, !conf->GetStringValue("s3.sk", &s3Opt->sk));
     LOG_IF(FATAL, !conf->GetStringValue("s3.bucket_name", &s3Opt->bucketName));
-    LOG_IF(FATAL, !conf->GetIntValue("s3.http_scheme", &s3Opt->scheme));
-    LOG_IF(FATAL, !conf->GetBoolValue("s3.verify_SSL", &s3Opt->verifySsl));
-    LOG_IF(FATAL,
-           !conf->GetIntValue("s3.max_connections", &s3Opt->maxConnections));
-    LOG_IF(FATAL,
-           !conf->GetIntValue("s3.connect_timeout", &s3Opt->connectTimeout));
-    LOG_IF(FATAL,
-           !conf->GetIntValue("s3.request_timeout", &s3Opt->requestTimeout));
-    LOG_IF(FATAL,
-           !conf->GetIntValue("s3.async_thread_num", &s3Opt->asyncThreadNum));
-    LOG_IF(FATAL, !conf->GetUInt64Value("s3.throttle.iopsTotalLimit",
-                                        &s3Opt->iopsTotalLimit));
-    LOG_IF(FATAL, !conf->GetUInt64Value("s3.throttle.iopsReadLimit",
-                                        &s3Opt->iopsReadLimit));
-    LOG_IF(FATAL, !conf->GetUInt64Value("s3.throttle.iopsWriteLimit",
-                                        &s3Opt->iopsWriteLimit));
-    LOG_IF(FATAL,
-           !conf->GetUInt64Value("s3.throttle.bpsTotalMB", &s3Opt->bpsTotalMB));
-    LOG_IF(FATAL,
-           !conf->GetUInt64Value("s3.throttle.bpsReadMB", &s3Opt->bpsReadMB));
-    LOG_IF(FATAL,
-           !conf->GetUInt64Value("s3.throttle.bpsWriteMB", &s3Opt->bpsWriteMB));
-
-    if (!conf->GetUInt64Value("s3.max_async_request_inflight_bytes",
-                              &s3Opt->maxAsyncRequestInflightBytes)) {
-        LOG(WARNING)
-            << "Not found s3.max_async_request_inflight_bytes in conf ";
-        s3Opt->maxAsyncRequestInflightBytes = 0;
-    }
 }
 
 void InitS3AdaptorOptionExceptS3InfoOption(Configuration* conf,
@@ -132,6 +102,8 @@ void InitS3AdaptorOptionExceptS3InfoOption(Configuration* conf,
         &s3Opt->bpsReadMB));
     LOG_IF(FATAL, !conf->GetUInt64Value("s3.throttle.bpsWriteMB",
         &s3Opt->bpsWriteMB));
+    LOG_IF(FATAL, !conf->GetBoolValue("s3.useVirtualAddressing",
+        &s3Opt->useVirtualAddressing));
 
     if (!conf->GetUInt64Value("s3.max_async_request_inflight_bytes",
                               &s3Opt->maxAsyncRequestInflightBytes)) {
@@ -191,7 +163,7 @@ void S3Adapter::Init(const S3AdapterOption &option) {
             Aws::Auth::AWSCredentials(s3Ak_, s3Sk_),
             *clientCfg_,
             Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
-            false);
+            option.useVirtualAddressing);
 
     ReadWriteThrottleParams params;
     params.iopsTotal.limit = option.iopsTotalLimit;

diff --git a/src/common/s3_adapter.h b/src/common/s3_adapter.h
@@ -85,6 +85,7 @@ struct S3AdapterOption {
     uint64_t bpsTotalMB;
     uint64_t bpsReadMB;
     uint64_t bpsWriteMB;
+    bool useVirtualAddressing;
 };
 
 struct S3InfoOption {