client: set nofile limit

Signed-off-by: Hanqing Wu <[email protected]>

conf/client.conf

@@ -159,6 +159,14 @@ global.metricDummyServerStartPort=9000
 # 是否关闭健康检查: true/关闭 false/不关闭
 global.turnOffHealthCheck=true
 
+# minimal open file limit
+# if set value to 0, then will skip check and set open file limit
+# NOTE: open file limit will affect how may sockets we can create,
+#       the number of sockets is related to the number of chunkserver and mds in the cluster,
+#       and during some exception handling processes, client will create additional sockets
+#       the SAFE value is 2 * (#chunkserver + #mds)
+global.minOpenFileLimit=1024
+
 #
 ### throttle config
 #

conf/cs_client.conf

@@ -144,6 +144,10 @@ global.logPath=/data/log/curve/  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEA
 #
 global.metricDummyServerStartPort=9000
 
+# minimal open file limit
+# if set value to 0, then will skip check and set open file limit
+global.minOpenFileLimit=0
+
 #
 # session map文件，存储打开文件的filename到path的映射
 #

conf/py_client.conf

@@ -123,6 +123,10 @@ global.fileMaxInFlightRPCNum=64
 # 文件IO下发到底层chunkserver最大的分片KB
 global.fileIOSplitMaxSizeKB=64
 
+# minimal open file limit
+# if set value to 0, then will skip check and set open file limit
+global.minOpenFileLimit=0
+
 #
 ################# log相关配置 ###############
 #

conf/snap_client.conf

@@ -144,6 +144,10 @@ global.logPath=/data/log/curve/  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEA
 #
 global.metricDummyServerStartPort=9000
 
+# minimal open file limit
+# if set value to 0, then will skip check and set open file limit
+global.minOpenFileLimit=0
+
 #
 # session map文件，存储打开文件的filename到path的映射
 #

src/client/client_config.cpp

@@ -240,6 +240,13 @@ int ClientConfig::Init(const std::string& configpath) {
         << "config no global.turnOffHealthCheck info, using default value "
         << fileServiceOption_.commonOpt.turnOffHealthCheck;
 
+    constexpr const char* kMinOpenFileLimit = "global.minOpenFileLimit";
+    ret = conf_.GetUInt32Value(kMinOpenFileLimit,
+                               &fileServiceOption_.commonOpt.minimalOpenFiles);
+    LOG_IF(WARNING, !ret) << "config no `" << kMinOpenFileLimit
+                          << "` info, using default value "
+                          << fileServiceOption_.commonOpt.minimalOpenFiles;
+
     ret = conf_.GetUInt32Value(
         "closefd.timeout",
         &fileServiceOption_.ioOpt.closeFdThreadOption.fdTimeout);

src/client/config_info.h

@@ -273,6 +273,14 @@ struct IOOption {
 struct CommonConfigOpt {
     bool mdsRegisterToMDS = false;
     bool turnOffHealthCheck = false;
+
+    // Minimal open file limit
+    // open file limit will affect how may sockets we can create,
+    // the number of sockets is related to the number of chunkserver and mds in
+    // the cluster and during some exception handling processes, client will
+    // create additional sockets the SAFE value is 2 * (#chunkserver + #mds)
+    // Default: 65535
+    uint32_t minimalOpenFiles = 65536;
 };
 
 /**

src/client/libcurve_file.cpp

@@ -27,11 +27,11 @@
 
 #include <algorithm>
 #include <cstdio>
+#include <list>
 #include <memory>
 #include <mutex>   // NOLINT
 #include <thread>  // NOLINT
 #include <utility>
-#include <list>
 
 #include "include/client/libcurve.h"
 #include "include/client/libcurve_define.h"
@@ -42,10 +42,11 @@
 #include "src/client/iomanager4file.h"
 #include "src/client/service_helper.h"
 #include "src/client/source_reader.h"
+#include "src/client/utils.h"
 #include "src/common/curve_version.h"
 #include "src/common/net_common.h"
-#include "src/common/uuid.h"
 #include "src/common/string_util.h"
+#include "src/common/uuid.h"
 
 bool globalclientinited_ = false;
 curve::client::FileClient *globalclient = nullptr;
@@ -153,21 +154,26 @@ int FileClient::Init(const std::string& configpath) {
         return -LIBCURVE_ERROR::FAILED;
     }
 
+    const auto& fileSvcOpts = clientconfig_.GetFileServiceOption();
+
+    static std::once_flag adjustOpenFileLimitFlag;
+    std::call_once(adjustOpenFileLimitFlag, AdjustOpenFileSoftLimitToHardLimit,
+                   fileSvcOpts.commonOpt.minimalOpenFiles);
+
     auto tmpMdsClient = std::make_shared<MDSClient>();
 
-    auto ret = tmpMdsClient->Initialize(
-        clientconfig_.GetFileServiceOption().metaServerOpt);
+    auto ret = tmpMdsClient->Initialize(fileSvcOpts.metaServerOpt);
     if (LIBCURVE_ERROR::OK != ret) {
         LOG(ERROR) << "Init global mds client failed!";
         return -LIBCURVE_ERROR::FAILED;
     }
 
-    if (clientconfig_.GetFileServiceOption().commonOpt.turnOffHealthCheck) {
+    if (fileSvcOpts.commonOpt.turnOffHealthCheck) {
         brpc::FLAGS_health_check_interval = -1;
     }
 
     // set option for source reader
-    SourceReader::GetInstance().SetOption(clientconfig_.GetFileServiceOption());
+    SourceReader::GetInstance().SetOption(fileSvcOpts);
 
     bool rc = StartDummyServer();
     if (rc == false) {
@@ -176,13 +182,12 @@ int FileClient::Init(const std::string& configpath) {
 
     mdsClient_ = std::move(tmpMdsClient);
 
-    int rc2 = csClient_->Init(clientconfig_.GetFileServiceOption().csClientOpt);
+    int rc2 = csClient_->Init(fileSvcOpts.csClientOpt);
     if (rc2 != 0) {
         LOG(ERROR) << "Init ChunkServer Client failed!";
         return -LIBCURVE_ERROR::FAILED;
     }
-    rc2 = csBroadCaster_->Init(
-        clientconfig_.GetFileServiceOption().csBroadCasterOpt);
+    rc2 = csBroadCaster_->Init(fileSvcOpts.csBroadCasterOpt);
     if (rc2 != 0) {
         LOG(ERROR) << "Init ChunkServer BroadCaster failed!";
         return -LIBCURVE_ERROR::FAILED;

src/client/utils.cpp

@@ -0,0 +1,61 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include "src/client/utils.h"
+
+#include <glog/logging.h>
+#include <sys/resource.h>
+
+#include <cerrno>
+
+namespace curve {
+namespace client {
+
+bool AdjustOpenFileSoftLimitToHardLimit(uint64_t limit) {
+    if (limit == 0) {
+        return true;
+    }
+
+    struct rlimit rlim;
+    int rc = getrlimit(RLIMIT_NOFILE, &rlim);
+
+    if (rc != 0) {
+        LOG(WARNING) << "getrlimit failed, error: " << strerror(errno);
+        return false;
+    }
+
+    if (rlim.rlim_max < limit) {
+        LOG(WARNING) << "hard limit: " << rlim.rlim_max
+                     << " is less than min limit: " << limit;
+        return false;
+    }
+
+    if (rlim.rlim_cur != rlim.rlim_max) {
+        rlim.rlim_cur = rlim.rlim_max;
+        rc = setrlimit(RLIMIT_NOFILE, &rlim);
+        if (rc != 0) {
+            LOG(WARNING) << "setrlimit failed, error: " << strerror(errno);
+            return false;
+        }
+    }
+
+    LOG(INFO) << "set hard limit: " << rlim.rlim_max
+              << " soft limit: " << rlim.rlim_cur;
+    return true;
+}
+
+}  // namespace client
+}  // namespace curve

src/client/utils.h

@@ -0,0 +1,33 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef SRC_CLIENT_UTILS_H_
+#define SRC_CLIENT_UTILS_H_
+
+#include <cstdint>
+
+namespace curve {
+namespace client {
+
+// Adjust the soft limit of open files to hard limit.
+// If |limit| equals 0, then directly return true.
+// If hard limit is less than |limit| than return false.
+bool AdjustOpenFileSoftLimitToHardLimit(uint64_t limit);
+
+}  // namespace client
+}  // namespace curve
+
+#endif  // SRC_CLIENT_UTILS_H_

test/client/client_utils_test.cpp

@@ -0,0 +1,48 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+#include <sys/resource.h>
+
+#include "src/client/utils.h"
+
+namespace curve {
+namespace client {
+
+TEST(AdjustOpenFileSoftLimitToHardLimitTest, TestAdjustToZero) {
+    ASSERT_TRUE(AdjustOpenFileSoftLimitToHardLimit(0));
+}
+
+TEST(AdjustOpenFileSoftLimitToHardLimitTest, TestExceedHardLimit) {
+    struct rlimit rlim;
+    ASSERT_EQ(0, getrlimit(RLIMIT_NOFILE, &rlim));
+    ASSERT_FALSE(AdjustOpenFileSoftLimitToHardLimit(rlim.rlim_max + 1));
+}
+
+TEST(AdjustOpenFileSoftLimitToHardLimitTest, TestAdjustToHardLimit) {
+    struct rlimit rlim;
+    ASSERT_EQ(0, getrlimit(RLIMIT_NOFILE, &rlim));
+
+    rlim.rlim_cur = rlim.rlim_max / 2;
+    ASSERT_EQ(0, setrlimit(RLIMIT_NOFILE, &rlim));
+
+    ASSERT_TRUE(AdjustOpenFileSoftLimitToHardLimit(rlim.rlim_max - 1));
+    ASSERT_EQ(0, getrlimit(RLIMIT_NOFILE, &rlim));
+    ASSERT_EQ(rlim.rlim_cur, rlim.rlim_max);
+}
+
+}  // namespace client
+}  // namespace curve