curvebs/mds|curvebs/client:create volume with specified diskType

Signed-off-by: jolly-sy <[email protected]>
opencurve · Dec 13, 2022 · b339c17 · b339c17
1 parent d3937fe
commit b339c17
Show file tree

Hide file tree

Showing 67 changed files with 491 additions and 226 deletions.
diff --git a/conf/mds.conf b/conf/mds.conf
@@ -197,6 +197,8 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
+# curvebs的默认的poolset name 
+mds.curvefs.defaultPoolset = ssdPoolset1
 
 #
 # chunkseverclient config

diff --git a/include/client/libcurve.h b/include/client/libcurve.h
@@ -138,7 +138,8 @@ int Create(const char* filename,
  */
 int Create2(const char* filename,
            const C_UserInfo_t* userinfo,
-           size_t size, uint64_t stripeUnit, uint64_t stripeCount);
+           size_t size, uint64_t stripeUnit,
+           uint64_t stripeCount, const char* poolsetName);
 
 /**
  * 同步模式读

diff --git a/proto/nameserver2.proto b/proto/nameserver2.proto
@@ -93,6 +93,7 @@ message FileInfo {
 
     optional    FileThrottleParams throttleParams = 17;
     optional    uint64      epoch = 18;
+    optional    string      poolsetName = 19;
 }
 
 // status code
@@ -166,11 +167,12 @@ enum  StatusCode {
     kRecoverFileError = 139;
     // epoch too old
     kEpochTooOld = 140;
-
     // 元数据存储错误
     kStorageError = 501;
     // 内部错误
     KInternalError = 502;
+    // poolset不存在
+    kPoolsetNotExist = 503;
 };
 
 //chunkinfo
@@ -202,6 +204,7 @@ message CreateFileRequest {
     required uint64     date = 6;
     optional uint64     stripeUnit = 7;
     optional uint64     stripeCount = 8;
+    optional string     poolsetName = 9;
 };
 
 message CreateFileResponse {

diff --git a/proto/topology.proto b/proto/topology.proto
@@ -92,7 +92,7 @@ message PhysicalPoolData {
     required uint32 physicalPoolId = 1;
     required string physicalPoolName = 2;
     required string desc = 3;
-    required uint32 poolsetId = 4;
+    optional uint32 poolsetId = 4;
 }
 
 message ZoneData {

diff --git a/src/client/libcurve_file.cpp b/src/client/libcurve_file.cpp
@@ -319,7 +319,7 @@ int FileClient::Create(const std::string& filename,
     const UserInfo_t& userinfo, size_t size) {
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
-        ret = mdsClient_->CreateFile(filename, userinfo, size);
+        ret = mdsClient_->CreateFile(filename, "", userinfo, size);
         LOG_IF(ERROR, ret != LIBCURVE_ERROR::OK)
             << "Create file failed, filename: " << filename << ", ret: " << ret;
     } else {
@@ -331,11 +331,12 @@ int FileClient::Create(const std::string& filename,
 
 int FileClient::Create2(const std::string& filename,
     const UserInfo_t& userinfo, size_t size,
-    uint64_t stripeUnit, uint64_t stripeCount) {
+    uint64_t stripeUnit, uint64_t stripeCount,
+    const std::string& poolsetName) {
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
-        ret = mdsClient_->CreateFile(filename, userinfo, size, true,
-                                     stripeUnit, stripeCount);
+        ret = mdsClient_->CreateFile(filename, poolsetName, userinfo,
+                                     size, true, stripeUnit, stripeCount);
         LOG_IF(ERROR, ret != LIBCURVE_ERROR::OK)
             << "Create file failed, filename: " << filename << ", ret: " << ret;
     } else {
@@ -575,7 +576,7 @@ int FileClient::Listdir(const std::string& dirpath,
 int FileClient::Mkdir(const std::string& dirpath, const UserInfo_t& userinfo) {
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
-        ret = mdsClient_->CreateFile(dirpath, userinfo, 0, false);
+        ret = mdsClient_->CreateFile(dirpath,"", userinfo, 0, false);
         if (ret != LIBCURVE_ERROR::OK) {
             if (ret == LIBCURVE_ERROR::EXISTS) {
                 LOG(WARNING) << "Create directory failed, " << dirpath
@@ -897,15 +898,16 @@ int Create(const char* filename, const C_UserInfo_t* userinfo, size_t size) {
 }
 
 int Create2(const char* filename, const C_UserInfo_t* userinfo, size_t size,
-                                uint64_t stripeUnit, uint64_t stripeCount) {
+                                uint64_t stripeUnit, uint64_t stripeCount,
+                                const char* poolsetName) {
     if (globalclient == nullptr) {
         LOG(ERROR) << "not inited!";
         return -LIBCURVE_ERROR::FAILED;
     }
 
     return globalclient->Create2(filename,
             UserInfo(userinfo->owner, userinfo->password),
-                       size, stripeUnit, stripeCount);
+                       size, stripeUnit, stripeCount, poolsetName);
 }
 
 int Rename(const C_UserInfo_t* userinfo,

diff --git a/src/client/libcurve_file.h b/src/client/libcurve_file.h
@@ -41,6 +41,7 @@ namespace curve {
 namespace client {
 
 using curve::common::BthreadRWLock;
+using PoolsetType = ::curve::mds::topology::PoolsetType;
 
 class FileClient {
  public:
@@ -111,7 +112,8 @@ class FileClient {
     virtual int Create2(const std::string& filename,
                         const UserInfo_t& userinfo,
                         size_t size, uint64_t stripeUnit,
-                        uint64_t stripeCount);
+                        uint64_t stripeCount,
+                        const std::string& poolsetName);
 
     /**
      * 同步模式读

diff --git a/src/client/mds_client.cpp b/src/client/mds_client.cpp
@@ -308,16 +308,18 @@ LIBCURVE_ERROR MDSClient::OpenFile(const std::string &filename,
 }
 
 LIBCURVE_ERROR MDSClient::CreateFile(const std::string &filename,
-                                     const UserInfo_t &userinfo, size_t size,
+                                     const std::string &poolsetName,
+                                     const UserInfo_t &userinfo,
+                                     size_t size,
                                      bool normalFile, uint64_t stripeUnit,
                                      uint64_t stripeCount) {
     auto task = RPCTaskDefine {
         CreateFileResponse response;
         mdsClientMetric_.createFile.qps.count << 1;
         LatencyGuard lg(&mdsClientMetric_.createFile.latency);
-        MDSClientBase::CreateFile(filename, userinfo, size, normalFile,
-                                  stripeUnit, stripeCount, &response, cntl,
-                                  channel);
+        MDSClientBase::CreateFile(filename, poolsetName, userinfo, size,
+                                  normalFile, stripeUnit, stripeCount,
+                                  &response, cntl, channel);
 
         if (cntl->Failed()) {
             mdsClientMetric_.createFile.eps.count << 1;

diff --git a/src/client/mds_client.h b/src/client/mds_client.h
@@ -181,7 +181,9 @@ class MDSClient : public MDSClientBase,
      *          如果认证失败返回LIBCURVE_ERROR::AUTHFAIL，
      */
     LIBCURVE_ERROR CreateFile(const std::string &filename,
-                              const UserInfo_t &userinfo, size_t size = 0,
+                              const std::string &poolsetName,
+                              const UserInfo_t &userinfo,
+                              size_t size = 0,
                               bool normalFile = true, uint64_t stripeUnit = 0,
                               uint64_t stripeCount = 0);
     /**

diff --git a/src/client/mds_client_base.cpp b/src/client/mds_client_base.cpp
@@ -61,6 +61,7 @@ void MDSClientBase::OpenFile(const std::string& filename,
 }
 
 void MDSClientBase::CreateFile(const std::string& filename,
+                               const std::string& poolsetName,
                                const UserInfo_t& userinfo,
                                size_t size,
                                bool normalFile,
@@ -80,14 +81,16 @@ void MDSClientBase::CreateFile(const std::string& filename,
 
     request.set_stripeunit(stripeUnit);
     request.set_stripecount(stripeCount);
+    request.set_poolsetname(poolsetName);
     FillUserInfo(&request, userinfo);
 
     LOG(INFO) << "CreateFile: filename = " << filename
                 << ", owner = " << userinfo.owner
                 << ", is normalfile: " << normalFile
                 << ", log id = " << cntl->log_id()
                 << ", stripeUnit = " << stripeUnit
-                << ", stripeCount = " << stripeCount;
+                << ", stripeCount = " << stripeCount
+                << ", poolsetName = " << poolsetName;
 
     curve::mds::CurveFSService_Stub stub(channel);
     stub.CreateFile(cntl, &request, response, NULL);

diff --git a/src/client/mds_client_base.h b/src/client/mds_client_base.h
@@ -120,6 +120,7 @@ class MDSClientBase {
      * @param[in]:channel是当前与mds建立的通道
      */
     void CreateFile(const std::string& filename,
+                    const std::string& poolsetName,
                     const UserInfo_t& userinfo,
                     size_t size,
                     bool normalFile,

diff --git a/src/mds/nameserver2/chunk_allocator.cpp b/src/mds/nameserver2/chunk_allocator.cpp
@@ -29,7 +29,8 @@ namespace curve {
 namespace mds {
 bool ChunkSegmentAllocatorImpl::AllocateChunkSegment(FileType type,
         SegmentSizeType segmentSize, ChunkSizeType chunkSize,
-        offset_t offset, PageFileSegment *segment)  {
+        std::string pstName, offset_t offset,
+        PageFileSegment *segment)  {
         if (segment == nullptr) {
             LOG(ERROR) << "segment pointer is null";
             return false;
@@ -54,7 +55,7 @@ bool ChunkSegmentAllocatorImpl::AllocateChunkSegment(FileType type,
         std::vector<CopysetIdInfo> copysets;
         if (!topologyChunkAllocator_->
                 AllocateChunkRoundRobinInSingleLogicalPool(
-                type, chunkNum, chunkSize, &copysets)) {
+                type, pstName, chunkNum, chunkSize, &copysets)) {
             LOG(ERROR) << "AllocateChunkRoundRobinInSingleLogicalPool error";
             return false;
         }

diff --git a/src/mds/nameserver2/chunk_allocator.h b/src/mds/nameserver2/chunk_allocator.h
@@ -26,12 +26,14 @@
 #include <stdint.h>
 #include <vector>
 #include <map>
+#include <string>
 #include <memory>
 #include "src/mds/common/mds_define.h"
 #include "src/mds/nameserver2/idgenerator/chunk_id_generator.h"
 #include "src/mds/topology/topology_chunk_allocator.h"
 
 using ::curve::mds::topology::TopologyChunkAllocator;
+using ::curve::mds::topology::PoolsetType;
 
 namespace curve {
 namespace mds {
@@ -42,10 +44,13 @@ class ChunkSegmentAllocator {
 
     virtual bool AllocateChunkSegment(FileType type,
         SegmentSizeType segmentSize, ChunkSizeType chunkSize,
-        offset_t offset, PageFileSegment *segment) = 0;
+        std::string pstName, offset_t offset,
+        PageFileSegment *segment) = 0;
+
     virtual void GetRemainingSpaceInLogicalPool(
         const std::vector<PoolIdType>& logicalPools,
-        std::map<PoolIdType, double>* remianingSpace) = 0;
+        std::map<PoolIdType, double>* remianingSpace,
+        const std::string& pstName) = 0;
 };
 
 
@@ -67,13 +72,15 @@ class ChunkSegmentAllocatorImpl: public ChunkSegmentAllocator {
 
     bool AllocateChunkSegment(FileType type,
         SegmentSizeType segmentSize, ChunkSizeType chunkSize,
-        offset_t offset, PageFileSegment *segment) override;
+        std::string pstName, offset_t offset,
+        PageFileSegment *segment) override;
 
     void GetRemainingSpaceInLogicalPool(
         const std::vector<PoolIdType>& logicalPools,
-        std::map<PoolIdType, double>* remianingSpace) {
+        std::map<PoolIdType, double>* remianingSpace,
+        const std::string & pstName) {
             return topologyChunkAllocator_->GetRemainingSpaceInLogicalPool(
-                            logicalPools, remianingSpace);
+                            logicalPools, remianingSpace, pstName);
         }
 
  private:

diff --git a/src/mds/nameserver2/curveadm b/src/mds/nameserver2/curveadm
diff --git a/src/mds/nameserver2/curvefs.cpp b/src/mds/nameserver2/curvefs.cpp
@@ -45,6 +45,9 @@ using curve::mds::topology::CopySetIdType;
 using curve::mds::topology::ChunkServer;
 using curve::mds::topology::ChunkServerStatus;
 using curve::mds::topology::OnlineState;
+using curve::mds::topology::Poolset;
+using curve::mds::topology::PoolsetIdType;
+using ::curve::mds::topology::UNINTIALIZE_ID;
 
 namespace curve {
 namespace mds {
@@ -138,6 +141,7 @@ bool CurveFS::Init(std::shared_ptr<NameServerStorage> storage,
     defaultSegmentSize_ = curveFSOptions.defaultSegmentSize;
     minFileLength_ = curveFSOptions.minFileLength;
     maxFileLength_ = curveFSOptions.maxFileLength;
+    poolsetName_ = curveFSOptions.poolsetName;
     topology_ = topology;
     snapshotCloneClient_ = snapshotCloneClient;
 
@@ -245,6 +249,7 @@ StatusCode CurveFS::SnapShotFile(const FileInfo * origFileInfo,
 }
 
 StatusCode CurveFS::CreateFile(const std::string & fileName,
+                               const std::string & poolsetName,
                                const std::string& owner,
                                FileType filetype, uint64_t length,
                                uint64_t stripeUnit, uint64_t stripeCount) {
@@ -286,8 +291,9 @@ StatusCode CurveFS::CreateFile(const std::string & fileName,
                 topology_->GetLogicalPoolInCluster();
     std::map<PoolIdType, double> remianingSpace;
     uint64_t allRemianingSpace = 0;
+
     chunkSegAllocator_->GetRemainingSpaceInLogicalPool(logicalPools,
-                                &remianingSpace);
+                                &remianingSpace, poolsetName);
 
     for (auto it = remianingSpace.begin(); it != remianingSpace.end(); it++) {
         allRemianingSpace +=it->second;
@@ -327,6 +333,7 @@ StatusCode CurveFS::CreateFile(const std::string & fileName,
 
         fileInfo.set_id(inodeID);
         fileInfo.set_filename(lastEntry);
+        fileInfo.set_poolsetname(poolsetName);
         fileInfo.set_parentid(parentFileInfo.id());
         fileInfo.set_filetype(filetype);
         fileInfo.set_owner(owner);
@@ -770,6 +777,16 @@ StatusCode CurveFS::DeleteFile(const std::string & filename, uint64_t fileId,
     }
 }
 
+StatusCode CurveFS::CheckPoolsetName(const std::string & pstName) {
+    PoolsetIdType res = topology_->FindPoolset(pstName);
+    if (res == UNINTIALIZE_ID) {
+        LOG(ERROR) << "not found the poolset, poolset name is "
+        << pstName;
+        return StatusCode::kPoolsetNotExist;
+    }
+    return StatusCode::kOK;
+}
+
 StatusCode CurveFS::RecoverFile(const std::string & originFileName,
                                 const std::string & recycleFileName,
                                 uint64_t fileId) {
@@ -1250,10 +1267,16 @@ StatusCode CurveFS::GetOrAllocateSegment(const std::string & filename,
                       << ", not allocated";
             return  StatusCode::kSegmentNotAllocated;
         } else {
-            // TODO(hzsunjianliang): check the user and define the logical pool
+            // TODO(hzsunjianliang): check the user and define the logical pool]
+            std::string pstName = poolsetName_;
+            LOG(INFO) << "default poolsetName is " << poolsetName_;
+            if (fileInfo.has_poolsetname()) {
+                pstName = fileInfo.poolsetname();
+            }
             auto ifok = chunkSegAllocator_->AllocateChunkSegment(
                             fileInfo.filetype(), fileInfo.segmentsize(),
-                            fileInfo.chunksize(), offset, segment);
+                            fileInfo.chunksize(), pstName,
+                            offset, segment);
             if (ifok == false) {
                 LOG(ERROR) << "AllocateChunkSegment error";
                 return StatusCode::kSegmentAllocateError;