【Feature】When the cluster capacity is almost full, make the cluster read only

conf/chunkserver.conf

@@ -128,6 +128,8 @@ copyset.sync_chunk_limits=2097152
 copyset.sync_threshold=65536
 # check syncing interval
 copyset.check_syncing_interval_ms=500
+# wait for retry time when disk space is insufficient
+copyset.wait_for_disk_freed_interval_ms=60000
 
 #
 # Clone settings
@@ -215,6 +217,11 @@ chunkfilepool.allocate_percent=80
 chunkfilepool.chunk_file_pool_size=1GB
 # The thread num for format chunks
 chunkfilepool.thread_num=1
+# When the chunkserver disk usage exceeds the percentage, heartbeat sets the disk status
+chunkfilepool.disk_usage_percent_limit=95
+# Reserve part of the chunk number, and the write operation returns readonly to the client
+# when the available value is too small to avoid chunkfilepool and walfilepool not being able to obtain the chunk.
+chunkfilepool.chunk_reserved=100
 
 #
 # WAL file pool

conf/chunkserver.conf.example

@@ -120,6 +120,8 @@ copyset.sync_chunk_limits=2097152
 copyset.sync_threshold=65536
 # check syncing interval
 copyset.check_syncing_interval_ms=500
+# wait for retry time when disk space is insufficient
+copyset.wait_for_disk_freed_interval_ms=60000
 
 #
 # Clone settings
@@ -207,6 +209,11 @@ chunkfilepool.allocate_percent=80
 chunkfilepool.chunk_file_pool_size=1GB
 # The thread num for format chunks
 chunkfilepool.thread_num=1
+# When the chunkserver disk usage exceeds the percentage, heartbeat sets the disk status
+chunkfilepool.disk_usage_percent_limit=95
+# Reserve part of the chunk number, and the write operation returns readonly to the client
+# when the available value is too small to avoid chunkfilepool and walfilepool not being able to obtain the chunk.
+chunkfilepool.chunk_reserved=0
 
 #
 # WAL file pool

proto/chunk.proto

@@ -85,6 +85,7 @@ enum CHUNK_OP_STATUS {
     CHUNK_OP_STATUS_BACKWARD = 10;          // 请求的版本落后当前chunk的版本
     CHUNK_OP_STATUS_CHUNK_EXIST = 11;       // chunk已存在
     CHUNK_OP_STATUS_EPOCH_TOO_OLD = 12;     // request epoch too old
+    CHUNK_OP_STATUS_READONLY = 13;          // If there is insufficient disk space, set the chunkserver to read-only
 };
 
 message ChunkResponse {

proto/heartbeat.proto

@@ -71,8 +71,13 @@ message CopysetStatistics {
     required uint32 writeIOPS = 4;
 }
 
+enum ErrorType {
+    NORMAL = 0;
+    DISKFULL = 1;
+}
+
 message DiskState {
-    required uint32 errType = 1;
+    required ErrorType errType = 1;
     required string errMsg = 2;
 }
 

proto/topology.proto

@@ -48,6 +48,7 @@ enum ChunkServerStatus {
 enum DiskState {
     DISKNORMAL = 0;
     DISKERROR = 1;
+    DISKFULL = 2;
 }
 
 enum OnlineState {

src/chunkserver/chunkserver.cpp

@@ -503,6 +503,9 @@ void ChunkServer::InitChunkFilePoolOptions(
     LOG_IF(FATAL, !conf->GetBoolValue(
         "chunkfilepool.enable_get_chunk_from_pool",
         &chunkFilePoolOptions->getFileFromPool));
+    LOG_IF(FATAL, !conf->GetUInt32Value(
+        "chunkfilepool.chunk_reserved",
+        &chunkFilePoolOptions->chunkReserved));
 
     if (chunkFilePoolOptions->getFileFromPool == false) {
         std::string chunkFilePoolUri;
@@ -710,6 +713,9 @@ void ChunkServer::InitCopysetNodeOptions(
         LOG_IF(FATAL, !conf->GetUInt32Value("copyset.sync_trigger_seconds",
                 &copysetNodeOptions->syncTriggerSeconds));
     }
+    LOG_IF(FATAL, !conf->GetUInt32Value(
+        "copyset.wait_for_disk_freed_interval_ms",
+        &copysetNodeOptions->waitForDiskFreedIntervalMs));
 }
 
 void ChunkServer::InitCopyerOptions(
@@ -781,6 +787,9 @@ void ChunkServer::InitHeartbeatOptions(
         &heartbeatOptions->intervalSec));
     LOG_IF(FATAL, !conf->GetUInt32Value("mds.heartbeat_timeout",
         &heartbeatOptions->timeout));
+    LOG_IF(FATAL, !conf->GetUInt32Value(
+        "chunkfilepool.disk_usage_percent_limit",
+        &heartbeatOptions->chunkserverDiskLimit));
 }
 
 void ChunkServer::InitRegisterOptions(

src/chunkserver/config_info.h

@@ -140,6 +140,8 @@ struct CopysetNodeOptions {
     uint64_t syncThreshold = 64 * 1024;
     // check syncing interval
     uint32_t checkSyncingIntervalMs = 500u;
+    // wait for retry time when disk space is insufficient
+    uint32_t waitForDiskFreedIntervalMs = 60000;
 
     CopysetNodeOptions();
 };

src/chunkserver/copyset_node.cpp

@@ -135,6 +135,8 @@ int CopysetNode::Init(const CopysetNodeOptions &options) {
     dsOptions.locationLimit = options.locationLimit;
     dsOptions.enableOdsyncWhenOpenChunkFile =
         options.enableOdsyncWhenOpenChunkFile;
+    dsOptions.waitForDiskFreedIntervalMs =
+        options.waitForDiskFreedIntervalMs;
     dataStore_ = std::make_shared<CSDataStore>(options.localFileSystem,
                                                options.chunkFilePool,
                                                dsOptions);
@@ -345,6 +347,10 @@ void CopysetNode::WaitSnapshotDone() {
     }
 }
 
+bool CopysetNode::ReadOnly() const {
+    return !dataStore_->EnoughChunk();
+}
+
 void CopysetNode::save_snapshot_background(::braft::SnapshotWriter *writer,
                                    ::braft::Closure *done) {
     brpc::ClosureGuard doneGuard(done);
@@ -529,7 +535,9 @@ void CopysetNode::on_leader_start(int64_t term) {
      *   https://github.com/opencurve/curve/pull/2448
      */
     ChunkServerMetric::GetInstance()->IncreaseLeaderCount();
-    concurrentapply_->Flush();
+    if (concurrentapply_ != nullptr) {
+        concurrentapply_->Flush();
+    }
     leaderTerm_.store(term, std::memory_order_release);
     LOG(INFO) << "Copyset: " << GroupIdString()
               << ", peer id: " << peerId_.to_string()

src/chunkserver/copyset_node.h

@@ -469,6 +469,8 @@ class CopysetNode : public braft::StateMachine,
 
     void WaitSnapshotDone();
 
+    bool ReadOnly() const;
+
  private:
     inline std::string GroupId() {
         return ToGroupId(logicPoolId_, copysetId_);

src/chunkserver/datastore/chunkserver_chunkfile.cpp

@@ -19,6 +19,7 @@
  * File Created: Thursday, 6th September 2018 10:49:53 am
  * Author: yangyaokai
  */
+#include <errno.h>
 #include <fcntl.h>
 #include <algorithm>
 #include <memory>
@@ -207,7 +208,8 @@ CSErrorCode CSChunkFile::Open(bool createFile) {
         if (rc != 0  && rc != -EEXIST) {
             LOG(ERROR) << "Error occured when create file."
                        << " filepath = " << chunkFilePath;
-            return CSErrorCode::InternalError;
+            return rc == -ENOSPC ? CSErrorCode::NoSpaceError :
+             CSErrorCode::InternalError;
         }
     }
     int rc = -1;
@@ -400,7 +402,8 @@ CSErrorCode CSChunkFile::Write(SequenceNum sn,
                    << "ChunkID: " << chunkId_
                    << ",request sn: " << sn
                    << ",chunk sn: " << metaPage_.sn;
-        return CSErrorCode::InternalError;
+        return rc == -ENOSPC ? CSErrorCode::NoSpaceError :
+         CSErrorCode::InternalError;
     }
     // If it is a clone chunk, the bitmap will be updated
     CSErrorCode errorCode = flush();
@@ -478,7 +481,8 @@ CSErrorCode CSChunkFile::Paste(const char * buf, off_t offset, size_t length) {
                        << "ChunkID: " << chunkId_
                        << ", offset: " << offset
                        << ", length: " << length;
-            return CSErrorCode::InternalError;
+            return rc == -ENOSPC ? CSErrorCode::NoSpaceError :
+             CSErrorCode::InternalError;
         }
     }
 

src/chunkserver/datastore/chunkserver_datastore.cpp

@@ -44,7 +44,8 @@ CSDataStore::CSDataStore(std::shared_ptr<LocalFileSystem> lfs,
       baseDir_(options.baseDir),
       chunkFilePool_(chunkFilePool),
       lfs_(lfs),
-      enableOdsyncWhenOpenChunkFile_(options.enableOdsyncWhenOpenChunkFile) {
+      enableOdsyncWhenOpenChunkFile_(options.enableOdsyncWhenOpenChunkFile),
+      waitForDiskFreedIntervalMs_(options.waitForDiskFreedIntervalMs) {
     CHECK(!baseDir_.empty()) << "Create datastore failed";
     CHECK(lfs_ != nullptr) << "Create datastore failed";
     CHECK(chunkFilePool_ != nullptr) << "Create datastore failed";
@@ -428,5 +429,9 @@ ChunkMap CSDataStore::GetChunkMap() {
     return metaCache_.GetMap();
 }
 
+bool CSDataStore::EnoughChunk() {
+    return chunkFilePool_->EnoughChunk();
+}
+
 }  // namespace chunkserver
 }  // namespace curve

src/chunkserver/datastore/chunkserver_datastore.h

@@ -63,6 +63,7 @@ struct DataStoreOptions {
     PageSizeType                        metaPageSize;
     uint32_t                            locationLimit;
     bool                                enableOdsyncWhenOpenChunkFile;
+    uint32_t                            waitForDiskFreedIntervalMs;
 };
 
 /**
@@ -328,6 +329,8 @@ class CSDataStore {
 
     virtual ChunkMap GetChunkMap();
 
+    virtual bool EnoughChunk();
+
     void SetCacheCondPtr(std::shared_ptr<std::condition_variable> cond) {
         metaCache_.SetCondPtr(cond);
     }
@@ -336,6 +339,10 @@ class CSDataStore {
         metaCache_.SetSyncChunkLimits(limit, threshold);
     }
 
+    void WaitForDiskFreed() {
+        bthread_usleep(waitForDiskFreedIntervalMs_);
+    }
+
  private:
     CSErrorCode loadChunkFile(ChunkID id);
     CSErrorCode CreateChunkFile(const ChunkOptions & ops,
@@ -362,6 +369,8 @@ class CSDataStore {
     DataStoreMetricPtr metric_;
     // enable O_DSYNC When Open ChunkFile
     bool enableOdsyncWhenOpenChunkFile_;
+    // wait for retry time when disk space is insufficient
+    uint32_t waitForDiskFreedIntervalMs_;
 };
 
 }  // namespace chunkserver

src/chunkserver/datastore/chunkserver_snapshot.cpp

@@ -20,6 +20,7 @@
  * Author: yangyaokai
  */
 
+#include <errno.h>
 #include <memory>
 #include "src/chunkserver/datastore/chunkserver_datastore.h"
 #include "src/chunkserver/datastore/chunkserver_snapshot.h"
@@ -154,7 +155,8 @@ CSErrorCode CSSnapshot::Open(bool createFile) {
         if (ret != 0) {
             LOG(ERROR) << "Error occured when create snapshot."
                    << " filepath = " << snapshotPath;
-            return CSErrorCode::InternalError;
+            return ret == -ENOSPC ? CSErrorCode::NoSpaceError :
+             CSErrorCode::InternalError;
         }
     }
     int rc = lfs_->Open(snapshotPath, O_RDWR|O_NOATIME|O_DSYNC);
@@ -216,7 +218,8 @@ CSErrorCode CSSnapshot::Write(const char * buf, off_t offset, size_t length) {
         LOG(ERROR) << "Write snapshot failed."
                    << "ChunkID: " << chunkId_
                    << ",snapshot sn: " << metaPage_.sn;
-        return CSErrorCode::InternalError;
+        return rc == -ENOSPC ? CSErrorCode::NoSpaceError :
+         CSErrorCode::InternalError;
     }
     uint32_t pageBeginIndex = offset / blockSize_;
     uint32_t pageEndIndex = (offset + length - 1) / blockSize_;

src/chunkserver/datastore/define.h

@@ -73,6 +73,8 @@ enum CSErrorCode {
     // The page has not been written, it will appear when the page that has not
     // been written is read when the clone chunk is read
     PageNerverWrittenError = 13,
+    // ENOSPC error
+    NoSpaceError = 14,
 };
 
 // Chunk details