1.heartbeat reports disk full error and mds set copyset availflag false.

2.copyset node leader set readonly when receive copyset availflag false from heartbeat. 3.When a disk error occurs, the copyset node leader calls set_error_and_rollback and selects a new leader. 4.Added new rpc method to delete copyset node with wrong status. Signed-off-by: liuminjian <[email protected]>
opencurve · Nov 8, 2023 · 01e9dc4 · 01e9dc4
1 parent 5e1d609
commit 01e9dc4
Show file tree

Hide file tree

Showing 7 changed files with 118 additions and 88 deletions.
diff --git a/WORKSPACE b/WORKSPACE
@@ -37,6 +37,7 @@ git_repository(
     commit = "d12de388c97998f5ccd5cb97ed0da728815ef438",
     patches = [
         "//:thirdparties/braft/0001-fix-change-set_error-to-set_errorv.patch",
+        "//:thirdparties/braft/add-iterator-has_error.patch",
     ],
     patch_args = [
         "-p1"

diff --git a/src/chunkserver/copyset_node.cpp b/src/chunkserver/copyset_node.cpp
@@ -293,8 +293,7 @@ void CopysetNode::on_apply(::braft::Iterator &iter) {
          */
         braft::Closure *closure = iter.done();
 
-        braft::Iterator* iterPtr = &iter;
-        std::shared_ptr<IteratorWrapper> wrapperPtr = std::make_shared<IteratorWrapper>(iterPtr);
+        std::shared_ptr<IteratorWrapper> wrapperPtr = std::make_shared<IteratorWrapper>(&iter);
         if (nullptr != closure) {
             /**
              * 1.closure不是null，那么说明当前节点正常，直接从内存中拿到Op
@@ -327,6 +326,8 @@ void CopysetNode::on_apply(::braft::Iterator &iter) {
                                    dataStore_, std::move(request), data, wrapperPtr);
         }
     }
+    // 等待写操作完成，否则on_apply结束后，异步有写错误无法调用set_error_and_rollback()
+    concurrentapply_->Flush();
 }
 
 void CopysetNode::on_shutdown() {
@@ -988,12 +989,6 @@ void CopysetNode::GetStatus(NodeStatus *status) {
     raftNode_->get_status(status);
 }
 
-bool CopysetNode::StatusOK() {
-    NodeStatus status;
-    GetStatus(&status);
-    return status.state != braft::STATE_ERROR;
-}
-
 void CopysetNode::GetLeaderLeaseStatus(braft::LeaderLeaseStatus *status) {
     raftNode_->get_leader_lease_status(status);
 }
@@ -1135,5 +1130,9 @@ void IteratorWrapper::set_error_and_rollback(size_t ntail, const butil::Status*
     iter_->set_error_and_rollback(ntail, st);
 }
 
+bool IteratorWrapper::has_error() const{
+    return iter_->has_error();
+}
+
 }  // namespace chunkserver
 }  // namespace curve
diff --git a/src/chunkserver/copyset_node.h b/src/chunkserver/copyset_node.h
@@ -131,6 +131,7 @@ class IteratorWrapper {
    IteratorWrapper(braft::Iterator *iter): iter_(iter) {}
    ~IteratorWrapper() {}
    virtual void set_error_and_rollback(size_t ntail = 1, const butil::Status* st = NULL);
+   virtual bool has_error() const;
  private:
    braft::Iterator *iter_;
 };
@@ -296,11 +297,6 @@ class CopysetNode : public braft::StateMachine,
      */
     virtual void GetStatus(NodeStatus *status);
 
-    /**
-     * @brief: 判断copyset node的status是否为ERROR
-     */
-    virtual bool StatusOK() ;
-
     /**
      * @brief: get raft node leader lease status
      * @param status[out]: raft node leader lease status