Check running job before create backup (#5013)

* refactor create bakcup processor test change back to snapshot lock and acquire lock when add job * fix initialize * improve log * fix job manager Co-authored-by: Sophie <[email protected]>
vesoft-inc · Dec 27, 2022 · 23dc0d0 · 23dc0d0
1 parent 45f5bfd
commit 23dc0d0
Show file tree

Hide file tree

Showing 8 changed files with 196 additions and 119 deletions.
diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp
@@ -933,8 +933,8 @@ Status MetaClient::handleResponse(const RESP& resp) {
       return Status::Error("Task report is out of date!");
     case nebula::cpp2::ErrorCode::E_BACKUP_FAILED:
       return Status::Error("Backup failure!");
-    case nebula::cpp2::ErrorCode::E_BACKUP_BUILDING_INDEX:
-      return Status::Error("Backup building indexes!");
+    case nebula::cpp2::ErrorCode::E_BACKUP_RUNNING_JOBS:
+      return Status::Error("Backup encounter running or queued jobs!");
     case nebula::cpp2::ErrorCode::E_BACKUP_SPACE_NOT_FOUND:
       return Status::Error("The space is not found when backup!");
     case nebula::cpp2::ErrorCode::E_RESTORE_FAILURE:

diff --git a/src/common/graph/Response.h b/src/common/graph/Response.h
@@ -113,7 +113,7 @@
   X(E_INVALID_JOB, -2065)                                                     \
                                                                               \
   /* Backup Failure */                                                        \
-  X(E_BACKUP_BUILDING_INDEX, -2066)                                           \
+  X(E_BACKUP_RUNNING_JOBS, -2066)                                             \
   X(E_BACKUP_SPACE_NOT_FOUND, -2067)                                          \
                                                                               \
   /* RESTORE Failure */                                                       \

diff --git a/src/interface/common.thrift b/src/interface/common.thrift
@@ -405,7 +405,7 @@ enum ErrorCode {
     E_INVALID_JOB                     = -2065,  // Invalid task
 
     // Backup Failure
-    E_BACKUP_BUILDING_INDEX           = -2066,  // Backup terminated (index being created)
+    E_BACKUP_RUNNING_JOBS             = -2066,  // Backup terminated (some data modification jobs running)
     E_BACKUP_SPACE_NOT_FOUND          = -2067,  // Graph space does not exist at the time of backup
 
     // RESTORE Failure

diff --git a/src/meta/processors/admin/CreateBackupProcessor.cpp b/src/meta/processors/admin/CreateBackupProcessor.cpp
@@ -84,17 +84,22 @@ void CreateBackupProcessor::process(const cpp2::CreateBackupReq& req) {
 
   // make sure there is no index job
   std::unordered_set<cpp2::JobType> jobTypes{cpp2::JobType::REBUILD_TAG_INDEX,
-                                             cpp2::JobType::REBUILD_EDGE_INDEX};
+                                             cpp2::JobType::REBUILD_EDGE_INDEX,
+                                             cpp2::JobType::COMPACT,
+                                             cpp2::JobType::INGEST,
+                                             cpp2::JobType::DATA_BALANCE,
+                                             cpp2::JobType::LEADER_BALANCE};
   auto result = jobMgr->checkTypeJobRunning(jobTypes);
   if (!nebula::ok(result)) {
-    LOG(INFO) << "Get Index status failed, not allowed to create backup.";
+    LOG(INFO) << "Get running job status failed, not allowed to create backup.";
     handleErrorCode(nebula::error(result));
     onFinished();
     return;
   }
   if (nebula::value(result)) {
-    LOG(INFO) << "Index is rebuilding, not allowed to create backup.";
-    handleErrorCode(nebula::cpp2::ErrorCode::E_BACKUP_BUILDING_INDEX);
+    LOG(INFO) << "There is some running or queued job mutating the data, not allowed to "
+                 "create backup now.";
+    handleErrorCode(nebula::cpp2::ErrorCode::E_BACKUP_RUNNING_JOBS);
     onFinished();
     return;
   }

diff --git a/src/meta/processors/job/AdminJobProcessor.cpp b/src/meta/processors/job/AdminJobProcessor.cpp
@@ -139,6 +139,7 @@ nebula::cpp2::ErrorCode AdminJobProcessor::addJobProcess(const cpp2::AdminJobReq
   }
 
   folly::SharedMutex::WriteHolder holder(LockUtils::lock());
+  folly::SharedMutex::ReadHolder snapHolder(LockUtils::snapshotLock());
   auto jobId = autoIncrementId();
   if (!nebula::ok(jobId)) {
     return nebula::error(jobId);

diff --git a/src/meta/processors/job/JobManager.cpp b/src/meta/processors/job/JobManager.cpp
@@ -1011,6 +1011,10 @@ ErrorOr<nebula::cpp2::ErrorCode, bool> JobManager::checkTypeJobRunning(
 
       auto status = jobDesc.getStatus();
       if (status == cpp2::JobStatus::QUEUE || status == cpp2::JobStatus::RUNNING) {
+        LOG(INFO) << folly::sformat("The {} job is {} in space {}",
+                                    apache::thrift::util::enumNameSafe(jType),
+                                    apache::thrift::util::enumNameSafe(status),
+                                    spaceId);
         return true;
       }
     }

diff --git a/src/meta/processors/job/JobManager.h b/src/meta/processors/job/JobManager.h
@@ -29,6 +29,7 @@ extern stats::CounterId kNumRunningJobs;
 class JobManager : public boost::noncopyable, public nebula::cpp::NonMovable {
   friend class JobManagerTest;
   friend class GetStatsTest;
+  friend class CreateBackupProcessorTest;
   FRIEND_TEST(JobManagerTest, AddJob);
   FRIEND_TEST(JobManagerTest, StatsJob);
   FRIEND_TEST(JobManagerTest, JobPriority);