Revert "Rollback vertex key (vesoft-inc#4629)"

This reverts commit 545137a.

docker/README.md

@@ -5,4 +5,4 @@ Following docker images will be ready in production.
 - [vesoft/nebula-graphd](https://hub.docker.com/r/vesoft/nebula-graphd): nebula-graphd service built with `Dockerfile.graphd`
 - [vesoft/nebula-metad](https://hub.docker.com/r/vesoft/nebula-metad): nebula-metad service built with `Dockerfile.metad`
 - [vesoft/nebula-storaged](https://hub.docker.com/r/vesoft/nebula-storaged): nebula-storaged service built with `Dockerfile.storaged`
-- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump and meta_dump
+- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump, meta_dump and db_upgrader

src/common/utils/NebulaKeyUtils.cpp

@@ -253,6 +253,7 @@ std::vector<std::string> NebulaKeyUtils::snapshotPrefix(PartitionID partId) {
   if (partId == 0) {
     result.emplace_back("");
   } else {
+    result.emplace_back(vertexPrefix(partId));
     result.emplace_back(tagPrefix(partId));
     result.emplace_back(edgePrefix(partId));
     result.emplace_back(IndexKeyUtils::indexPrefix(partId));

src/common/utils/NebulaKeyUtils.h

@@ -155,12 +155,6 @@ class NebulaKeyUtils final {
     return static_cast<NebulaKeyType>(type) == NebulaKeyType::kEdge;
   }
 
-  static bool isVertex(const folly::StringPiece& rawKey) {
-    constexpr int32_t len = static_cast<int32_t>(sizeof(NebulaKeyType));
-    auto type = readInt<uint32_t>(rawKey.data(), len) & kTypeMask;
-    return static_cast<NebulaKeyType>(type) == NebulaKeyType::kVertex;
-  }
-
   static bool isLock(size_t vIdLen, const folly::StringPiece& rawKey) {
     return isEdge(vIdLen, rawKey, kLockVersion);
   }

src/graph/service/GraphFlags.cpp

@@ -103,5 +103,3 @@ DEFINE_uint32(
     gc_worker_size,
     0,
     "Background garbage clean workers, default number is 0 which means using hardware core size.");
-
-DEFINE_bool(graph_use_vertex_key, false, "whether allow insert or query the vertex key");

src/graph/service/GraphFlags.h

@@ -65,7 +65,4 @@ DECLARE_int32(max_job_size);
 
 DECLARE_bool(enable_async_gc);
 DECLARE_uint32(gc_worker_size);
-
-DECLARE_bool(graph_use_vertex_key);
-
 #endif  // GRAPH_GRAPHFLAGS_H_

src/graph/validator/MutateValidator.cpp

@@ -46,9 +46,6 @@ Status InsertVerticesValidator::check() {
   }
 
   auto tagItems = sentence->tagItems();
-  if (!FLAGS_graph_use_vertex_key && tagItems.empty()) {
-    return Status::SemanticError("Insert vertex is forbidden, please speicify the tag");
-  }
 
   schemas_.reserve(tagItems.size());
 

src/storage/CompactionFilter.h

@@ -14,7 +14,6 @@
 #include "common/utils/OperationKeyUtils.h"
 #include "kvstore/CompactionFilter.h"
 #include "storage/CommonUtils.h"
-#include "storage/StorageFlags.h"
 
 namespace nebula {
 namespace storage {
@@ -37,8 +36,6 @@ class StorageCompactionFilter final : public kvstore::KVFilter {
       return !edgeValid(spaceId, key, val);
     } else if (IndexKeyUtils::isIndexKey(key)) {
       return !indexValid(spaceId, key, val);
-    } else if (!FLAGS_use_vertex_key && NebulaKeyUtils::isVertex(key)) {
-      return true;
     } else if (NebulaKeyUtils::isLock(vIdLen_, key)) {
       return !lockValid(spaceId, key);
     } else {

src/storage/StorageFlags.cpp

@@ -46,5 +46,3 @@ DEFINE_bool(query_concurrently,
             false,
             "whether to run query of each part concurrently, only lookup and "
             "go are supported");
-
-DEFINE_bool(use_vertex_key, false, "whether allow insert or query the vertex key");

src/storage/StorageFlags.h

@@ -34,6 +34,4 @@ DECLARE_int32(max_edge_returned_per_vertex);
 
 DECLARE_bool(query_concurrently);
 
-DECLARE_bool(use_vertex_key);
-
 #endif  // STORAGE_STORAGEFLAGS_H_

src/storage/admin/StatsTask.cpp

@@ -111,6 +111,8 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
   std::unique_ptr<kvstore::KVIterator> tagIter;
   auto edgePrefix = NebulaKeyUtils::edgePrefix(part);
   std::unique_ptr<kvstore::KVIterator> edgeIter;
+  auto vertexPrefix = NebulaKeyUtils::vertexPrefix(part);
+  std::unique_ptr<kvstore::KVIterator> vertexIter;
 
   // When the storage occurs leader change, continue to read data from the
   // follower instead of reporting an error.
@@ -124,6 +126,11 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
     LOG(INFO) << "Stats task failed";
     return ret;
   }
+  ret = env_->kvstore_->prefix(spaceId, part, vertexPrefix, &vertexIter, true);
+  if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
+    LOG(INFO) << "Stats task failed";
+    return ret;
+  }
   std::unordered_map<TagID, int64_t> tagsVertices;
   std::unordered_map<EdgeType, int64_t> edgetypeEdges;
   std::unordered_map<PartitionID, int64_t> positiveRelevancy;
@@ -166,7 +173,6 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
       tagIter->next();
       continue;
     }
-    spaceVertices++;
     tagsVertices[tagId] += 1;
     tagIter->next();
     sleepIfScannedSomeRecord(++countToSleep);
@@ -221,6 +227,11 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
     edgeIter->next();
     sleepIfScannedSomeRecord(++countToSleep);
   }
+  while (vertexIter && vertexIter->valid()) {
+    spaceVertices++;
+    vertexIter->next();
+    sleepIfScannedSomeRecord(++countToSleep);
+  }
   nebula::meta::cpp2::StatsItem statsItem;
 
   // convert tagId/edgeType to tagName/edgeName

src/storage/exec/GetPropNode.h

@@ -42,23 +42,18 @@ class GetTagPropNode : public QueryNode<VertexID> {
       return ret;
     }
 
-    // If none of the tag node valid, will check vertex key if use_vertex_key is true,
-    // do not emplace the row if the flag is false
+    // if none of the tag node and vertex valid, do not emplace the row
     if (!std::any_of(tagNodes_.begin(), tagNodes_.end(), [](const auto& tagNode) {
           return tagNode->valid();
         })) {
-      if (FLAGS_use_vertex_key) {
-        auto kvstore = context_->env()->kvstore_;
-        auto vertexKey = NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId);
-        std::string value;
-        ret = kvstore->get(context_->spaceId(), partId, vertexKey, &value);
-        if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) {
-          return nebula::cpp2::ErrorCode::SUCCEEDED;
-        } else if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
-          return ret;
-        }
-      } else {
+      auto kvstore = context_->env()->kvstore_;
+      auto vertexKey = NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId);
+      std::string value;
+      ret = kvstore->get(context_->spaceId(), partId, vertexKey, &value);
+      if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) {
         return nebula::cpp2::ErrorCode::SUCCEEDED;
+      } else if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
+        return ret;
       }
     }
 

src/storage/exec/UpdateNode.h

@@ -488,6 +488,7 @@ class UpdateTagNode : public UpdateNode<VertexID> {
       }
     }
     // step 3, insert new vertex data
+    batchHolder->put(NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId), "");
     batchHolder->put(std::move(key_), std::move(nVal));
     return encodeBatchValue(batchHolder->getBatch());
   }

src/storage/mutate/AddVerticesProcessor.cpp

@@ -81,6 +81,7 @@ void AddVerticesProcessor::doProcess(const cpp2::AddVerticesRequest& req) {
         code = nebula::cpp2::ErrorCode::E_INVALID_VID;
         break;
       }
+      data.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid), "");
       for (auto& newTag : newTags) {
         auto tagId = newTag.get_tag_id();
         VLOG(3) << "PartitionID: " << partId << ", VertexID: " << vid << ", TagID: " << tagId;
@@ -159,6 +160,7 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re
         break;
       }
 
+      verticeData.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid));
       for (const auto& newTag : newTags) {
         auto tagId = newTag.get_tag_id();
         VLOG(3) << "PartitionID: " << partId << ", VertexID: " << vid << ", TagID: " << tagId;

src/tools/CMakeLists.txt

@@ -69,3 +69,4 @@ nebula_add_subdirectory(simple-kv-verify)
 endif()
 nebula_add_subdirectory(meta-dump)
 nebula_add_subdirectory(db-dump)
+nebula_add_subdirectory(db-upgrade)

src/tools/db-upgrade/CMakeLists.txt

@@ -0,0 +1,29 @@
+nebula_add_executable(
+    NAME
+        db_upgrader
+    SOURCES
+        DbUpgraderTool.cpp
+        NebulaKeyUtilsV1.cpp
+        NebulaKeyUtilsV2.cpp
+        NebulaKeyUtilsV3.cpp
+        DbUpgrader.cpp
+    OBJECTS
+        ${tools_test_deps}
+    LIBRARIES
+        ${ROCKSDB_LIBRARIES}
+        ${THRIFT_LIBRARIES}
+        ${PROXYGEN_LIBRARIES}
+)
+
+install(
+    TARGETS
+        db_upgrader
+    PERMISSIONS
+        OWNER_EXECUTE OWNER_WRITE OWNER_READ
+        GROUP_EXECUTE GROUP_READ
+        WORLD_EXECUTE WORLD_READ
+    DESTINATION
+        bin
+    COMPONENT
+        tool
+)