Merge branch 'master' into tck_session

.github/pull_request_template.md

@@ -12,16 +12,16 @@
 #### Special notes for your reviewer, ex. impact of this fix, etc:
 
 
-#### Additional context:
+#### Additional context/ Design document:
 
 
-#### Checklist：
-- [ ] Documentation affected （Please add the label if documentation needs to be modified.)
-- [ ] Incompatible （If it is incompatible, please describe it and add corresponding label.）
-- [ ] Need to cherry-pick （If need to cherry-pick to some branches, please label the destination version(s).)
+#### Checklist:
+- [ ] Documentation affected (Please add the label if documentation needs to be modified.)
+- [ ] Incompatibility (If it breaks the compatibility, please describe it and add the corresponding label.）
+- [ ] If it's needed to cherry-pick (If cherry-pick to some branches is required, please label the destination version(s).)
 - [ ] Performance impacted: Consumes more CPU/Memory
 
+#### Release notes:
 
-#### Release notes：
-Please confirm whether to reflect in release notes and how to describe:
+Please confirm whether to be reflected in release notes and how to describe:
 >                                                                 `

src/clients/storage/StorageClient.cpp

@@ -112,7 +112,8 @@ StorageRpcRespFuture<cpp2::ExecResponse> StorageClient::addVertices(
     const CommonRequestParam& param,
     std::vector<cpp2::NewVertex> vertices,
     std::unordered_map<TagID, std::vector<std::string>> propNames,
-    bool ifNotExists) {
+    bool ifNotExists,
+    bool ignoreExistedIndex) {
   auto cbStatus = getIdFromNewVertex(param.space);
   if (!cbStatus.ok()) {
     return folly::makeFuture<StorageRpcResponse<cpp2::ExecResponse>>(
@@ -133,6 +134,7 @@ StorageRpcRespFuture<cpp2::ExecResponse> StorageClient::addVertices(
     auto& req = requests[host];
     req.set_space_id(param.space);
     req.set_if_not_exists(ifNotExists);
+    req.set_ignore_existed_index(ignoreExistedIndex);
     req.set_parts(std::move(c.second));
     req.set_prop_names(propNames);
     req.set_common(common);
@@ -149,7 +151,8 @@ StorageRpcRespFuture<cpp2::ExecResponse> StorageClient::addVertices(
 StorageRpcRespFuture<cpp2::ExecResponse> StorageClient::addEdges(const CommonRequestParam& param,
                                                                  std::vector<cpp2::NewEdge> edges,
                                                                  std::vector<std::string> propNames,
-                                                                 bool ifNotExists) {
+                                                                 bool ifNotExists,
+                                                                 bool ignoreExistedIndex) {
   auto cbStatus = getIdFromNewEdge(param.space);
   if (!cbStatus.ok()) {
     return folly::makeFuture<StorageRpcResponse<cpp2::ExecResponse>>(
@@ -170,6 +173,7 @@ StorageRpcRespFuture<cpp2::ExecResponse> StorageClient::addEdges(const CommonReq
     auto& req = requests[host];
     req.set_space_id(param.space);
     req.set_if_not_exists(ifNotExists);
+    req.set_ignore_existed_index(ignoreExistedIndex);
     req.set_parts(std::move(c.second));
     req.set_prop_names(propNames);
     req.set_common(common);

src/clients/storage/StorageClient.h

@@ -81,12 +81,14 @@ class StorageClient : public StorageClientBase<cpp2::GraphStorageServiceAsyncCli
       const CommonRequestParam& param,
       std::vector<cpp2::NewVertex> vertices,
       std::unordered_map<TagID, std::vector<std::string>> propNames,
-      bool ifNotExists);
+      bool ifNotExists,
+      bool ignoreExistedIndex);
 
   StorageRpcRespFuture<cpp2::ExecResponse> addEdges(const CommonRequestParam& param,
                                                     std::vector<cpp2::NewEdge> edges,
                                                     std::vector<std::string> propNames,
-                                                    bool ifNotExists);
+                                                    bool ifNotExists,
+                                                    bool ignoreExistedIndex);
 
   StorageRpcRespFuture<cpp2::ExecResponse> deleteEdges(const CommonRequestParam& param,
                                                        std::vector<storage::cpp2::EdgeKey> edges);

src/clients/storage/StorageClientBase-inl.h

@@ -242,7 +242,6 @@ void StorageClientBase<ClientType>::getResponseImpl(
         auto client = clientsMan_->client(host, evb, false, FLAGS_storage_client_timeout_ms);
         auto spaceId = request.second.get_space_id();
         auto partsId = getReqPartsId(request.second);
-        LOG(INFO) << "Send request to storage " << host;
         remoteFunc(client.get(), request.second)
             .via(evb)
             .thenValue([spaceId, pro, this](Response&& resp) mutable {

src/graph/executor/mutate/InsertExecutor.cpp

@@ -27,7 +27,11 @@ folly::Future<Status> InsertVerticesExecutor::insertVertices() {
       ivNode->getSpace(), qctx()->rctx()->session()->id(), plan->id(), plan->isProfileEnabled());
   return qctx()
       ->getStorageClient()
-      ->addVertices(param, ivNode->getVertices(), ivNode->getPropNames(), ivNode->getIfNotExists())
+      ->addVertices(param,
+                    ivNode->getVertices(),
+                    ivNode->getPropNames(),
+                    ivNode->getIfNotExists(),
+                    ivNode->getIgnoreExistedIndex())
       .via(runner())
       .ensure([addVertTime]() {
         VLOG(1) << "Add vertices time: " << addVertTime.elapsedInUSec() << "us";
@@ -52,7 +56,11 @@ folly::Future<Status> InsertEdgesExecutor::insertEdges() {
   param.useExperimentalFeature = FLAGS_enable_experimental_feature;
   return qctx()
       ->getStorageClient()
-      ->addEdges(param, ieNode->getEdges(), ieNode->getPropNames(), ieNode->getIfNotExists())
+      ->addEdges(param,
+                 ieNode->getEdges(),
+                 ieNode->getPropNames(),
+                 ieNode->getIfNotExists(),
+                 ieNode->getIgnoreExistedIndex())
       .via(runner())
       .ensure(
           [addEdgeTime]() { VLOG(1) << "Add edge time: " << addEdgeTime.elapsedInUSec() << "us"; })

src/graph/planner/plan/Mutate.h

@@ -22,9 +22,15 @@ class InsertVertices final : public SingleDependencyNode {
                               GraphSpaceID spaceId,
                               std::vector<storage::cpp2::NewVertex> vertices,
                               std::unordered_map<TagID, std::vector<std::string>> tagPropNames,
-                              bool ifNotExists) {
-    return qctx->objPool()->add(new InsertVertices(
-        qctx, input, spaceId, std::move(vertices), std::move(tagPropNames), ifNotExists));
+                              bool ifNotExists,
+                              bool ignoreExistedIndex) {
+    return qctx->objPool()->add(new InsertVertices(qctx,
+                                                   input,
+                                                   spaceId,
+                                                   std::move(vertices),
+                                                   std::move(tagPropNames),
+                                                   ifNotExists,
+                                                   ignoreExistedIndex));
   }
 
   std::unique_ptr<PlanNodeDescription> explain() const override;
@@ -39,24 +45,29 @@ class InsertVertices final : public SingleDependencyNode {
 
   bool getIfNotExists() const { return ifNotExists_; }
 
+  bool getIgnoreExistedIndex() const { return ignoreExistedIndex_; }
+
  private:
   InsertVertices(QueryContext* qctx,
                  PlanNode* input,
                  GraphSpaceID spaceId,
                  std::vector<storage::cpp2::NewVertex> vertices,
                  std::unordered_map<TagID, std::vector<std::string>> tagPropNames,
-                 bool ifNotExists)
+                 bool ifNotExists,
+                 bool ignoreExistedIndex)
       : SingleDependencyNode(qctx, Kind::kInsertVertices, input),
         spaceId_(spaceId),
         vertices_(std::move(vertices)),
         tagPropNames_(std::move(tagPropNames)),
-        ifNotExists_(ifNotExists) {}
+        ifNotExists_(ifNotExists),
+        ignoreExistedIndex_(ignoreExistedIndex) {}
 
  private:
   GraphSpaceID spaceId_{-1};
   std::vector<storage::cpp2::NewVertex> vertices_;
   std::unordered_map<TagID, std::vector<std::string>> tagPropNames_;
   bool ifNotExists_{false};
+  bool ignoreExistedIndex_{false};
 };
 
 class InsertEdges final : public SingleDependencyNode {
@@ -67,9 +78,16 @@ class InsertEdges final : public SingleDependencyNode {
                            std::vector<storage::cpp2::NewEdge> edges,
                            std::vector<std::string> propNames,
                            bool ifNotExists,
+                           bool ignoreExistedIndex,
                            bool useChainInsert = false) {
-    return qctx->objPool()->add(new InsertEdges(
-        qctx, input, spaceId, std::move(edges), std::move(propNames), ifNotExists, useChainInsert));
+    return qctx->objPool()->add(new InsertEdges(qctx,
+                                                input,
+                                                spaceId,
+                                                std::move(edges),
+                                                std::move(propNames),
+                                                ifNotExists,
+                                                ignoreExistedIndex,
+                                                useChainInsert));
   }
 
   std::unique_ptr<PlanNodeDescription> explain() const override;
@@ -80,6 +98,8 @@ class InsertEdges final : public SingleDependencyNode {
 
   bool getIfNotExists() const { return ifNotExists_; }
 
+  bool getIgnoreExistedIndex() const { return ignoreExistedIndex_; }
+
   GraphSpaceID getSpace() const { return spaceId_; }
 
   bool useChainInsert() const { return useChainInsert_; }
@@ -91,19 +111,22 @@ class InsertEdges final : public SingleDependencyNode {
               std::vector<storage::cpp2::NewEdge> edges,
               std::vector<std::string> propNames,
               bool ifNotExists,
+              bool ignoreExistedIndex,
               bool useChainInsert)
       : SingleDependencyNode(qctx, Kind::kInsertEdges, input),
         spaceId_(spaceId),
         edges_(std::move(edges)),
         propNames_(std::move(propNames)),
         ifNotExists_(ifNotExists),
+        ignoreExistedIndex_(ignoreExistedIndex),
         useChainInsert_(useChainInsert) {}
 
  private:
   GraphSpaceID spaceId_{-1};
   std::vector<storage::cpp2::NewEdge> edges_;
   std::vector<std::string> propNames_;
   bool ifNotExists_{false};
+  bool ignoreExistedIndex_{false};
   // if this enabled, add edge request will only sent to
   // outbound edges. (toss)
   bool useChainInsert_{false};

src/graph/validator/MutateValidator.cpp

@@ -22,8 +22,13 @@ Status InsertVerticesValidator::validateImpl() {
 }
 
 Status InsertVerticesValidator::toPlan() {
-  auto doNode = InsertVertices::make(
-      qctx_, nullptr, spaceId_, std::move(vertices_), std::move(tagPropNames_), ifNotExists_);
+  auto doNode = InsertVertices::make(qctx_,
+                                     nullptr,
+                                     spaceId_,
+                                     std::move(vertices_),
+                                     std::move(tagPropNames_),
+                                     ifNotExists_,
+                                     ignoreExistedIndex_);
   root_ = doNode;
   tail_ = root_;
   return Status::OK();
@@ -32,6 +37,7 @@ Status InsertVerticesValidator::toPlan() {
 Status InsertVerticesValidator::check() {
   auto sentence = static_cast<InsertVerticesSentence *>(sentence_);
   ifNotExists_ = sentence->isIfNotExists();
+  ignoreExistedIndex_ = sentence->ignoreExistedIndex();
   rows_ = sentence->rows();
   if (rows_.empty()) {
     return Status::SemanticError("VALUES cannot be empty");
@@ -150,6 +156,7 @@ Status InsertEdgesValidator::toPlan() {
                                   std::move(edges_),
                                   std::move(entirePropNames_),
                                   ifNotExists_,
+                                  ignoreExistedIndex_,
                                   useChainInsert);
   root_ = doNode;
   tail_ = root_;
@@ -159,6 +166,7 @@ Status InsertEdgesValidator::toPlan() {
 Status InsertEdgesValidator::check() {
   auto sentence = static_cast<InsertEdgesSentence *>(sentence_);
   ifNotExists_ = sentence->isIfNotExists();
+  ignoreExistedIndex_ = sentence->ignoreExistedIndex();
   auto edgeStatus = qctx_->schemaMng()->toEdgeType(spaceId_, *sentence->edge());
   NG_RETURN_IF_ERROR(edgeStatus);
   edgeType_ = edgeStatus.value();

src/graph/validator/MutateValidator.h

@@ -35,6 +35,7 @@ class InsertVerticesValidator final : public Validator {
   std::vector<std::pair<TagID, TagSchema>> schemas_;
   uint16_t propSize_{0};
   bool ifNotExists_{false};
+  bool ignoreExistedIndex_{false};
   std::vector<storage::cpp2::NewVertex> vertices_;
 };
 
@@ -54,6 +55,7 @@ class InsertEdgesValidator final : public Validator {
  private:
   GraphSpaceID spaceId_{-1};
   bool ifNotExists_{false};
+  bool ignoreExistedIndex_{false};
   EdgeType edgeType_{-1};
   std::shared_ptr<const meta::SchemaProviderIf> schema_;
   std::vector<std::string> propNames_;

src/interface/storage.thrift

@@ -343,7 +343,8 @@ struct AddVerticesRequest {
         (cpp.template = "std::unordered_map")   prop_names,
     // if true, when (vertexID,tagID) already exists, do nothing
     4: bool                                     if_not_exists,
-    5: optional RequestCommon                   common,
+    5: bool                                     ignore_existed_index = false,
+    6: optional RequestCommon                   common,
 }
 
 struct AddEdgesRequest {
@@ -356,7 +357,9 @@ struct AddEdgesRequest {
     3: list<binary>                             prop_names,
     // if true, when edge already exists, do nothing
     4: bool                                     if_not_exists,
-    5: optional RequestCommon                   common,
+    // If true, existed index won't be removed
+    5: bool                                     ignore_existed_index = false,
+    6: optional RequestCommon                   common,
 }
 
 /*

src/parser/MutateSentences.h

@@ -136,10 +136,14 @@ class VertexRowList final {
 
 class InsertVerticesSentence final : public Sentence {
  public:
-  InsertVerticesSentence(VertexTagList *tagList, VertexRowList *rows, bool ifNotExists) {
+  InsertVerticesSentence(VertexTagList *tagList,
+                         VertexRowList *rows,
+                         bool ifNotExists,
+                         bool ignoreExistedIndex) {
     tagList_.reset(tagList);
     rows_.reset(rows);
     ifNotExists_ = ifNotExists;
+    ignoreExistedIndex_ = ignoreExistedIndex;
     kind_ = Kind::kInsertVertices;
   }
 
@@ -151,8 +155,11 @@ class InsertVerticesSentence final : public Sentence {
 
   bool isIfNotExists() const { return ifNotExists_; }
 
+  bool ignoreExistedIndex() const { return ignoreExistedIndex_; }
+
  private:
   bool ifNotExists_{false};
+  bool ignoreExistedIndex_{false};
   std::unique_ptr<VertexTagList> tagList_;
   std::unique_ptr<VertexRowList> rows_;
 };
@@ -209,11 +216,15 @@ class EdgeRowList final {
 
 class InsertEdgesSentence final : public Sentence {
  public:
-  explicit InsertEdgesSentence(std::string *edge, EdgeRowList *rows, bool ifNotExists)
+  explicit InsertEdgesSentence(std::string *edge,
+                               EdgeRowList *rows,
+                               bool ifNotExists,
+                               bool ignoreExistedIndex)
       : Sentence(Kind::kInsertEdges) {
     edge_.reset(edge);
     rows_.reset(rows);
     ifNotExists_ = ifNotExists;
+    ignoreExistedIndex_ = ignoreExistedIndex;
   }
 
   const std::string *edge() const { return edge_.get(); }
@@ -231,6 +242,8 @@ class InsertEdgesSentence final : public Sentence {
 
   bool isIfNotExists() const { return ifNotExists_; }
 
+  bool ignoreExistedIndex() const { return ignoreExistedIndex_; }
+
   void setDefaultPropNames() { isDefaultPropNames_ = true; }
 
   bool isDefaultPropNames() const { return isDefaultPropNames_; }
@@ -240,6 +253,7 @@ class InsertEdgesSentence final : public Sentence {
  private:
   bool isDefaultPropNames_{false};
   bool ifNotExists_{false};
+  bool ignoreExistedIndex_{false};
   std::unique_ptr<std::string> edge_;
   std::unique_ptr<PropertyList> properties_;
   std::unique_ptr<EdgeRowList> rows_;