Cherry pick 3.5.0 (0401-0419) (#5479)

* refactor traverse output (#5464) * refactor traverse output * fix pruneproperties error & none_direct_dst * fix test error * fix shortest path * Change the compaction filter logic to let periodic compaction go through custom compaction filter, to gc expired data (#5447) * Push filter down cross join (#5473) * fix comment * push down filter through cross join --------- Co-authored-by: Sophie <[email protected]> * Fix shortest path crash (#5472) * fix crash of geo (#5475) * fix crash of geo * change log(fatal) to log(error) * fix miss arg $GITHUB_OUTPUT (#5478) * Split optimizer rules (#5470) Fix compile small rename Fix tck Fix tck fmt Fix tck Fix tck * Enhancement/optimize edge all predicate (#5481) * fix eval contains filter on storaged (#5485) * fix eval contains filter on storaged * add tck case * add tck case * fix tck * fix lint * fix lint * Fix expression util function (#5487) fmt Co-authored-by: Sophie <[email protected]> * fix ContainsFilter random fail (#5489) * Fixed graphd startup issue (#5493) * fix prunproperties (#5494) * stop the pushing down of not expressions that are not rewritten to proper forms. (#5502) * Fix edge all predicate with rank function (#5503) Co-authored-by: Sophie <[email protected]> * rewrite param in subgraph & path (#5500) * check param in subgraph * rewrite param in path --------- Co-authored-by: Sophie <[email protected]> * Fix concurrent bug about session count (#5496) * Fix regex expression (#5507) * Update requirements.txt (#5512) Solidified tomli version to solve centos7 compatibility issues * Update cluster id (#5514) --------- Co-authored-by: jimingquan <[email protected]> Co-authored-by: Ryan <[email protected]> Co-authored-by: Yee <[email protected]> Co-authored-by: jie.wang <[email protected]> Co-authored-by: George <[email protected]> Co-authored-by: kyle.cao <[email protected]> Co-authored-by: codesigner <[email protected]> Co-authored-by: dutor <[email protected]> Co-authored-by: Cheng Xuntao <[email protected]> Co-authored-by: Yichen Wang <[email protected]>
vesoft-inc · Apr 20, 2023 · 40c4bc7 · 40c4bc7
1 parent 11f018f
commit 40c4bc7
Show file tree

Hide file tree

Showing 75 changed files with 1,988 additions and 310 deletions.
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -46,7 +46,7 @@ jobs:
           find pkg-build/cpack_output -type f \( -iname \*.deb -o -iname \*.rpm -o -iname \*.tar.gz \) -exec bash -c "sha256sum {} > {}.sha256sum.txt" \;
           subdir=$(date -u +%Y.%m.%d)
           echo "subdir=$subdir" >> $GITHUB_OUTPUT
-      # - uses: actions/upload-artifact@v1
+      # - uses: actions/upload-artifact@v3
       #   with:
       #     name: ${{ matrix.os }}-nightly
       #     path: pkg-build/cpack_output

diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml
@@ -199,7 +199,7 @@ jobs:
           run: sh -c "find . -mindepth 1 -delete"
       - uses: actions/checkout@v3
       - id: tag
-        run: echo tagnum=${{ github.event.inputs.version }}
+        run: echo tagnum=${{ github.event.inputs.version }} >> $GITHUB_OUTPUT
 
       - id: oss_package
         run: |

diff --git a/resources/gflags.json b/resources/gflags.json
@@ -7,7 +7,6 @@
         "clean_wal_interval_secs",
         "wal_ttl",
         "clean_wal_interval_secs",
-        "custom_filter_interval_secs",
         "accept_partial_success",
         "system_memory_high_watermark_ratio",
         "num_rows_to_check_memory",

diff --git a/src/common/expression/Expression.h b/src/common/expression/Expression.h
@@ -167,6 +167,10 @@ class Expression {
     return false;
   }
 
+  virtual bool isPropertyExpr() const {
+    return false;
+  }
+
   virtual bool isContainerExpr() const {
     return false;
   }

diff --git a/src/common/expression/PropertyExpression.h b/src/common/expression/PropertyExpression.h
@@ -32,6 +32,10 @@ class PropertyExpression : public Expression {
  public:
   bool operator==(const Expression& rhs) const override;
 
+  bool isPropertyExpr() const override {
+    return true;
+  }
+
   const Value& eval(ExpressionContext& ctx) override;
 
   const std::string& ref() const {

diff --git a/src/common/expression/RelationalExpression.cpp b/src/common/expression/RelationalExpression.cpp
@@ -37,7 +37,8 @@ const Value& RelationalExpression::eval(ExpressionContext& ctx) {
     case Kind::kRelREG: {
       if (lhs.isBadNull() || rhs.isBadNull()) {
         result_ = Value::kNullBadType;
-      } else if ((!lhs.isNull() && !lhs.isStr()) || (!rhs.isNull() && !rhs.isStr())) {
+      } else if ((!lhs.isNull() && !lhs.empty() && !lhs.isStr()) ||
+                 (!rhs.isNull() && !rhs.empty() && !rhs.isStr())) {
         result_ = Value::kNullBadType;
       } else if (lhs.isStr() && rhs.isStr()) {
         try {
@@ -115,7 +116,8 @@ const Value& RelationalExpression::eval(ExpressionContext& ctx) {
     case Kind::kContains: {
       if (lhs.isBadNull() || rhs.isBadNull()) {
         result_ = Value::kNullBadType;
-      } else if ((!lhs.isNull() && !lhs.isStr()) || (!rhs.isNull() && !rhs.isStr())) {
+      } else if ((!lhs.isNull() && !lhs.empty() && !lhs.isStr()) ||
+                 (!rhs.isNull() && !rhs.empty() && !rhs.isStr())) {
         result_ = Value::kNullBadType;
       } else if (lhs.isStr() && rhs.isStr()) {
         result_ = lhs.getStr().size() >= rhs.getStr().size() &&
@@ -128,7 +130,8 @@ const Value& RelationalExpression::eval(ExpressionContext& ctx) {
     case Kind::kNotContains: {
       if (lhs.isBadNull() || rhs.isBadNull()) {
         result_ = Value::kNullBadType;
-      } else if ((!lhs.isNull() && !lhs.isStr()) || (!rhs.isNull() && !rhs.isStr())) {
+      } else if ((!lhs.isNull() && !lhs.empty() && !lhs.isStr()) ||
+                 (!rhs.isNull() && !rhs.empty() && !rhs.isStr())) {
         result_ = Value::kNullBadType;
       } else if (lhs.isStr() && rhs.isStr()) {
         result_ = !(lhs.getStr().size() >= rhs.getStr().size() &&
@@ -141,7 +144,8 @@ const Value& RelationalExpression::eval(ExpressionContext& ctx) {
     case Kind::kStartsWith: {
       if (lhs.isBadNull() || rhs.isBadNull()) {
         result_ = Value::kNullBadType;
-      } else if ((!lhs.isNull() && !lhs.isStr()) || (!rhs.isNull() && !rhs.isStr())) {
+      } else if ((!lhs.isNull() && !lhs.empty() && !lhs.isStr()) ||
+                 (!rhs.isNull() && !rhs.empty() && !rhs.isStr())) {
         result_ = Value::kNullBadType;
       } else if (lhs.isStr() && rhs.isStr()) {
         result_ =
@@ -154,7 +158,8 @@ const Value& RelationalExpression::eval(ExpressionContext& ctx) {
     case Kind::kNotStartsWith: {
       if (lhs.isBadNull() || rhs.isBadNull()) {
         result_ = Value::kNullBadType;
-      } else if ((!lhs.isNull() && !lhs.isStr()) || (!rhs.isNull() && !rhs.isStr())) {
+      } else if ((!lhs.isNull() && !lhs.empty() && !lhs.isStr()) ||
+                 (!rhs.isNull() && !rhs.empty() && !rhs.isStr())) {
         result_ = Value::kNullBadType;
       } else if (lhs.isStr() && rhs.isStr()) {
         result_ =
@@ -167,7 +172,8 @@ const Value& RelationalExpression::eval(ExpressionContext& ctx) {
     case Kind::kEndsWith: {
       if (lhs.isBadNull() || rhs.isBadNull()) {
         result_ = Value::kNullBadType;
-      } else if ((!lhs.isNull() && !lhs.isStr()) || (!rhs.isNull() && !rhs.isStr())) {
+      } else if ((!lhs.isNull() && !lhs.empty() && !lhs.isStr()) ||
+                 (!rhs.isNull() && !rhs.empty() && !rhs.isStr())) {
         result_ = Value::kNullBadType;
       } else if (lhs.isStr() && rhs.isStr()) {
         result_ =
@@ -182,7 +188,8 @@ const Value& RelationalExpression::eval(ExpressionContext& ctx) {
     case Kind::kNotEndsWith: {
       if (lhs.isBadNull() || rhs.isBadNull()) {
         result_ = Value::kNullBadType;
-      } else if ((!lhs.isNull() && !lhs.isStr()) || (!rhs.isNull() && !rhs.isStr())) {
+      } else if ((!lhs.isNull() && !lhs.empty() && !lhs.isStr()) ||
+                 (!rhs.isNull() && !rhs.empty() && !rhs.isStr())) {
         result_ = Value::kNullBadType;
       } else if (lhs.isStr() && rhs.isStr()) {
         result_ = !(lhs.getStr().size() >= rhs.getStr().size() &&

diff --git a/src/common/function/FunctionManager.cpp b/src/common/function/FunctionManager.cpp
@@ -2017,7 +2017,7 @@ FunctionManager::FunctionManager() {
     // More information of encoding could be found in `NebulaKeyUtils.h`
     auto &attr = functions_["none_direct_dst"];
     attr.minArity_ = 1;
-    attr.maxArity_ = 1;
+    attr.maxArity_ = 2;
     attr.isAlwaysPure_ = true;
     attr.body_ = [](const auto &args) -> Value {
       switch (args[0].get().type()) {
@@ -2035,6 +2035,13 @@ FunctionManager::FunctionManager() {
         case Value::Type::LIST: {
           const auto &listVal = args[0].get().getList().values;
           if (listVal.empty()) {
+            if (args.size() == 2) {
+              if (args[1].get().type() == Value::Type::VERTEX) {
+                const auto &v = args[1].get().getVertex();
+                return v.vid;
+              }
+              return Value::kNullBadType;
+            }
             return Value::kNullBadType;
           }
           auto &lastVal = listVal.back();

diff --git a/src/common/meta/GflagsManager.cpp b/src/common/meta/GflagsManager.cpp
@@ -58,7 +58,6 @@ std::unordered_map<std::string, std::pair<cpp2::ConfigMode, bool>> GflagsManager
       {"meta_client_retry_times", {cpp2::ConfigMode::MUTABLE, false}},
       {"wal_ttl", {cpp2::ConfigMode::MUTABLE, false}},
       {"clean_wal_interval_secs", {cpp2::ConfigMode::MUTABLE, false}},
-      {"custom_filter_interval_secs", {cpp2::ConfigMode::MUTABLE, false}},
       {"accept_partial_success", {cpp2::ConfigMode::MUTABLE, false}},
 
       {"rocksdb_db_options", {cpp2::ConfigMode::MUTABLE, true}},

diff --git a/src/common/meta/SchemaManager.h b/src/common/meta/SchemaManager.h
@@ -40,13 +40,33 @@ class SchemaManager {
 
   virtual StatusOr<int32_t> getPartsNum(GraphSpaceID space) = 0;
 
+  std::shared_ptr<const NebulaSchemaProvider> getTagSchema(GraphSpaceID space,
+                                                           const std::string& tag,
+                                                           SchemaVer ver = -1) {
+    auto tagId = toTagID(space, tag);
+    if (!tagId.ok()) {
+      return nullptr;
+    }
+    return getTagSchema(space, tagId.value(), ver);
+  }
+
   virtual std::shared_ptr<const NebulaSchemaProvider> getTagSchema(GraphSpaceID space,
                                                                    TagID tag,
                                                                    SchemaVer ver = -1) = 0;
 
   // Returns a negative number when the schema does not exist
   virtual StatusOr<SchemaVer> getLatestTagSchemaVersion(GraphSpaceID space, TagID tag) = 0;
 
+  std::shared_ptr<const NebulaSchemaProvider> getEdgeSchema(GraphSpaceID space,
+                                                            const std::string& edge,
+                                                            SchemaVer ver = -1) {
+    auto edgeType = toEdgeType(space, edge);
+    if (!edgeType.ok()) {
+      return nullptr;
+    }
+    return getEdgeSchema(space, edgeType.value(), ver);
+  }
+
   virtual std::shared_ptr<const NebulaSchemaProvider> getEdgeSchema(GraphSpaceID space,
                                                                     EdgeType edge,
                                                                     SchemaVer ver = -1) = 0;

diff --git a/src/common/session/SessionManager.h b/src/common/session/SessionManager.h
@@ -6,6 +6,7 @@
 #ifndef COMMON_SESSION_SESSIONMANAGER_H_
 #define COMMON_SESSION_SESSIONMANAGER_H_
 
+#include <folly/RWSpinLock.h>
 #include <folly/concurrency/ConcurrentHashMap.h>
 
 #include "clients/meta/MetaClient.h"
@@ -23,7 +24,7 @@ namespace nebula {
 
 class SessionCount {
  private:
-  std::atomic<int32_t> count_ = 1;
+  std::atomic<int32_t> count_{0};
 
  public:
   int fetch_add(int step) {
@@ -75,11 +76,48 @@ class SessionManager {
  protected:
   using SessionPtr = std::shared_ptr<SessionType>;
   using SessionCountPtr = std::shared_ptr<SessionCount>;
+
+  // Get session count pointer according to key
+  SessionCountPtr sessionCnt(const std::string& key) {
+    folly::RWSpinLock::ReadHolder rh(&sessCntLock_);
+    auto iter = userIpSessionCount_.find(key);
+    if (iter != userIpSessionCount_.end()) {
+      return iter->second;
+    }
+    return nullptr;
+  }
+
+  // add sessionCount
+  void addSessionCount(std::string& key) {
+    auto sessCntPtr = sessionCnt(key);
+    if (!sessCntPtr) {
+      folly::RWSpinLock::WriteHolder wh(&sessCntLock_);
+      auto iter = userIpSessionCount_.emplace(key, std::make_shared<SessionCount>());
+      sessCntPtr = iter.first->second;
+    }
+    sessCntPtr->fetch_add(1);
+  }
+
+  // sub sessionCount
+  void subSessionCount(std::string& key) {
+    auto countFindPtr = sessionCnt(key);
+    if (countFindPtr) {
+      auto count = countFindPtr->fetch_sub(1);
+      if (count == 1) {
+        folly::RWSpinLock::WriteHolder wh(&sessCntLock_);
+        userIpSessionCount_.erase(key);
+      }
+    }
+  }
+
   folly::ConcurrentHashMap<SessionID, SessionPtr> activeSessions_;
-  folly::ConcurrentHashMap<std::string, SessionCountPtr> userIpSessionCount_;
   std::unique_ptr<thread::GenericWorker> scavenger_;
   meta::MetaClient* metaClient_{nullptr};
   HostAddr myAddr_;
+
+ private:
+  folly::RWSpinLock sessCntLock_;
+  std::unordered_map<std::string, SessionCountPtr> userIpSessionCount_;
 };
 
 }  // namespace nebula

diff --git a/src/graph/context/ast/CypherAstContext.h b/src/graph/context/ast/CypherAstContext.h
@@ -51,6 +51,8 @@ struct EdgeInfo {
   MatchEdge::Direction direction{MatchEdge::Direction::OUT_EDGE};
   std::vector<std::string> types;
   std::string alias;
+  // use for construct path struct
+  std::string innerAlias;
   const MapExpression* props{nullptr};
   Expression* filter{nullptr};
 };

diff --git a/src/graph/executor/algo/BatchShortestPath.cpp b/src/graph/executor/algo/BatchShortestPath.cpp
@@ -16,6 +16,9 @@ namespace graph {
 folly::Future<Status> BatchShortestPath::execute(const HashSet& startVids,
                                                  const HashSet& endVids,
                                                  DataSet* result) {
+  if (maxStep_ == 0) {
+    return Status::OK();
+  }
   // MemoryTrackerVerified
   size_t rowSize = init(startVids, endVids);
   std::vector<folly::Future<Status>> futures;

diff --git a/src/graph/executor/algo/CartesianProductExecutor.cpp b/src/graph/executor/algo/CartesianProductExecutor.cpp
@@ -5,6 +5,7 @@
 #include "graph/executor/algo/CartesianProductExecutor.h"
 
 #include "graph/planner/plan/Algo.h"
+#include "graph/planner/plan/Query.h"
 
 namespace nebula {
 namespace graph {

diff --git a/src/graph/executor/algo/SingleShortestPath.cpp b/src/graph/executor/algo/SingleShortestPath.cpp
@@ -14,6 +14,9 @@ namespace graph {
 folly::Future<Status> SingleShortestPath::execute(const HashSet& startVids,
                                                   const HashSet& endVids,
                                                   DataSet* result) {
+  if (maxStep_ == 0) {
+    return Status::OK();
+  }
   size_t rowSize = startVids.size() * endVids.size();
   init(startVids, endVids, rowSize);
   std::vector<folly::Future<Status>> futures;