Refactor storage index

src/common/utils/IndexKeyUtils.h

@@ -6,13 +6,14 @@
 #ifndef COMMON_UTILS_INDEXKEYUTILS_H_
 #define COMMON_UTILS_INDEXKEYUTILS_H_
 
+#include <cmath>
+
 #include "codec/RowReader.h"
 #include "common/base/Base.h"
 #include "common/base/StatusOr.h"
 #include "common/geo/GeoIndex.h"
 #include "common/utils/Types.h"
 #include "interface/gen-cpp2/meta_types.h"
-
 namespace nebula {
 
 using PropertyType = nebula::cpp2::PropertyType;
@@ -197,39 +198,39 @@ class IndexKeyUtils final {
    */
 
   static std::string encodeDouble(double v) {
-    if (v < 0) {
-      /**
-       *   TODO : now, the -(std::numeric_limits<double>::min())
-       *   have a problem of precision overflow. current return value is -nan.
-       */
-      auto* c1 = reinterpret_cast<const char*>(&v);
-      auto i = *reinterpret_cast<const int64_t*>(c1);
-      i = -(std::numeric_limits<int64_t>::max() + i);
-      auto* c2 = reinterpret_cast<const char*>(&i);
-      v = *reinterpret_cast<const double*>(c2);
+    if (std::isnan(v)) {
+      return std::string(sizeof(double), '\xFF');
+    } else if (v >= 0) {
+      auto val = folly::Endian::big(v);
+      auto* c = reinterpret_cast<char*>(&val);
+      c[0] |= 0x80;
+      std::string raw;
+      raw.reserve(sizeof(double));
+      raw.append(c, sizeof(double));
+      return raw;
+    } else {
+      int64_t* x = reinterpret_cast<int64_t*>(&v);
+      *x = ~(*x);
+      auto val = folly::Endian::big(v);
+      auto* c = reinterpret_cast<char*>(&val);
+      std::string raw;
+      raw.reserve(sizeof(double));
+      raw.append(c, sizeof(double));
+      return raw;
     }
-    auto val = folly::Endian::big(v);
-    auto* c = reinterpret_cast<char*>(&val);
-    c[0] ^= 0x80;
-    std::string raw;
-    raw.reserve(sizeof(double));
-    raw.append(c, sizeof(double));
-    return raw;
   }
 
   static double decodeDouble(const folly::StringPiece& raw) {
-    char* v = const_cast<char*>(raw.data());
-    v[0] ^= 0x80;
-    auto val = *reinterpret_cast<const double*>(v);
+    int64_t val = *reinterpret_cast<const int64_t*>(raw.data());
     val = folly::Endian::big(val);
     if (val < 0) {
-      auto* c1 = reinterpret_cast<const char*>(&val);
-      auto i = *reinterpret_cast<const int64_t*>(c1);
-      i = -(std::numeric_limits<int64_t>::max() + i);
-      auto* c2 = reinterpret_cast<const char*>(&i);
-      val = *reinterpret_cast<const double*>(c2);
+      val &= 0x7fffffffffffffff;
+    } else {
+      val = ~val;
     }
-    return val;
+    double ret;
+    ::memcpy(&ret, &val, 8);
+    return ret;
   }
 
   static std::string encodeTime(const nebula::Time& t) {

src/common/utils/test/CMakeLists.txt

@@ -11,6 +11,7 @@ nebula_add_test(
         $<TARGET_OBJECTS:wkt_wkb_io_obj>
         $<TARGET_OBJECTS:meta_thrift_obj>
         $<TARGET_OBJECTS:common_thrift_obj>
+        $<TARGET_OBJECTS:storage_thrift_obj>
         $<TARGET_OBJECTS:geo_index_obj>
     LIBRARIES
         gtest
@@ -30,6 +31,7 @@ nebula_add_test(
         $<TARGET_OBJECTS:wkt_wkb_io_obj>
         $<TARGET_OBJECTS:meta_thrift_obj>
         $<TARGET_OBJECTS:common_thrift_obj>
+        $<TARGET_OBJECTS:storage_thrift_obj>
         $<TARGET_OBJECTS:geo_index_obj>
     LIBRARIES
         gtest
@@ -52,6 +54,7 @@ nebula_add_test(
         $<TARGET_OBJECTS:wkt_wkb_io_obj>
         $<TARGET_OBJECTS:meta_thrift_obj>
         $<TARGET_OBJECTS:common_thrift_obj>
+        $<TARGET_OBJECTS:storage_thrift_obj>
         $<TARGET_OBJECTS:geo_index_obj>
     LIBRARIES
         gtest

src/daemons/CMakeLists.txt

@@ -45,6 +45,7 @@ set(storage_meta_deps
     $<TARGET_OBJECTS:raftex_thrift_obj>
     $<TARGET_OBJECTS:hdfs_helper_obj>
     $<TARGET_OBJECTS:ft_es_storage_adapter_obj>
+    $<TARGET_OBJECTS:storage_thrift_obj>
     $<TARGET_OBJECTS:geo_index_obj>
 )
 
@@ -128,6 +129,7 @@ nebula_add_executable(
         $<TARGET_OBJECTS:charset_obj>
         $<TARGET_OBJECTS:graph_obj>
         $<TARGET_OBJECTS:ft_es_graph_adapter_obj>
+        $<TARGET_OBJECTS:storage_thrift_obj>
         $<TARGET_OBJECTS:geo_index_obj>
         ${common_deps}
     LIBRARIES

src/graph/optimizer/OptimizerUtils.cpp

@@ -464,18 +464,7 @@ Value OptimizerUtils::normalizeValue(const meta::cpp2::ColumnDef& col, const Val
       if (!col.type.type_length_ref().has_value()) {
         return Value::kNullBadType;
       }
-      if (!v.isStr()) {
-        return v;
-      }
-      auto len = static_cast<size_t>(*col.get_type().get_type_length());
-      if (v.getStr().size() > len) {
-        return Value(v.getStr().substr(0, len));
-      } else {
-        std::string s;
-        s.reserve(len);
-        s.append(v.getStr()).append(len - v.getStr().size(), '\0');
-        return Value(std::move(s));
-      }
+      return v;
     }
     case Value::Type::__EMPTY__:
     case Value::Type::NULLVALUE:

src/interface/storage.thrift

@@ -506,6 +506,10 @@ struct IndexColumnHint {
     2: ScanType                 scan_type,
     3: common.Value             begin_value,
     4: common.Value             end_value,
+    // When `columnhint` means ` >= begin_value`, `include_begin` is true
+    // and include_end is similar
+    5: bool                     include_begin = true,
+    6: bool                     include_end = false,
 }
 
 struct IndexQueryContext {

src/kvstore/test/CMakeLists.txt

@@ -34,6 +34,7 @@ set(KVSTORE_TEST_LIBS
     $<TARGET_OBJECTS:ft_es_storage_adapter_obj>
     $<TARGET_OBJECTS:version_obj>
     $<TARGET_OBJECTS:ssl_obj>
+    $<TARGET_OBJECTS:storage_thrift_obj>
     $<TARGET_OBJECTS:geo_index_obj>
 )
 

src/storage/CMakeLists.txt

@@ -27,6 +27,7 @@ nebula_add_library(
 nebula_add_library(
     graph_storage_service_handler OBJECT
     GraphStorageServiceHandler.cpp
+    ExprVisitorBase.cpp
     context/StorageExpressionContext.cpp
     mutate/AddVerticesProcessor.cpp
     mutate/DeleteVerticesProcessor.cpp
@@ -40,6 +41,14 @@ nebula_add_library(
     query/ScanVertexProcessor.cpp
     query/ScanEdgeProcessor.cpp
     index/LookupProcessor.cpp
+    exec/IndexNode.cpp
+    exec/IndexDedupNode.cpp
+    exec/IndexEdgeScanNode.cpp
+    exec/IndexLimitNode.cpp
+    exec/IndexProjectionNode.cpp
+    exec/IndexScanNode.cpp
+    exec/IndexSelectionNode.cpp
+    exec/IndexVertexScanNode.cpp
 )
 
 nebula_add_library(

src/storage/CommonUtils.h

@@ -200,6 +200,9 @@ struct RuntimeContext {
   ObjectPool* objPool() { return &planContext_->objPool_; }
 
   bool isPlanKilled() {
+    if (env() == nullptr) {
+      return false;
+    }
     return env()->metaClient_ &&
            env()->metaClient_->checkIsPlanKilled(planContext_->sessionId_, planContext_->planId_);
   }

src/storage/ExprVisitorBase.cpp

@@ -0,0 +1,85 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License.
+ */
+#include "storage/ExprVisitorBase.h"
+namespace nebula {
+namespace storage {
+void ExprVisitorBase::visit(ConstantExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(UnaryExpression *expr) { expr->operand()->accept(this); }
+void ExprVisitorBase::visit(TypeCastingExpression *expr) { expr->operand()->accept(this); }
+void ExprVisitorBase::visit(LabelExpression *expr) { fatal(expr); }
+void ExprVisitorBase::visit(LabelAttributeExpression *expr) { fatal(expr); }
+// binary expression
+void ExprVisitorBase::visit(ArithmeticExpression *expr) {
+  expr->left()->accept(this);
+  expr->right()->accept(this);
+}
+void ExprVisitorBase::visit(RelationalExpression *expr) {
+  expr->left()->accept(this);
+  expr->right()->accept(this);
+}
+void ExprVisitorBase::visit(SubscriptExpression *expr) {
+  expr->left()->accept(this);
+  expr->right()->accept(this);
+}
+void ExprVisitorBase::visit(AttributeExpression *expr) { fatal(expr); }
+void ExprVisitorBase::visit(LogicalExpression *expr) {
+  for (auto operand : expr->operands()) {
+    operand->accept(this);
+  }
+}
+// function call
+void ExprVisitorBase::visit(FunctionCallExpression *expr) {
+  for (auto arg : expr->args()->args()) {
+    arg->accept(this);
+  }
+}
+void ExprVisitorBase::visit(AggregateExpression *expr) { fatal(expr); }
+void ExprVisitorBase::visit(UUIDExpression *expr) { UNUSED(expr); }
+// variable expression
+void ExprVisitorBase::visit(VariableExpression *expr) { fatal(expr); }
+void ExprVisitorBase::visit(VersionedVariableExpression *expr) { fatal(expr); }
+// container expression
+void ExprVisitorBase::visit(ListExpression *expr) {
+  for (auto item : expr->items()) {
+    item->accept(this);
+  }
+}
+void ExprVisitorBase::visit(SetExpression *expr) {
+  for (auto item : expr->items()) {
+    item->accept(this);
+  }
+}
+void ExprVisitorBase::visit(MapExpression *expr) { UNUSED(expr); }
+// property Expression
+void ExprVisitorBase::visit(TagPropertyExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(EdgePropertyExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(InputPropertyExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(VariablePropertyExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(DestPropertyExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(SourcePropertyExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(EdgeSrcIdExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(EdgeTypeExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(EdgeRankExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(EdgeDstIdExpression *expr) { UNUSED(expr); }
+// vertex/edge expression
+void ExprVisitorBase::visit(VertexExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(EdgeExpression *expr) { UNUSED(expr); }
+// case expression
+void ExprVisitorBase::visit(CaseExpression *expr) { UNUSED(expr); }
+// path build expression
+void ExprVisitorBase::visit(PathBuildExpression *expr) { fatal(expr); }
+// column expression
+void ExprVisitorBase::visit(ColumnExpression *expr) { fatal(expr); }
+// predicate expression
+void ExprVisitorBase::visit(PredicateExpression *expr) { fatal(expr); }
+// list comprehension expression
+void ExprVisitorBase::visit(ListComprehensionExpression *expr) { fatal(expr); }
+// reduce expression
+void ExprVisitorBase::visit(ReduceExpression *expr) { fatal(expr); }
+// subscript range expression
+void ExprVisitorBase::visit(SubscriptRangeExpression *expr) { fatal(expr); }
+
+}  // namespace storage
+}  // namespace nebula

src/storage/ExprVisitorBase.h

@@ -0,0 +1,71 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License.
+ */
+
+#pragma once
+#include "common/expression/ExprVisitor.h"
+namespace nebula {
+namespace storage {
+
+class ExprVisitorBase : public ::nebula::ExprVisitor {
+ public:
+  void visit(ConstantExpression *expr) override;
+  void visit(UnaryExpression *expr) override;
+  void visit(TypeCastingExpression *expr) override;
+  void visit(LabelExpression *expr) override;
+  void visit(LabelAttributeExpression *expr) override;
+  // binary expression
+  void visit(ArithmeticExpression *expr) override;
+  void visit(RelationalExpression *expr) override;
+  void visit(SubscriptExpression *expr) override;
+  void visit(AttributeExpression *expr) override;
+  void visit(LogicalExpression *expr) override;
+  // function call
+  void visit(FunctionCallExpression *expr) override;
+  void visit(AggregateExpression *expr) override;
+  void visit(UUIDExpression *expr) override;
+  // variable expression
+  void visit(VariableExpression *expr) override;
+  void visit(VersionedVariableExpression *expr) override;
+  // container expression
+  void visit(ListExpression *expr) override;
+  void visit(SetExpression *expr) override;
+  void visit(MapExpression *expr) override;
+  // property Expression
+  void visit(TagPropertyExpression *expr) override;
+  void visit(EdgePropertyExpression *expr) override;
+  void visit(InputPropertyExpression *expr) override;
+  void visit(VariablePropertyExpression *expr) override;
+  void visit(DestPropertyExpression *expr) override;
+  void visit(SourcePropertyExpression *expr) override;
+  void visit(EdgeSrcIdExpression *expr) override;
+  void visit(EdgeTypeExpression *expr) override;
+  void visit(EdgeRankExpression *expr) override;
+  void visit(EdgeDstIdExpression *expr) override;
+  // vertex/edge expression
+  void visit(VertexExpression *expr) override;
+  void visit(EdgeExpression *expr) override;
+  // case expression
+  void visit(CaseExpression *expr) override;
+  // path build expression
+  void visit(PathBuildExpression *expr) override;
+  // column expression
+  void visit(ColumnExpression *expr) override;
+  // predicate expression
+  void visit(PredicateExpression *expr) override;
+  // list comprehension expression
+  void visit(ListComprehensionExpression *expr) override;
+  // reduce expression
+  void visit(ReduceExpression *expr) override;
+  // subscript range expression
+  void visit(SubscriptRangeExpression *expr) override;
+
+ private:
+  using ::nebula::ExprVisitor::visit;
+  inline void fatal(Expression *expr) {
+    LOG(FATAL) << "Unexpect expression kind " << static_cast<int>(expr->kind()) << " at storage";
+  }
+};
+}  // namespace storage
+}  // namespace nebula