diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index d4766213d32..1d56abdce57 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -346,6 +346,8 @@ bool MetaClient::loadSchemas(GraphSpaceID spaceId, bool hasDef = col.default_value_ref().has_value(); auto& colType = col.get_type(); size_t len = colType.type_length_ref().has_value() ? *colType.get_type_length() : 0; + cpp2::GeoShape geoShape = + colType.geo_shape_ref().has_value() ? *colType.get_geo_shape() : cpp2::GeoShape::ANY; bool nullable = col.nullable_ref().has_value() ? *col.get_nullable() : false; Expression* defaultValueExpr = nullptr; if (hasDef) { @@ -359,8 +361,12 @@ bool MetaClient::loadSchemas(GraphSpaceID spaceId, } } - schema->addField( - col.get_name(), colType.get_type(), len, nullable, hasDef ? defaultValueExpr : nullptr); + schema->addField(col.get_name(), + colType.get_type(), + len, + nullable, + hasDef ? defaultValueExpr : nullptr, + geoShape); }; for (auto& tagIt : tagItemVec) { diff --git a/src/codec/RowReaderV2.cpp b/src/codec/RowReaderV2.cpp index 0e8bb247c75..39442ac30ca 100644 --- a/src/codec/RowReaderV2.cpp +++ b/src/codec/RowReaderV2.cpp @@ -176,8 +176,22 @@ Value RowReaderV2::getValueByIndex(const int64_t index) const noexcept { return dt; } case meta::cpp2::PropertyType::GEOGRAPHY: { - // TODO(jie) - return Geography(); + int32_t strOffset; + int32_t strLen; + memcpy(reinterpret_cast(&strOffset), &data_[offset], sizeof(int32_t)); + memcpy(reinterpret_cast(&strLen), &data_[offset + sizeof(int32_t)], sizeof(int32_t)); + if (static_cast(strOffset) == data_.size() && strLen == 0) { + return Value::kEmpty; // Is it ok to return Value::kEmpty? + } + CHECK_LT(strOffset, data_.size()); + auto wkb = std::string(&data_[strOffset], strLen); + // Parse a geography from the wkb, normalize it and then verify its validity. + auto geogRet = Geography::fromWKB(wkb, true, true); + if (!geogRet.ok()) { + LOG(ERROR) << "Geography::fromWKB failed: " << geogRet.status(); + return Value::kNullBadData; // Is it ok to return Value::kNullBadData? + } + return std::move(geogRet).value(); } case meta::cpp2::PropertyType::UNKNOWN: break; diff --git a/src/codec/RowWriterV2.cpp b/src/codec/RowWriterV2.cpp index 1953e63d29a..a7073e41393 100644 --- a/src/codec/RowWriterV2.cpp +++ b/src/codec/RowWriterV2.cpp @@ -127,7 +127,7 @@ RowWriterV2::RowWriterV2(RowReader& reader) : RowWriterV2(reader.getSchema()) { set(i, v.moveDateTime()); break; case Value::Type::GEOGRAPHY: - // TODO(jie) + set(i, v.moveGeography()); break; default: LOG(FATAL) << "Invalid data: " << v << ", type: " << v.typeName(); @@ -207,8 +207,7 @@ WriteResult RowWriterV2::setValue(ssize_t index, const Value& val) noexcept { case Value::Type::DATETIME: return write(index, val.getDateTime()); case Value::Type::GEOGRAPHY: - // TODO(jie) - return WriteResult::TYPE_MISMATCH; + return write(index, val.getGeography()); default: return WriteResult::TYPE_MISMATCH; } @@ -762,6 +761,17 @@ WriteResult RowWriterV2::write(ssize_t index, const DateTime& v) noexcept { return WriteResult::SUCCEEDED; } +WriteResult RowWriterV2::write(ssize_t index, const Geography& v) noexcept { + auto field = schema_->field(index); + auto geoShape = field->geoShape(); + if (geoShape != meta::cpp2::GeoShape::ANY && + folly::to(geoShape) != folly::to(v.shape())) { + return WriteResult::TYPE_MISMATCH; + } + std::string wkb = v.asWKB(); + return write(index, folly::StringPiece(wkb)); +} + WriteResult RowWriterV2::checkUnsetFields() noexcept { DefaultValueContext expCtx; for (size_t i = 0; i < schema_->getNumFields(); i++) { @@ -802,7 +812,7 @@ WriteResult RowWriterV2::checkUnsetFields() noexcept { r = write(i, defVal.getDateTime()); break; case Value::Type::GEOGRAPHY: - // TODO(jie) + r = write(i, defVal.getGeography()); break; default: LOG(FATAL) << "Unsupported default value type: " << defVal.typeName() diff --git a/src/codec/RowWriterV2.h b/src/codec/RowWriterV2.h index d3bec72064f..16030a00afd 100644 --- a/src/codec/RowWriterV2.h +++ b/src/codec/RowWriterV2.h @@ -63,6 +63,7 @@ enum class WriteResult { TIMESTAMP (8 bytes) DATE (4 bytes) DATETIME (15 bytes) + GEOGRAPHY (8 bytes) * All except STRING typed properties are stored in-place. The STRING property stored the offset of the string content in the first 4 bytes and the length @@ -188,6 +189,8 @@ class RowWriterV2 { WriteResult write(ssize_t index, const Date& v) noexcept; WriteResult write(ssize_t index, const Time& v) noexcept; WriteResult write(ssize_t index, const DateTime& v) noexcept; + + WriteResult write(ssize_t index, const Geography& v) noexcept; }; } // namespace nebula diff --git a/src/codec/test/ResultSchemaProvider.cpp b/src/codec/test/ResultSchemaProvider.cpp index 359ca2bc80f..80b23ae1b26 100644 --- a/src/codec/test/ResultSchemaProvider.cpp +++ b/src/codec/test/ResultSchemaProvider.cpp @@ -23,14 +23,16 @@ ResultSchemaProvider::ResultSchemaField::ResultSchemaField(std::string name, bool nullable, int32_t offset, size_t nullFlagPos, - Expression* defaultValue) + Expression* defaultValue, + meta::cpp2::GeoShape geoShape) : name_(std::move(name)), type_(type), size_(size), nullable_(nullable), offset_(offset), nullFlagPos_(nullFlagPos), - defaultValue_(defaultValue) {} + defaultValue_(defaultValue), + geoShape_(geoShape) {} const char* ResultSchemaProvider::ResultSchemaField::name() const { return name_.c_str(); } @@ -50,6 +52,8 @@ size_t ResultSchemaProvider::ResultSchemaField::offset() const { return offset_; size_t ResultSchemaProvider::ResultSchemaField::nullFlagPos() const { return nullFlagPos_; } +meta::cpp2::GeoShape ResultSchemaProvider::ResultSchemaField::geoShape() const { return geoShape_; } + /*********************************** * * ResultSchemaProvider diff --git a/src/codec/test/ResultSchemaProvider.h b/src/codec/test/ResultSchemaProvider.h index f0a6bff44ca..4c51288163e 100644 --- a/src/codec/test/ResultSchemaProvider.h +++ b/src/codec/test/ResultSchemaProvider.h @@ -22,7 +22,8 @@ class ResultSchemaProvider : public meta::SchemaProviderIf { bool nullable, int32_t offset, size_t nullFlagPos, - Expression* defaultValue = nullptr); + Expression* defaultValue = nullptr, + meta::cpp2::GeoShape = meta::cpp2::GeoShape::ANY); const char* name() const override; meta::cpp2::PropertyType type() const override; @@ -32,6 +33,7 @@ class ResultSchemaProvider : public meta::SchemaProviderIf { size_t size() const override; size_t offset() const override; size_t nullFlagPos() const override; + meta::cpp2::GeoShape geoShape() const override; private: std::string name_; @@ -41,6 +43,7 @@ class ResultSchemaProvider : public meta::SchemaProviderIf { int32_t offset_; size_t nullFlagPos_; Expression* defaultValue_; + meta::cpp2::GeoShape geoShape_; }; public: diff --git a/src/codec/test/SchemaWriter.cpp b/src/codec/test/SchemaWriter.cpp index d7c11ebe1eb..41e56bd22cd 100644 --- a/src/codec/test/SchemaWriter.cpp +++ b/src/codec/test/SchemaWriter.cpp @@ -17,7 +17,8 @@ SchemaWriter& SchemaWriter::appendCol(folly::StringPiece name, PropertyType type, int32_t fixedStrLen, bool nullable, - Expression* defaultValue) noexcept { + Expression* defaultValue, + meta::cpp2::GeoShape geoShape) noexcept { using folly::hash::SpookyHashV2; uint64_t hash = SpookyHashV2::Hash64(name.data(), name.size(), 0); DCHECK(nameIndex_.find(hash) == nameIndex_.end()); @@ -87,7 +88,8 @@ SchemaWriter& SchemaWriter::appendCol(folly::StringPiece name, nullFlagPos = numNullableFields_++; } - columns_.emplace_back(name.toString(), type, size, nullable, offset, nullFlagPos, defaultValue); + columns_.emplace_back( + name.toString(), type, size, nullable, offset, nullFlagPos, defaultValue, geoShape); nameIndex_.emplace(std::make_pair(hash, columns_.size() - 1)); return *this; diff --git a/src/codec/test/SchemaWriter.h b/src/codec/test/SchemaWriter.h index 516d120f519..55c5dfb356d 100644 --- a/src/codec/test/SchemaWriter.h +++ b/src/codec/test/SchemaWriter.h @@ -21,7 +21,8 @@ class SchemaWriter : public ResultSchemaProvider { meta::cpp2::PropertyType type, int32_t fixedStrLen = 0, bool nullable = false, - Expression* defaultValue = nullptr) noexcept; + Expression* defaultValue = nullptr, + meta::cpp2::GeoShape geoShape = meta::cpp2::GeoShape::ANY) noexcept; private: }; diff --git a/src/common/datatypes/Geography.cpp b/src/common/datatypes/Geography.cpp index 922e2525301..aef9a457599 100644 --- a/src/common/datatypes/Geography.cpp +++ b/src/common/datatypes/Geography.cpp @@ -19,20 +19,25 @@ namespace nebula { +constexpr double kMaxLongitude = 180.0; +constexpr double kMaxLatitude = 90.0; + void Coordinate::normalize() { // Reduce the x(longitude) to the range [-180, 180] degrees x = std::remainder(x, 360.0); // Reduce the y(latitude) to the range [-90, 90] degrees double tmp = remainder(y, 360.0); - if (tmp > 90.0) { + if (tmp > kMaxLatitude) { y = 180.0 - tmp; - } else if (tmp < -90.0) { + } else if (tmp < -kMaxLatitude) { y = -180.0 - tmp; } } -bool Coordinate::isValid() const { return std::abs(x) <= 180.0 && std::abs(y) <= 90.0; } +bool Coordinate::isValid() const { + return std::abs(x) <= kMaxLongitude && std::abs(y) <= kMaxLatitude; +} void Point::normalize() {} @@ -46,6 +51,7 @@ bool LineString::isValid() const { return false; } auto s2Region = geo::GeoUtils::s2RegionFromGeography(*this); + CHECK_NOTNULL(s2Region); return static_cast(s2Region.get())->IsValid(); } @@ -67,6 +73,7 @@ bool Polygon::isValid() const { } } auto s2Region = geo::GeoUtils::s2RegionFromGeography(*this); + CHECK_NOTNULL(s2Region); return static_cast(s2Region.get())->IsValid(); } @@ -91,6 +98,27 @@ StatusOr Geography::fromWKT(const std::string& wkt, return geog; } +StatusOr Geography::fromWKB(const std::string& wkb, + bool needNormalize, + bool verifyValidity) { + auto geogRet = geo::WKBReader().read(wkb); + if (!geogRet.ok()) { + return geogRet; + } + auto geog = std::move(geogRet).value(); + if (needNormalize) { + geog.normalize(); + } + if (verifyValidity) { + if (!geog.isValid()) { + return Status::Error("Failed to parse an valid Geography instance from the wkb `%s'", + wkb.c_str()); + } + } + + return geog; +} + GeoShape Geography::shape() const { switch (geo_.index()) { case 0: @@ -184,6 +212,32 @@ bool Geography::isValid() const { } } +Point Geography::centroid() const { + switch (shape()) { + case GeoShape::POINT: { + return this->point(); + } + case GeoShape::LINESTRING: { + auto s2Region = geo::GeoUtils::s2RegionFromGeography(*this); + CHECK_NOTNULL(s2Region); + S2Point s2Point = static_cast(s2Region.get())->GetCentroid(); + return Point(geo::GeoUtils::coordinateFromS2Point(s2Point)); + } + case GeoShape::POLYGON: { + auto s2Region = geo::GeoUtils::s2RegionFromGeography(*this); + CHECK_NOTNULL(s2Region); + S2Point s2Point = static_cast(s2Region.get())->GetCentroid(); + return Point(geo::GeoUtils::coordinateFromS2Point(s2Point)); + } + case GeoShape::UNKNOWN: + default: { + LOG(ERROR) + << "Geography shapes other than Point/LineString/Polygon are not currently supported"; + return Point(); + } + } +} + std::string Geography::asWKT() const { return geo::WKTWriter().write(*this); } std::string Geography::asWKB() const { return geo::WKBWriter().write(*this); } diff --git a/src/common/datatypes/Geography.h b/src/common/datatypes/Geography.h index 5f2ecd31790..5e9b92de31c 100644 --- a/src/common/datatypes/Geography.h +++ b/src/common/datatypes/Geography.h @@ -58,8 +58,9 @@ struct Coordinate { } void __clear() { clear(); } - // TODO(jie) compare double correctly - bool operator==(const Coordinate& rhs) const { return x == rhs.x && y == rhs.y; } + bool operator==(const Coordinate& rhs) const { + return std::abs(x - rhs.x) < kEpsilon && std::abs(y - rhs.y) < kEpsilon; + } bool operator!=(const Coordinate& rhs) const { return !(*this == rhs); } bool operator<(const Coordinate& rhs) const { if (x != rhs.x) { @@ -135,6 +136,10 @@ struct Geography { bool needNormalize = false, bool verifyValidity = false); + static StatusOr fromWKB(const std::string& wkb, + bool needNormalize = false, + bool verifyValidity = false); + Geography() {} Geography(const Point& v) : geo_(v) {} // NOLINT Geography(Point&& v) : geo_(std::move(v)) {} // NOLINT @@ -156,6 +161,8 @@ struct Geography { void normalize(); bool isValid() const; + Point centroid() const; + std::string asWKT() const; std::string asWKB() const; diff --git a/src/common/datatypes/test/GeographyTest.cpp b/src/common/datatypes/test/GeographyTest.cpp index 0b15b0da36c..d76c90dbf2b 100644 --- a/src/common/datatypes/test/GeographyTest.cpp +++ b/src/common/datatypes/test/GeographyTest.cpp @@ -21,14 +21,14 @@ TEST(Geography, shape) { EXPECT_EQ(GeoShape::POINT, g.shape()); } { - std::string wkt = "LINESTRING(28.4 79.20,134.25 -28.34)"; + std::string wkt = "LINESTRING(28.4 79.20, 134.25 -28.34)"; auto gRet = Geography::fromWKT(wkt); ASSERT_TRUE(gRet.ok()); auto g = gRet.value(); EXPECT_EQ(GeoShape::LINESTRING, g.shape()); } { - std::string wkt = "POLYGON((1 2,3 4,5 6,1 2))"; + std::string wkt = "POLYGON((1 2, 3 4, 5 6, 1 2))"; auto gRet = Geography::fromWKT(wkt); ASSERT_TRUE(gRet.ok()); auto g = gRet.value(); @@ -46,7 +46,7 @@ TEST(Geography, asWKT) { EXPECT_EQ(wkt, got); } { - std::string wkt = "LINESTRING(28.4 79.2,134.25 -28.34)"; + std::string wkt = "LINESTRING(28.4 79.2, 134.25 -28.34)"; auto gRet = Geography::fromWKT(wkt); ASSERT_TRUE(gRet.ok()); auto g = gRet.value(); @@ -54,7 +54,7 @@ TEST(Geography, asWKT) { EXPECT_EQ(wkt, got); } { - std::string wkt = "POLYGON((1 2,3 4,5 6,1 2))"; + std::string wkt = "POLYGON((1 2, 3 4, 5 6, 1 2))"; auto gRet = Geography::fromWKT(wkt); ASSERT_TRUE(gRet.ok()); auto g = gRet.value(); diff --git a/src/common/function/FunctionManager.cpp b/src/common/function/FunctionManager.cpp index a25c38fdcf2..07b402e74bb 100644 --- a/src/common/function/FunctionManager.cpp +++ b/src/common/function/FunctionManager.cpp @@ -334,9 +334,15 @@ std::unordered_map> FunctionManager::typ { TypeSignature({Value::Type::GEOGRAPHY}, Value::Type::STRING), }}, - {"st_asbinary", + // This function requires binary data to be support first. + // {"st_asbinary", + // { + // TypeSignature({Value::Type::GEOGRAPHY}, Value::Type::STRING), + // }}, + // geo transformations + {"st_centroid", { - TypeSignature({Value::Type::GEOGRAPHY}, Value::Type::STRING), + TypeSignature({Value::Type::GEOGRAPHY}, Value::Type::GEOGRAPHY), }}, // geo accessors {"st_isvalid", @@ -362,6 +368,11 @@ std::unordered_map> FunctionManager::typ { TypeSignature({Value::Type::GEOGRAPHY, Value::Type::GEOGRAPHY, Value::Type::FLOAT}, Value::Type::BOOL), + TypeSignature({Value::Type::GEOGRAPHY, + Value::Type::GEOGRAPHY, + Value::Type::FLOAT, + Value::Type::BOOL}, + Value::Type::BOOL), }}, // geo measures {"st_distance", @@ -2401,8 +2412,22 @@ FunctionManager::FunctionManager() { return g.asWKT(); }; } + // { + // auto &attr = functions_["st_asbinary"]; + // attr.minArity_ = 1; + // attr.maxArity_ = 1; + // attr.isPure_ = true; + // attr.body_ = [](const auto &args) -> Value { + // if (!args[0].get().isGeography()) { + // return Value::kNullBadType; + // } + // const Geography &g = args[0].get().getGeography(); + // return g.asWKBHex(); + // }; + // } + // geo transformations { - auto &attr = functions_["st_asbinary"]; + auto &attr = functions_["st_centroid"]; attr.minArity_ = 1; attr.maxArity_ = 1; attr.isPure_ = true; @@ -2410,8 +2435,7 @@ FunctionManager::FunctionManager() { if (!args[0].get().isGeography()) { return Value::kNullBadType; } - const Geography &g = args[0].get().getGeography(); - return g.asWKBHex(); + return Geography(args[0].get().getGeography().centroid()); }; } // geo accessors @@ -2469,17 +2493,24 @@ FunctionManager::FunctionManager() { { auto &attr = functions_["st_dwithin"]; attr.minArity_ = 3; - attr.maxArity_ = 3; + attr.maxArity_ = 4; attr.isPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isGeography() || !args[1].get().isGeography() || !args[2].get().isFloat()) { return Value::kNullBadType; } + bool exclusive = false; + if (args.size() == 4) { + if (!args[3].get().isBool()) { + return Value::kNullBadType; + } + exclusive = args[3].get().getBool(); + } return geo::GeoFunction::dWithin(args[0].get().getGeography(), args[1].get().getGeography(), args[2].get().getFloat(), - true); + exclusive); }; } // geo measures @@ -2515,6 +2546,11 @@ FunctionManager::FunctionManager() { return Value::kNullBadData; } } + const auto &geog = args[0].get().getGeography(); + if (geog.shape() != GeoShape::POINT) { + LOG(ERROR) << "S2_CellIdFromPoint only accepts point argument"; + return Value::kNullBadData; + } // TODO(jie) Should return uint64_t Value uint64_t cellId = geo::GeoFunction::s2CellIdFromPoint(args[0].get().getGeography(), level); const char *tmp = reinterpret_cast(&cellId); diff --git a/src/common/geo/CMakeLists.txt b/src/common/geo/CMakeLists.txt index f7eafd34bb0..a9645de0833 100644 --- a/src/common/geo/CMakeLists.txt +++ b/src/common/geo/CMakeLists.txt @@ -3,5 +3,10 @@ # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. +nebula_add_library( + geo_index_obj OBJECT + GeoIndex.cpp +) + nebula_add_subdirectory(io) nebula_add_subdirectory(test) diff --git a/src/common/geo/GeoFunction.cpp b/src/common/geo/GeoFunction.cpp index 84bdc7ecb6d..5ceedb47b54 100644 --- a/src/common/geo/GeoFunction.cpp +++ b/src/common/geo/GeoFunction.cpp @@ -23,7 +23,7 @@ namespace geo { bool GeoFunction::intersects(const Geography& a, const Geography& b) { auto aRegion = a.asS2(); auto bRegion = b.asS2(); - if (!aRegion || !bRegion) { + if (UNLIKELY(!aRegion || !bRegion)) { return false; } @@ -99,7 +99,7 @@ bool GeoFunction::intersects(const Geography& a, const Geography& b) { bool GeoFunction::covers(const Geography& a, const Geography& b) { auto aRegion = a.asS2(); auto bRegion = b.asS2(); - if (!aRegion || !bRegion) { + if (UNLIKELY(!aRegion || !bRegion)) { return false; } @@ -178,10 +178,10 @@ bool GeoFunction::covers(const Geography& a, const Geography& b) { bool GeoFunction::coveredBy(const Geography& a, const Geography& b) { return covers(b, a); } -bool GeoFunction::dWithin(const Geography& a, const Geography& b, double distance, bool inclusive) { +bool GeoFunction::dWithin(const Geography& a, const Geography& b, double distance, bool exclusive) { auto aRegion = a.asS2(); auto bRegion = b.asS2(); - if (!aRegion || !bRegion) { + if (UNLIKELY(!aRegion || !bRegion)) { return false; } @@ -192,15 +192,15 @@ bool GeoFunction::dWithin(const Geography& a, const Geography& b, double distanc case GeoShape::POINT: { const S2Point& bPoint = static_cast(bRegion.get())->point(); double closestDistance = S2Earth::GetDistanceMeters(aPoint, bPoint); - return inclusive ? closestDistance <= distance : closestDistance < distance; + return exclusive ? closestDistance < distance : closestDistance <= distance; } case GeoShape::LINESTRING: { S2Polyline* bLine = static_cast(bRegion.get()); - return s2PointAndS2PolylineAreWithinDistance(aPoint, bLine, distance, inclusive); + return s2PointAndS2PolylineAreWithinDistance(aPoint, bLine, distance, exclusive); } case GeoShape::POLYGON: { S2Polygon* bPolygon = static_cast(bRegion.get()); - return s2PointAndS2PolygonAreWithinDistance(aPoint, bPolygon, distance, inclusive); + return s2PointAndS2PolygonAreWithinDistance(aPoint, bPolygon, distance, exclusive); } case GeoShape::UNKNOWN: default: { @@ -215,7 +215,7 @@ bool GeoFunction::dWithin(const Geography& a, const Geography& b, double distanc switch (b.shape()) { case GeoShape::POINT: { const S2Point& bPoint = static_cast(bRegion.get())->point(); - return s2PointAndS2PolylineAreWithinDistance(bPoint, aLine, distance, inclusive); + return s2PointAndS2PolylineAreWithinDistance(bPoint, aLine, distance, exclusive); } case GeoShape::LINESTRING: { S2Polyline* bLine = static_cast(bRegion.get()); @@ -224,16 +224,16 @@ bool GeoFunction::dWithin(const Geography& a, const Geography& b, double distanc bIndex.Add(std::make_unique(bLine)); S2ClosestEdgeQuery query(&aIndex); S2ClosestEdgeQuery::ShapeIndexTarget target(&bIndex); - if (inclusive) { - return query.IsDistanceLessOrEqual( - &target, S2Earth::ToChordAngle(util::units::Meters(distance))); + if (exclusive) { + return query.IsDistanceLess(&target, + S2Earth::ToChordAngle(util::units::Meters(distance))); } - return query.IsDistanceLess(&target, - S2Earth::ToChordAngle(util::units::Meters(distance))); + return query.IsDistanceLessOrEqual(&target, + S2Earth::ToChordAngle(util::units::Meters(distance))); } case GeoShape::POLYGON: { S2Polygon* bPolygon = static_cast(bRegion.get()); - return s2PolylineAndS2PolygonAreWithinDistance(aLine, bPolygon, distance, inclusive); + return s2PolylineAndS2PolygonAreWithinDistance(aLine, bPolygon, distance, exclusive); } case GeoShape::UNKNOWN: default: { @@ -248,22 +248,22 @@ bool GeoFunction::dWithin(const Geography& a, const Geography& b, double distanc switch (b.shape()) { case GeoShape::POINT: { const S2Point& bPoint = static_cast(bRegion.get())->point(); - return s2PointAndS2PolygonAreWithinDistance(bPoint, aPolygon, distance, inclusive); + return s2PointAndS2PolygonAreWithinDistance(bPoint, aPolygon, distance, exclusive); } case GeoShape::LINESTRING: { S2Polyline* bLine = static_cast(bRegion.get()); - return s2PolylineAndS2PolygonAreWithinDistance(bLine, aPolygon, distance, inclusive); + return s2PolylineAndS2PolygonAreWithinDistance(bLine, aPolygon, distance, exclusive); } case GeoShape::POLYGON: { S2Polygon* bPolygon = static_cast(bRegion.get()); S2ClosestEdgeQuery query(&aPolygon->index()); S2ClosestEdgeQuery::ShapeIndexTarget target(&bPolygon->index()); - if (inclusive) { - return query.IsDistanceLessOrEqual( - &target, S2Earth::ToChordAngle(util::units::Meters(distance))); + if (exclusive) { + return query.IsDistanceLess(&target, + S2Earth::ToChordAngle(util::units::Meters(distance))); } - return query.IsDistanceLess(&target, - S2Earth::ToChordAngle(util::units::Meters(distance))); + return query.IsDistanceLessOrEqual(&target, + S2Earth::ToChordAngle(util::units::Meters(distance))); } case GeoShape::UNKNOWN: default: { @@ -287,7 +287,7 @@ bool GeoFunction::dWithin(const Geography& a, const Geography& b, double distanc double GeoFunction::distance(const Geography& a, const Geography& b) { auto aRegion = a.asS2(); auto bRegion = b.asS2(); - if (!aRegion || !bRegion) { + if (UNLIKELY(!aRegion || !bRegion)) { return -1.0; } @@ -381,7 +381,7 @@ double GeoFunction::distance(const Geography& a, const Geography& b) { uint64_t GeoFunction::s2CellIdFromPoint(const Geography& a, int level) { auto aRegion = a.asS2(); - if (!aRegion) { + if (UNLIKELY(!aRegion)) { return -1; } if (level < 0 || level > 30) { @@ -410,7 +410,7 @@ uint64_t GeoFunction::s2CellIdFromPoint(const Geography& a, int level) { std::vector GeoFunction::s2CoveringCellIds( const Geography& a, int minLevel, int maxLevel, int maxCells, double bufferInMeters) { auto aRegion = a.asS2(); - if (!aRegion) { + if (UNLIKELY(!aRegion)) { return {}; } if (minLevel < 0 || minLevel > 30) { @@ -507,47 +507,41 @@ double GeoFunction::distanceOfS2PolygonWithS2Point(const S2Polygon* aPolygon, bool GeoFunction::s2PointAndS2PolylineAreWithinDistance(const S2Point& aPoint, const S2Polyline* bLine, double distance, - bool inclusive) { + bool exclusive) { MutableS2ShapeIndex bIndex; bIndex.Add(std::make_unique(bLine)); S2ClosestEdgeQuery query(&bIndex); S2ClosestEdgeQuery::PointTarget target(aPoint); - if (inclusive) { - return query.IsDistanceLessOrEqual(&target, - S2Earth::ToChordAngle(util::units::Meters(distance))); - } else { + if (exclusive) { return query.IsDistanceLess(&target, S2Earth::ToChordAngle(util::units::Meters(distance))); } + return query.IsDistanceLessOrEqual(&target, S2Earth::ToChordAngle(util::units::Meters(distance))); } bool GeoFunction::s2PointAndS2PolygonAreWithinDistance(const S2Point& aPoint, const S2Polygon* bPolygon, double distance, - bool inclusive) { + bool exclusive) { S2ClosestEdgeQuery query(&bPolygon->index()); S2ClosestEdgeQuery::PointTarget target(aPoint); - if (inclusive) { - return query.IsDistanceLessOrEqual(&target, - S2Earth::ToChordAngle(util::units::Meters(distance))); - } else { + if (exclusive) { return query.IsDistanceLess(&target, S2Earth::ToChordAngle(util::units::Meters(distance))); } + return query.IsDistanceLessOrEqual(&target, S2Earth::ToChordAngle(util::units::Meters(distance))); } bool GeoFunction::s2PolylineAndS2PolygonAreWithinDistance(const S2Polyline* aLine, const S2Polygon* bPolygon, double distance, - bool inclusive) { + bool exclusive) { MutableS2ShapeIndex aIndex; aIndex.Add(std::make_unique(aLine)); S2ClosestEdgeQuery::ShapeIndexTarget target(&aIndex); S2ClosestEdgeQuery query(&bPolygon->index()); - if (inclusive) { - return query.IsDistanceLessOrEqual(&target, - S2Earth::ToChordAngle(util::units::Meters(distance))); - } else { + if (exclusive) { return query.IsDistanceLess(&target, S2Earth::ToChordAngle(util::units::Meters(distance))); } + return query.IsDistanceLessOrEqual(&target, S2Earth::ToChordAngle(util::units::Meters(distance))); } } // namespace geo diff --git a/src/common/geo/GeoFunction.h b/src/common/geo/GeoFunction.h index 5b37a998c15..7480a3187e4 100644 --- a/src/common/geo/GeoFunction.h +++ b/src/common/geo/GeoFunction.h @@ -29,7 +29,10 @@ class GeoFunction { // Returns true if any of a is within distance meters of b. // We don't need to find the closest points. We just need to find the first point pair whose // distance is less than or less equal than the given distance. (Early quit) - static bool dWithin(const Geography& a, const Geography& b, double distance, bool inclusive); + static bool dWithin(const Geography& a, + const Geography& b, + double distance, + bool exclusive = false); // Return the closest distance in meters of a and b. static double distance(const Geography& a, const Geography& b); @@ -56,17 +59,17 @@ class GeoFunction { static bool s2PointAndS2PolylineAreWithinDistance(const S2Point& aPoint, const S2Polyline* bLine, double distance, - bool inclusive); + bool exclusive); static bool s2PointAndS2PolygonAreWithinDistance(const S2Point& aPoint, const S2Polygon* bPolygon, double distance, - bool inclusive); + bool exclusive); static bool s2PolylineAndS2PolygonAreWithinDistance(const S2Polyline* aLine, const S2Polygon* bPolygon, double distance, - bool inclusive); + bool exclusive); }; } // namespace geo diff --git a/src/common/geo/GeoIndex.cpp b/src/common/geo/GeoIndex.cpp new file mode 100644 index 00000000000..23f297e0ea7 --- /dev/null +++ b/src/common/geo/GeoIndex.cpp @@ -0,0 +1,182 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This sourc_e code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "common/geo/GeoIndex.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common/datatypes/Geography.h" +#include "common/utils/IndexKeyUtils.h" +#include "interface/gen-cpp2/storage_types.h" + +namespace nebula { +namespace geo { + +nebula::storage::cpp2::IndexColumnHint ScanRange::toIndexColumnHint() { + nebula::storage::cpp2::IndexColumnHint hint; + // column_name should be set by the caller + if (isRangeScan) { + hint.set_scan_type(nebula::storage::cpp2::ScanType::RANGE); + // Encode uint64_t as string in advance + hint.set_begin_value(IndexKeyUtils::encodeUint64(rangeMin)); + hint.set_end_value(IndexKeyUtils::encodeUint64(rangeMax)); + } else { + hint.set_scan_type(nebula::storage::cpp2::ScanType::PREFIX); + hint.set_begin_value(IndexKeyUtils::encodeUint64(rangeMin)); + } + return hint; +} + +std::vector GeoIndex::indexCells(const Geography& g) const noexcept { + auto r = g.asS2(); + if (UNLIKELY(!r)) { + return {}; + } + + auto cells = coveringCells(*r, g.shape() == GeoShape::POINT); + std::vector cellIds; + cellIds.reserve(cells.size()); + for (auto& cell : cells) { + cellIds.push_back(cell.id()); + } + return cellIds; +} + +std::vector GeoIndex::intersects(const Geography& g) const noexcept { + auto r = g.asS2(); + if (UNLIKELY(!r)) { + return {}; + } + + return intersects(*r, g.shape() == GeoShape::POINT); +} + +// covers degenerates to intersects currently +std::vector GeoIndex::covers(const Geography& g) const noexcept { return intersects(g); } + +// coveredBy degenerates to intersects currently +std::vector GeoIndex::coveredBy(const Geography& g) const noexcept { + return intersects(g); +} + +std::vector GeoIndex::dWithin(const Geography& g, double distance) const noexcept { + auto r = g.asS2(); + if (UNLIKELY(!r)) { + return {}; + } + + S1Angle radius = S2Earth::ToAngle(util::units::Meters(distance)); + // First expand the region, then build the covering + switch (g.shape()) { + case GeoShape::POINT: { + const S2Point& gPoint = static_cast(r.get())->point(); + S2Cap gCap(gPoint, radius); + return intersects(gCap); + } + case GeoShape::LINESTRING: { + S2Polyline* gLine = static_cast(r.get()); + MutableS2ShapeIndex index; + index.Add(std::make_unique(gLine)); + S2ShapeIndexBufferedRegion gBuffer(&index, radius); + return intersects(gBuffer); + } + case GeoShape::POLYGON: { + S2Polygon* gPolygon = static_cast(r.get()); + S2ShapeIndexBufferedRegion gBuffer(&gPolygon->index(), radius); + return intersects(gBuffer); + } + default: + LOG(FATAL) + << "Geography shapes other than Point/LineString/Polygon are not currently supported"; + return {}; + } +} + +std::vector GeoIndex::intersects(const S2Region& r, bool isPoint) const noexcept { + auto cells = coveringCells(r, isPoint); + std::vector scanRanges; + for (const S2CellId& cellId : cells) { + if (cellId.is_leaf()) { + scanRanges.emplace_back(cellId.id()); + } else { + scanRanges.emplace_back(cellId.range_min().id(), cellId.range_max().id()); + } + } + + // For the indexed column which only contains point, we don't need to get the ancestor cells, + // because the point is at max level(30). + if (!pointsOnly_) { + auto ancestors = ancestorCells(cells); + for (const S2CellId& cellId : ancestors) { + scanRanges.emplace_back(cellId.id()); + } + } + + return scanRanges; +} + +std::vector GeoIndex::coveringCells(const S2Region& r, bool isPoint) const noexcept { + // Currently we don't apply region coverer params to point, because it's useless. + // Point always use level 30. + if (isPoint) { + const S2Point& gPoint = static_cast(&r)->point(); + return {S2CellId(gPoint)}; + } + S2RegionCoverer rc(rcParams_.s2RegionCovererOpts()); + std::vector covering; + rc.GetCovering(r, &covering); + // 1. NO NEED TO CALL S2RegionCoverer::CanonicalizeCovering(covering), because the covering is + // already canonical, which means that is sorted, non-overlapping and satisfy the desired + // constraints min_level, max_level. + // 2. DO NOT CALL S2CellUnion::Normalize(covering), it will replacing groups of 4 child cells by + // their parent cell, In this case, it may cause the covering don't satisfy the desired + // constraints min_level. + return covering; +} + +std::vector GeoIndex::ancestorCells(const std::vector& cells) const noexcept { + // DCHECK(rc.IsCanonical(cells)); + std::vector ancestors; + std::unordered_set seen; + for (const auto& cellId : cells) { + for (auto l = cellId.level() - 1; l >= rcParams_.minCellLevel_; --l) { + S2CellId parentCellId = cellId.parent(l); + if (seen.find(parentCellId) != seen.end()) { + break; + } + seen.emplace(parentCellId); + ancestors.push_back(parentCellId); + } + } + // The ancestors here is non-overlapping but unsorted. Do we need to sort it? + // May need to call S2RegionCoverer::CanonicalizeCovering(&ancestors)? + return ancestors; +} + +} // namespace geo +} // namespace nebula + +namespace std { + +// Inject a customized hash function +std::size_t hash::operator()(const S2CellId& c) const noexcept { + return hash{}(c.id()); +} + +} // namespace std diff --git a/src/common/geo/GeoIndex.h b/src/common/geo/GeoIndex.h new file mode 100644 index 00000000000..3143c126920 --- /dev/null +++ b/src/common/geo/GeoIndex.h @@ -0,0 +1,101 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#pragma once + +#include + +#include +#include + +#include "common/datatypes/Geography.h" + +namespace nebula { + +namespace storage { +namespace cpp2 { +class IndexColumnHint; +} // namespace cpp2 +} // namespace storage + +namespace geo { + +struct RegionCoverParams { + // TODO(jie): Find the best default params + int minCellLevel_ = 4; + int maxCellLevel_ = 23; // About 1m + int maxCellNum_ = 8; + + RegionCoverParams() = default; + + RegionCoverParams(int minLevl, int maxLevl, int maxCells) + : minCellLevel_(minLevl), maxCellLevel_(maxLevl), maxCellNum_(maxCells) {} + + S2RegionCoverer::Options s2RegionCovererOpts() const { + S2RegionCoverer::Options opts; + opts.set_min_level(minCellLevel_); + opts.set_max_level(maxCellLevel_); + opts.set_max_cells(maxCellNum_); + return opts; + } +}; + +// scan type: PREFIX or RANGE +struct ScanRange { + uint64_t rangeMin; + uint64_t rangeMax; + bool isRangeScan; + + ScanRange(uint64_t min, uint64_t max) : rangeMin(min), rangeMax(max), isRangeScan(true) {} + + explicit ScanRange(uint64_t v) : rangeMin(v), isRangeScan(false) {} + + nebula::storage::cpp2::IndexColumnHint toIndexColumnHint(); +}; + +class GeoIndex { + public: + explicit GeoIndex(const RegionCoverParams& params, bool pointsOnly = false) + : rcParams_(params), pointsOnly_(pointsOnly) {} + + // Build geo index for the geography g + std::vector indexCells(const Geography& g) const noexcept; + + // Query the geo index + // ST_Intersects(g, x), x is the indexed geography column + std::vector intersects(const Geography& g) const noexcept; + // ST_Covers(g, x), x is the indexed geography column + std::vector covers(const Geography& g) const noexcept; + // ST_CoveredBy(g, x), x is the indexed geography column + std::vector coveredBy(const Geography& g) const noexcept; + // ST_Distance(g, x, distance), x is the indexed geography column + std::vector dWithin(const Geography& g, double distance) const noexcept; + + private: + std::vector intersects(const S2Region& r, bool isPoint = false) const noexcept; + + std::vector coveringCells(const S2Region& r, bool isPoint = false) const noexcept; + + std::vector ancestorCells(const std::vector& cells) const noexcept; + + private: + RegionCoverParams rcParams_; + // For the column Geography(Point), we don't need to build ancestor cells + bool pointsOnly_{false}; +}; + +} // namespace geo +} // namespace nebula + +namespace std { + +// Inject a customized hash function +template <> +struct hash { + std::size_t operator()(const S2CellId& h) const noexcept; +}; + +} // namespace std diff --git a/src/common/geo/GeoUtils.h b/src/common/geo/GeoUtils.h index 7424399f76c..4623ee1a621 100644 --- a/src/common/geo/GeoUtils.h +++ b/src/common/geo/GeoUtils.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include @@ -26,28 +27,24 @@ class GeoUtils final { } case GeoShape::LINESTRING: { const auto& lineString = geog.lineString(); - auto coordList = lineString.coordList; + const auto& coordList = lineString.coordList; auto s2Points = s2PointsFromCoordinateList(coordList); - auto s2Polyline = std::make_unique(s2Points, S2Debug::DISABLE); - return s2Polyline; + return std::make_unique(s2Points, S2Debug::DISABLE); } case GeoShape::POLYGON: { const auto& polygon = geog.polygon(); uint32_t numCoordList = polygon.numCoordList(); std::vector> s2Loops; s2Loops.reserve(numCoordList); - for (size_t i = 0; i < numCoordList; ++i) { - auto coordList = polygon.coordListList[i]; - if (!coordList.empty()) { - coordList.pop_back(); // Remove redundant last coordinate - } - auto s2Points = s2PointsFromCoordinateList(coordList); + for (const auto& coordList : polygon.coordListList) { + // S2 doesn't need the redundant last point + auto s2Points = s2PointsFromCoordinateList(coordList, true); auto s2Loop = std::make_unique(std::move(s2Points), S2Debug::DISABLE); - s2Loop->Normalize(); // All loops must be oriented CCW(counterclockwise) for S2 + // All loops must be oriented CCW(counterclockwise) for S2 + s2Loop->Normalize(); s2Loops.emplace_back(std::move(s2Loop)); } - auto s2Polygon = std::make_unique(std::move(s2Loops), S2Debug::DISABLE); - return s2Polygon; + return std::make_unique(std::move(s2Loops), S2Debug::DISABLE); } default: LOG(FATAL) @@ -57,14 +54,27 @@ class GeoUtils final { } static S2Point s2PointFromCoordinate(const Coordinate& coord) { - auto latlng = S2LatLng::FromDegrees( - coord.y, coord.x); // Note: S2Point requires latitude to be first, and longitude to be last + // Note: S2Point requires latitude to be first, and longitude to be last + auto latlng = S2LatLng::FromDegrees(coord.y, coord.x); return latlng.ToPoint(); } - static std::vector s2PointsFromCoordinateList(const std::vector& coordList) { + static Coordinate coordinateFromS2Point(const S2Point& s2Point) { + S2LatLng s2Latlng(s2Point); + return Coordinate(s2Latlng.lng().degrees(), s2Latlng.lat().degrees()); + } + + static std::vector s2PointsFromCoordinateList(const std::vector& coordList, + bool excludeTheLast = false) { std::vector s2Points; uint32_t numCoords = coordList.size(); + if (excludeTheLast) { + numCoords -= 1; + } + if (numCoords == 0) { + return {}; + } + s2Points.reserve(numCoords); for (size_t i = 0; i < numCoords; ++i) { auto coord = coordList[i]; diff --git a/src/common/geo/io/wkt/WKTWriter.cpp b/src/common/geo/io/wkt/WKTWriter.cpp index 84bfcb7ab64..0a4f8c97b8a 100644 --- a/src/common/geo/io/wkt/WKTWriter.cpp +++ b/src/common/geo/io/wkt/WKTWriter.cpp @@ -61,9 +61,10 @@ void WKTWriter::writeCoordinateList(std::string& wkt, const std::vector& coordList) const { for (size_t i = 0; i < coordList.size(); ++i) { writeCoordinate(wkt, coordList[i]); - wkt.append(","); + wkt.append(", "); } wkt.pop_back(); + wkt.pop_back(); } void WKTWriter::WKTWriter::writeCoordinateListList( @@ -75,9 +76,10 @@ void WKTWriter::WKTWriter::writeCoordinateListList( wkt.append("("); writeCoordinateList(wkt, coordList); wkt.append(")"); - wkt.append(","); + wkt.append(", "); } wkt.pop_back(); + wkt.pop_back(); } void WKTWriter::writeDouble(std::string& wkt, double v) const { diff --git a/src/common/geo/test/GeoFunctionTest.cpp b/src/common/geo/test/GeoFunctionTest.cpp index d525536691e..5bfc5ba489d 100644 --- a/src/common/geo/test/GeoFunctionTest.cpp +++ b/src/common/geo/test/GeoFunctionTest.cpp @@ -546,43 +546,43 @@ TEST(DWithin, point2Point) { { auto point1 = Geography::fromWKT("POINT(1.0 1.0)").value(); auto point2 = Geography::fromWKT("POINT(1.0 1.0)").value(); - bool b = GeoFunction::dWithin(point1, point2, 0.0, true); + bool b = GeoFunction::dWithin(point1, point2, 0.0, false); EXPECT_EQ(true, b); } { auto point1 = Geography::fromWKT("POINT(1.0 1.0)").value(); auto point2 = Geography::fromWKT("POINT(1.0 1.0)").value(); - bool b = GeoFunction::dWithin(point1, point2, 0.0, false); + bool b = GeoFunction::dWithin(point1, point2, 0.0, true); EXPECT_EQ(false, b); } { auto point1 = Geography::fromWKT("POINT(1.0 1.0)").value(); auto point2 = Geography::fromWKT("POINT(1.0 1.0)").value(); - bool b = GeoFunction::dWithin(point1, point2, 0.1, false); + bool b = GeoFunction::dWithin(point1, point2, 0.1, true); EXPECT_EQ(true, b); } { auto point1 = Geography::fromWKT("POINT(1.0 1.0)").value(); auto point2 = Geography::fromWKT("POINT(1.0 1.2)").value(); - bool b = GeoFunction::dWithin(point1, point2, 22239.020235496788, true); + bool b = GeoFunction::dWithin(point1, point2, 22239.020235496788, false); EXPECT_EQ(true, b); } { auto point1 = Geography::fromWKT("POINT(1.0 1.0)").value(); auto point2 = Geography::fromWKT("POINT(1.0 1.2)").value(); - bool b = GeoFunction::dWithin(point1, point2, 22239.020235496788, false); + bool b = GeoFunction::dWithin(point1, point2, 22239.020235496788, true); EXPECT_EQ(false, b); } { auto point1 = Geography::fromWKT("POINT(1.0 1.0)").value(); auto point2 = Geography::fromWKT("POINT(1.0 1.2)").value(); - bool b = GeoFunction::dWithin(point1, point2, 22240, true); + bool b = GeoFunction::dWithin(point1, point2, 22240, false); EXPECT_EQ(true, b); } { auto point1 = Geography::fromWKT("POINT(-118.4079 33.9434)").value(); auto point2 = Geography::fromWKT("POINT(2.5559 49.0083)").value(); - bool b = GeoFunction::dWithin(point1, point2, 10000, false); + bool b = GeoFunction::dWithin(point1, point2, 10000, true); EXPECT_EQ(false, b); } } @@ -591,25 +591,25 @@ TEST(DWithin, point2LineString) { { auto point1 = Geography::fromWKT("POINT(1.0 1.0)").value(); auto line2 = Geography::fromWKT("LINESTRING(1.0 1.0, 2.0 2.0)").value(); - bool b = GeoFunction::dWithin(point1, line2, 0.0, true); + bool b = GeoFunction::dWithin(point1, line2, 0.0, false); EXPECT_EQ(true, b); } { auto point1 = Geography::fromWKT("POINT(1.5 1.6)").value(); auto line2 = Geography::fromWKT("LINESTRING(1.0 1.0, 1.5 1.6, 2.0 2.2)").value(); - bool b = GeoFunction::dWithin(point1, line2, 0.0, true); + bool b = GeoFunction::dWithin(point1, line2, 0.0, false); EXPECT_EQ(true, b); } { auto point1 = Geography::fromWKT("POINT(1.5 1.6)").value(); auto line2 = Geography::fromWKT("LINESTRING(1.0 1.0, 2.0 2.2, 1.5 1.6)").value(); - bool b = GeoFunction::dWithin(point1, line2, 0.0, true); + bool b = GeoFunction::dWithin(point1, line2, 0.0, false); EXPECT_EQ(true, b); } { auto point1 = Geography::fromWKT("POINT(1.2 3.8)").value(); auto point2 = Geography::fromWKT("LINESTRING(1.0 1.0, 1.5 1.6, 2.0 2.2)").value(); - bool b = GeoFunction::dWithin(point1, point2, 1.0, false); + bool b = GeoFunction::dWithin(point1, point2, 1.0, true); EXPECT_EQ(false, b); } } @@ -618,28 +618,28 @@ TEST(DWithin, point2Polygon) { { auto point1 = Geography::fromWKT("POINT(1.0 1.0)").value(); auto polygon2 = Geography::fromWKT("POLYGON((1.0 1.0, 2.0 2.0, 0.0 2.0, 1.0 1.0))").value(); - bool b = GeoFunction::dWithin(point1, polygon2, 0.0, true); + bool b = GeoFunction::dWithin(point1, polygon2, 0.0, false); EXPECT_EQ(true, b); } // Point lies on the interior of the polygon { auto point1 = Geography::fromWKT("POINT(1.5 1.9)").value(); auto polygon2 = Geography::fromWKT("POLYGON((1.0 1.0, 2.0 2.0, 0.0 2.0, 1.0 1.0))").value(); - bool b = GeoFunction::dWithin(point1, polygon2, 0.0, true); + bool b = GeoFunction::dWithin(point1, polygon2, 0.0, false); EXPECT_EQ(true, b); } { auto point1 = Geography::fromWKT("POINT(0.5 0.5)").value(); auto polygon2 = Geography::fromWKT("POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0))").value(); - bool b = GeoFunction::dWithin(point1, polygon2, 0.0, true); + bool b = GeoFunction::dWithin(point1, polygon2, 0.0, false); EXPECT_EQ(true, b); } { auto point1 = Geography::fromWKT("POINT(1.5 1.6)").value(); auto polygon2 = Geography::fromWKT("POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0))").value(); - bool b = GeoFunction::dWithin(point1, polygon2, 99999, false); + bool b = GeoFunction::dWithin(point1, polygon2, 99999, true); EXPECT_EQ(true, b); } { @@ -648,7 +648,7 @@ TEST(DWithin, point2Polygon) { "POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0), (0.2 0.2, 0.2 0.4, " "0.4 0.4, 0.4 0.2, 0.2 0.2))") .value(); - bool b = GeoFunction::dWithin(point1, polygon2, 10000, true); + bool b = GeoFunction::dWithin(point1, polygon2, 10000, false); EXPECT_EQ(false, b); } } @@ -657,31 +657,31 @@ TEST(DWithin, lineString2LineString) { { auto line1 = Geography::fromWKT("LINESTRING(2.0 2.0, 3.0 3.0)").value(); auto line2 = Geography::fromWKT("LINESTRING(2.0 2.0, 3.0 3.0)").value(); - bool b = GeoFunction::dWithin(line1, line2, 0.0, true); + bool b = GeoFunction::dWithin(line1, line2, 0.0, false); EXPECT_EQ(true, b); } { auto line1 = Geography::fromWKT("LINESTRING(1.0 1.0, 1.5 1.5, 2.0 2.0)").value(); auto line2 = Geography::fromWKT("LINESTRING(2.0 2.0, 3.0 3.0)").value(); - bool b = GeoFunction::dWithin(line1, line2, 0.0, true); + bool b = GeoFunction::dWithin(line1, line2, 0.0, false); EXPECT_EQ(true, b); } { auto line1 = Geography::fromWKT("LINESTRING(0.0 0.0, 1.0 1.0)").value(); auto line2 = Geography::fromWKT("LINESTRING(5.0 5.0, 6.0 6.0)").value(); - bool b = GeoFunction::dWithin(line1, line2, 628519, true); + bool b = GeoFunction::dWithin(line1, line2, 628519, false); EXPECT_EQ(false, b); } { auto line1 = Geography::fromWKT("LINESTRING(1.0 1.0, 1.5 1.5, 2.0 2.0)").value(); auto line2 = Geography::fromWKT("LINESTRING(1.0 2.0, 2.0 1.0)").value(); - bool b = GeoFunction::dWithin(line1, line2, 1.0, false); + bool b = GeoFunction::dWithin(line1, line2, 1.0, true); EXPECT_EQ(true, b); } { auto line1 = Geography::fromWKT("LINESTRING(1.0 1.0, 1.5 1.5, 2.0 2.0)").value(); auto line2 = Geography::fromWKT("LINESTRING(1.0 2.0, 1.0 4.0)").value(); - bool b = GeoFunction::dWithin(line1, line2, 78609, false); + bool b = GeoFunction::dWithin(line1, line2, 78609, true); EXPECT_EQ(true, b); } } @@ -691,21 +691,21 @@ TEST(DWithin, lineString2Polygon) { auto line1 = Geography::fromWKT("LINESTRING(1.0 1.0, 1.5 1.5, 2.0 2.0)").value(); auto polygon2 = Geography::fromWKT("POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0))").value(); - bool b = GeoFunction::dWithin(line1, polygon2, 0.0, true); + bool b = GeoFunction::dWithin(line1, polygon2, 0.0, false); EXPECT_EQ(true, b); } { auto line1 = Geography::fromWKT("LINESTRING(0.2 0.2, 0.4 0.4)").value(); auto polygon2 = Geography::fromWKT("POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0))").value(); - bool b = GeoFunction::dWithin(line1, polygon2, 0.0, false); + bool b = GeoFunction::dWithin(line1, polygon2, 0.0, true); EXPECT_EQ(false, b); } { auto line1 = Geography::fromWKT("LINESTRING(-0.5 0.5, 0.5 0.5)").value(); auto polygon2 = Geography::fromWKT("POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0))").value(); - bool b = GeoFunction::dWithin(line1, polygon2, 9999, false); + bool b = GeoFunction::dWithin(line1, polygon2, 9999, true); EXPECT_EQ(true, b); } { @@ -714,7 +714,7 @@ TEST(DWithin, lineString2Polygon) { "POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0), (0.2 0.2, 0.2 0.4, " "0.4 0.4, 0.4 0.2, 0.2 0.2))") .value(); - bool b = GeoFunction::dWithin(line1, polygon2, 5555, true); + bool b = GeoFunction::dWithin(line1, polygon2, 5555, false); EXPECT_EQ(false, b); } { @@ -723,7 +723,7 @@ TEST(DWithin, lineString2Polygon) { "POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0), (0.2 0.2, 0.2 0.4, " "0.4 0.4, 0.4 0.2, 0.2 0.2))") .value(); - bool b = GeoFunction::dWithin(line1, polygon2, 0.1, false); + bool b = GeoFunction::dWithin(line1, polygon2, 0.1, true); EXPECT_EQ(true, b); } { @@ -732,7 +732,7 @@ TEST(DWithin, lineString2Polygon) { "POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0), (0.2 0.2, 0.2 0.4, " "0.4 0.4, 0.4 0.2, 0.2 0.2))") .value(); - bool b = GeoFunction::dWithin(line1, polygon2, 628520, false); + bool b = GeoFunction::dWithin(line1, polygon2, 628520, true); EXPECT_EQ(true, b); } } @@ -743,7 +743,7 @@ TEST(DWithin, polygon2Polygon) { Geography::fromWKT("POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0))").value(); auto polygon2 = Geography::fromWKT("POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0))").value(); - bool b = GeoFunction::dWithin(polygon1, polygon2, 0.0, true); + bool b = GeoFunction::dWithin(polygon1, polygon2, 0.0, false); EXPECT_EQ(true, b); } { @@ -751,7 +751,7 @@ TEST(DWithin, polygon2Polygon) { Geography::fromWKT("POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0))").value(); auto polygon2 = Geography::fromWKT("POLYGON((0.2 0.2, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.2 0.2))").value(); - bool b = GeoFunction::dWithin(polygon1, polygon2, 1.1, false); + bool b = GeoFunction::dWithin(polygon1, polygon2, 1.1, true); EXPECT_EQ(true, b); } { @@ -759,7 +759,7 @@ TEST(DWithin, polygon2Polygon) { Geography::fromWKT("POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0))").value(); auto polygon2 = Geography::fromWKT("POLYGON((3.0 3.0, 4.0 3.0, 4.0 4.0, 3.0 4.0, 3.0 3.0))").value(); - bool b = GeoFunction::dWithin(polygon1, polygon2, 314402, true); + bool b = GeoFunction::dWithin(polygon1, polygon2, 314402, false); EXPECT_EQ(false, b); } { @@ -770,7 +770,7 @@ TEST(DWithin, polygon2Polygon) { "POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0), (0.2 0.2, 0.2 0.4, " "0.4 0.4, 0.4 0.2, 0.2 0.2))") .value(); - bool b = GeoFunction::dWithin(polygon1, polygon2, 5560, false); + bool b = GeoFunction::dWithin(polygon1, polygon2, 5560, true); EXPECT_EQ(true, b); } { @@ -779,7 +779,7 @@ TEST(DWithin, polygon2Polygon) { auto polygon2 = Geography::fromWKT("POLYGON((-8.0 -8.0, -4.0 -8.0, -4.0 -4.0, -8.0 -4.0, -8.0 -8.0))") .value(); - bool b = GeoFunction::dWithin(polygon1, polygon2, 628750, false); + bool b = GeoFunction::dWithin(polygon1, polygon2, 628750, true); EXPECT_EQ(false, b); } } diff --git a/src/common/meta/NebulaSchemaProvider.cpp b/src/common/meta/NebulaSchemaProvider.cpp index dc5aab4a2c5..a9bf2f5e66f 100644 --- a/src/common/meta/NebulaSchemaProvider.cpp +++ b/src/common/meta/NebulaSchemaProvider.cpp @@ -91,7 +91,8 @@ void NebulaSchemaProvider::addField(folly::StringPiece name, cpp2::PropertyType type, size_t fixedStrLen, bool nullable, - Expression* defaultValue) { + Expression* defaultValue, + cpp2::GeoShape geoShape) { size_t size = fieldSize(type, fixedStrLen); size_t offset = 0; @@ -112,7 +113,8 @@ void NebulaSchemaProvider::addField(folly::StringPiece name, defaultValue, size, offset, - nullFlagPos); + nullFlagPos, + geoShape); fieldNameIndex_.emplace(name.toString(), static_cast(fields_.size() - 1)); } diff --git a/src/common/meta/NebulaSchemaProvider.h b/src/common/meta/NebulaSchemaProvider.h index 0b185c7ed3a..31a381a4a83 100644 --- a/src/common/meta/NebulaSchemaProvider.h +++ b/src/common/meta/NebulaSchemaProvider.h @@ -29,7 +29,8 @@ class NebulaSchemaProvider : public SchemaProviderIf { Expression* defaultValue, size_t size, size_t offset, - size_t nullFlagPos) + size_t nullFlagPos, + cpp2::GeoShape geoShape) : name_(std::move(name)), type_(std::move(type)), nullable_(nullable), @@ -37,7 +38,8 @@ class NebulaSchemaProvider : public SchemaProviderIf { defaultValue_(defaultValue), size_(size), offset_(offset), - nullFlagPos_(nullFlagPos) {} + nullFlagPos_(nullFlagPos), + geoShape_(geoShape) {} const char* name() const override { return name_.c_str(); } @@ -58,6 +60,8 @@ class NebulaSchemaProvider : public SchemaProviderIf { return nullFlagPos_; } + cpp2::GeoShape geoShape() const override { return geoShape_; } + private: std::string name_; cpp2::PropertyType type_; @@ -67,6 +71,7 @@ class NebulaSchemaProvider : public SchemaProviderIf { size_t size_; size_t offset_; size_t nullFlagPos_; + cpp2::GeoShape geoShape_; }; public: @@ -93,7 +98,8 @@ class NebulaSchemaProvider : public SchemaProviderIf { cpp2::PropertyType type, size_t fixedStrLen = 0, bool nullable = false, - Expression* defaultValue = nullptr); + Expression* defaultValue = nullptr, + cpp2::GeoShape geoShape = cpp2::GeoShape::ANY); static std::size_t fieldSize(cpp2::PropertyType type, std::size_t fixedStrLimit); diff --git a/src/common/meta/SchemaProviderIf.h b/src/common/meta/SchemaProviderIf.h index b880c7f01cd..f643aab28cb 100644 --- a/src/common/meta/SchemaProviderIf.h +++ b/src/common/meta/SchemaProviderIf.h @@ -40,6 +40,10 @@ class SchemaProviderIf { // In v2, if the field is nullable, it returns the position of // the null flag bit, otherwise, it returns 0 virtual size_t nullFlagPos() const = 0; + // In v1, this always returns cpp2::GeoShape::ANY + // In v2, if the field type is cpp2::PropertyType::Geography, + // it returns the specified geo shape type + virtual cpp2::GeoShape geoShape() const = 0; }; // Inherited classes do not need to implement the Iterator diff --git a/src/common/network/NetworkUtils.cpp b/src/common/network/NetworkUtils.cpp index 10209047905..e82aa481ad4 100644 --- a/src/common/network/NetworkUtils.cpp +++ b/src/common/network/NetworkUtils.cpp @@ -306,5 +306,32 @@ std::string NetworkUtils::toHostsStr(const std::vector& hosts) { return hostsString; } +Status NetworkUtils::validateHostOrIp(const std::string& hostOrIp) { + if (hostOrIp.empty()) { + return Status::Error("local_ip is empty, need to config it through config file."); + } + const std::regex ipv4( + "((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}" + "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"); + if (std::regex_match(hostOrIp, ipv4)) { + const std::regex loopbackOrAny( + "^127(?:\\.[0-9]+){0,2}\\.[0-9]+$|^(?:0*\\:)*?:?0*1$|(0\\.){3}0"); + if (std::regex_match(hostOrIp, loopbackOrAny)) { + return Status::OK(); + } + auto ipsStatus = listIPv4s(); + NG_RETURN_IF_ERROR(ipsStatus); + const auto& ips = ipsStatus.value(); + auto result = std::find(ips.begin(), ips.end(), hostOrIp); + if (result == ips.end()) { + return Status::Error("%s is not a valid ip in current host, candidates: %s", + hostOrIp.c_str(), + folly::join(",", ips).c_str()); + } + } else { + NG_RETURN_IF_ERROR(resolveHost(hostOrIp, 0)); + } + return Status::OK(); +} } // namespace network } // namespace nebula diff --git a/src/common/network/NetworkUtils.h b/src/common/network/NetworkUtils.h index 72236215d17..9d97a52fe48 100644 --- a/src/common/network/NetworkUtils.h +++ b/src/common/network/NetworkUtils.h @@ -51,6 +51,8 @@ class NetworkUtils final { static StatusOr> toHosts(const std::string& peersStr); static std::string toHostsStr(const std::vector& hosts); + static Status validateHostOrIp(const std::string& HostOrIp); + private: }; diff --git a/src/common/network/test/NetworkUtilsTest.cpp b/src/common/network/test/NetworkUtilsTest.cpp index dd80de363fe..8e227055b03 100644 --- a/src/common/network/test/NetworkUtilsTest.cpp +++ b/src/common/network/test/NetworkUtilsTest.cpp @@ -86,6 +86,40 @@ TEST(NetworkUtils, toHosts) { ASSERT_FALSE(s.ok()); } +TEST(NetworkUtils, ValidateHostOrIp) { + std::string hostOrIp = "127.0.0.1"; + auto result = NetworkUtils::validateHostOrIp(hostOrIp); + EXPECT_TRUE(result.ok()); + + hostOrIp = "127.0.1.1"; + result = NetworkUtils::validateHostOrIp(hostOrIp); + EXPECT_TRUE(result.ok()); + + hostOrIp = "0.0.0.0"; + result = NetworkUtils::validateHostOrIp(hostOrIp); + EXPECT_TRUE(result.ok()); + + hostOrIp = "000.000.000.000"; + result = NetworkUtils::validateHostOrIp(hostOrIp); + EXPECT_FALSE(result.ok()); + + hostOrIp = "0.0.0.0.0"; + result = NetworkUtils::validateHostOrIp(hostOrIp); + EXPECT_FALSE(result.ok()); + + hostOrIp = "127.0.0.0.1"; + result = NetworkUtils::validateHostOrIp(hostOrIp); + EXPECT_FALSE(result.ok()); + + hostOrIp = "localhost"; + result = NetworkUtils::validateHostOrIp(hostOrIp); + EXPECT_TRUE(result.ok()); + + hostOrIp = "NonvalidHostName"; + result = NetworkUtils::validateHostOrIp(hostOrIp); + EXPECT_FALSE(result.ok()); +} + } // namespace network } // namespace nebula diff --git a/src/common/utils/IndexKeyUtils.cpp b/src/common/utils/IndexKeyUtils.cpp index 864e6c7c99c..0dc9b49279c 100644 --- a/src/common/utils/IndexKeyUtils.cpp +++ b/src/common/utils/IndexKeyUtils.cpp @@ -11,74 +11,111 @@ namespace nebula { // static -std::string IndexKeyUtils::encodeValues(std::vector&& values, - const std::vector& cols) { +std::vector IndexKeyUtils::encodeValues( + std::vector&& values, const std::vector& cols) { bool hasNullCol = false; // An index has a maximum of 16 columns. 2 byte (16 bit) is enough. u_short nullableBitSet = 0; - std::string index; - - for (size_t i = 0; i < values.size(); i++) { - auto isNullable = cols[i].nullable_ref().value_or(false); - if (isNullable) { - hasNullCol = true; - } + auto findGeo = [](const meta::cpp2::ColumnDef& col) { + return col.get_type().get_type() == meta::cpp2::PropertyType::GEOGRAPHY; + }; + bool hasGeo = std::find_if(cols.begin(), cols.end(), findGeo) != cols.end(); + // Only support to create index on a single geography column currently; + DCHECK(!hasGeo || cols.size() == 1); + std::vector indexes; + + if (!hasGeo) { + std::string index; + for (size_t i = 0; i < values.size(); i++) { + auto isNullable = cols[i].nullable_ref().value_or(false); + if (isNullable) { + hasNullCol = true; + } - if (!values[i].isNull()) { - // string index need to fill with '\0' if length is less than schema - if (cols[i].type.type == meta::cpp2::PropertyType::FIXED_STRING) { - auto len = static_cast(*cols[i].type.get_type_length()); - index.append(encodeValue(values[i], len)); + if (!values[i].isNull()) { + // string index need to fill with '\0' if length is less than schema + if (cols[i].type.type == meta::cpp2::PropertyType::FIXED_STRING) { + auto len = static_cast(*cols[i].type.get_type_length()); + index.append(encodeValue(values[i], len)); + } else { + index.append(encodeValue(values[i])); + } } else { - index.append(encodeValue(values[i])); + nullableBitSet |= 0x8000 >> i; + auto type = IndexKeyUtils::toValueType(cols[i].type.get_type()); + index.append(encodeNullValue(type, cols[i].type.get_type_length())); } + } + indexes.emplace_back(std::move(index)); + } else { + hasNullCol = cols.back().nullable_ref().value_or(false); + DCHECK_EQ(values.size(), 1); + const auto& value = values.back(); + if (!value.isNull()) { + DCHECK(value.type() == Value::Type::GEOGRAPHY); + indexes = encodeGeography(value.getGeography()); } else { - nullableBitSet |= 0x8000 >> i; - auto type = IndexKeyUtils::toValueType(cols[i].type.get_type()); - index.append(encodeNullValue(type, cols[i].type.get_type_length())); + nullableBitSet |= 0x8000; + auto type = IndexKeyUtils::toValueType(cols.back().type.get_type()); + indexes.emplace_back(encodeNullValue(type, nullptr)); } } // if has nullable field, append nullableBitSet to the end if (hasNullCol) { - index.append(reinterpret_cast(&nullableBitSet), sizeof(u_short)); + for (auto& index : indexes) { + index.append(reinterpret_cast(&nullableBitSet), sizeof(u_short)); + } } - return index; + return indexes; } // static -std::string IndexKeyUtils::vertexIndexKey( - size_t vIdLen, PartitionID partId, IndexID indexId, const VertexID& vId, std::string&& values) { +std::vector IndexKeyUtils::vertexIndexKeys(size_t vIdLen, + PartitionID partId, + IndexID indexId, + const VertexID& vId, + std::vector&& values) { int32_t item = (partId << kPartitionOffset) | static_cast(NebulaKeyType::kIndex); - std::string key; - key.reserve(256); - key.append(reinterpret_cast(&item), sizeof(int32_t)) - .append(reinterpret_cast(&indexId), sizeof(IndexID)) - .append(values) - .append(vId.data(), vId.size()) - .append(vIdLen - vId.size(), '\0'); - return key; + std::vector keys; + keys.reserve(values.size()); + for (const auto& value : values) { + std::string key; + key.reserve(256); + key.append(reinterpret_cast(&item), sizeof(int32_t)) + .append(reinterpret_cast(&indexId), sizeof(IndexID)) + .append(value) + .append(vId.data(), vId.size()) + .append(vIdLen - vId.size(), '\0'); + keys.emplace_back(key); + } + return keys; } // static -std::string IndexKeyUtils::edgeIndexKey(size_t vIdLen, - PartitionID partId, - IndexID indexId, - const VertexID& srcId, - EdgeRanking rank, - const VertexID& dstId, - std::string&& values) { +std::vector IndexKeyUtils::edgeIndexKeys(size_t vIdLen, + PartitionID partId, + IndexID indexId, + const VertexID& srcId, + EdgeRanking rank, + const VertexID& dstId, + std::vector&& values) { int32_t item = (partId << kPartitionOffset) | static_cast(NebulaKeyType::kIndex); - std::string key; - key.reserve(256); - key.append(reinterpret_cast(&item), sizeof(int32_t)) - .append(reinterpret_cast(&indexId), sizeof(IndexID)) - .append(values) - .append(srcId.data(), srcId.size()) - .append(vIdLen - srcId.size(), '\0') - .append(IndexKeyUtils::encodeRank(rank)) - .append(dstId.data(), dstId.size()) - .append(vIdLen - dstId.size(), '\0'); - return key; + std::vector keys; + keys.reserve(values.size()); + for (const auto& value : values) { + std::string key; + key.reserve(256); + key.append(reinterpret_cast(&item), sizeof(int32_t)) + .append(reinterpret_cast(&indexId), sizeof(IndexID)) + .append(value) + .append(srcId.data(), srcId.size()) + .append(vIdLen - srcId.size(), '\0') + .append(IndexKeyUtils::encodeRank(rank)) + .append(dstId.data(), dstId.size()) + .append(vIdLen - dstId.size(), '\0'); + keys.emplace_back(key); + } + return keys; } // static @@ -120,7 +157,7 @@ Value IndexKeyUtils::parseIndexTTL(const folly::StringPiece& raw) { } // static -StatusOr IndexKeyUtils::collectIndexValues( +StatusOr> IndexKeyUtils::collectIndexValues( RowReader* reader, const std::vector& cols) { if (reader == nullptr) { return Status::Error("Invalid row reader"); diff --git a/src/common/utils/IndexKeyUtils.h b/src/common/utils/IndexKeyUtils.h index 23d699abce8..049fabebebb 100644 --- a/src/common/utils/IndexKeyUtils.h +++ b/src/common/utils/IndexKeyUtils.h @@ -10,6 +10,7 @@ #include "codec/RowReader.h" #include "common/base/Base.h" #include "common/base/StatusOr.h" +#include "common/geo/GeoIndex.h" #include "common/utils/Types.h" #include "interface/gen-cpp2/meta_types.h" @@ -138,7 +139,7 @@ class IndexKeyUtils final { return encodeDateTime(v.getDateTime()); } case Value::Type::GEOGRAPHY: { - // TODO(jie) + LOG(FATAL) << "Should call encodeGeography separately"; return ""; } default: @@ -299,6 +300,19 @@ class IndexKeyUtils final { return buf; } + static std::vector encodeGeography(const nebula::Geography& gg) { + // TODO(jie): Get index params from meta to construct RegionCoverParams + geo::RegionCoverParams rc; + // TODO(jie): Get schema meta to know if it's point only + geo::GeoIndex geoIndex(rc, false); + auto cellIds = geoIndex.indexCells(gg); + std::vector bufs; + for (auto cellId : cellIds) { + bufs.emplace_back(encodeUint64(cellId)); + } + return bufs; + } + static nebula::DateTime decodeDateTime(const folly::StringPiece& raw) { int16_t year = *reinterpret_cast(raw.data()); int8_t month = *reinterpret_cast(raw.data() + sizeof(int16_t)); @@ -480,30 +494,30 @@ class IndexKeyUtils final { /** * Generate vertex|edge index key for kv store **/ - static std::string encodeValues(std::vector&& values, - const std::vector& cols); + static std::vector encodeValues( + std::vector&& values, const std::vector& cols); /** * param valueTypes ： column type of each index column. If there are no *nullable columns in the index, the parameter can be empty. **/ - static std::string vertexIndexKey(size_t vIdLen, - PartitionID partId, - IndexID indexId, - const VertexID& vId, - std::string&& values); + static std::vector vertexIndexKeys(size_t vIdLen, + PartitionID partId, + IndexID indexId, + const VertexID& vId, + std::vector&& values); /** * param valueTypes ： column type of each index column. If there are no *nullable columns in the index, the parameter can be empty. **/ - static std::string edgeIndexKey(size_t vIdLen, - PartitionID partId, - IndexID indexId, - const VertexID& srcId, - EdgeRanking rank, - const VertexID& dstId, - std::string&& values); + static std::vector edgeIndexKeys(size_t vIdLen, + PartitionID partId, + IndexID indexId, + const VertexID& srcId, + EdgeRanking rank, + const VertexID& dstId, + std::vector&& values); static std::string indexPrefix(PartitionID partId, IndexID indexId); @@ -513,7 +527,7 @@ class IndexKeyUtils final { static Value parseIndexTTL(const folly::StringPiece& raw); - static StatusOr collectIndexValues( + static StatusOr> collectIndexValues( RowReader* reader, const std::vector& cols); private: diff --git a/src/common/utils/test/CMakeLists.txt b/src/common/utils/test/CMakeLists.txt index 7e9d59739ff..d02df8575dc 100644 --- a/src/common/utils/test/CMakeLists.txt +++ b/src/common/utils/test/CMakeLists.txt @@ -11,6 +11,7 @@ nebula_add_test( $ $ $ + $ LIBRARIES gtest ${THRIFT_LIBRARIES} @@ -29,6 +30,7 @@ nebula_add_test( $ $ $ + $ LIBRARIES gtest ${THRIFT_LIBRARIES} @@ -47,6 +49,7 @@ nebula_add_test( $ $ $ + $ LIBRARIES gtest ${THRIFT_LIBRARIES} diff --git a/src/common/utils/test/IndexKeyUtilsTest.cpp b/src/common/utils/test/IndexKeyUtilsTest.cpp index e5e7ea1c44e..3f17023fc80 100644 --- a/src/common/utils/test/IndexKeyUtilsTest.cpp +++ b/src/common/utils/test/IndexKeyUtilsTest.cpp @@ -145,7 +145,7 @@ TEST(IndexKeyUtilsTest, encodeDouble) { TEST(IndexKeyUtilsTest, vertexIndexKeyV1) { auto values = getIndexValues(); - auto key = IndexKeyUtils::vertexIndexKey(8, 1, 1, getStringId(1), std::move(values)); + auto key = IndexKeyUtils::vertexIndexKeys(8, 1, 1, getStringId(1), {std::move(values)})[0]; ASSERT_EQ(1, IndexKeyUtils::getIndexId(key)); ASSERT_EQ(getStringId(1), IndexKeyUtils::getIndexVertexID(8, key)); ASSERT_EQ(true, IndexKeyUtils::isIndexKey(key)); @@ -153,19 +153,21 @@ TEST(IndexKeyUtilsTest, vertexIndexKeyV1) { TEST(IndexKeyUtilsTest, vertexIndexKeyV2) { auto values = getIndexValues(); - auto key = IndexKeyUtils::vertexIndexKey(100, 1, 1, "vertex_1_1_1_1", std::move(values)); - ASSERT_EQ(1, IndexKeyUtils::getIndexId(key)); - - VertexID vid = "vertex_1_1_1_1"; - vid.append(100 - vid.size(), '\0'); - ASSERT_EQ(vid, IndexKeyUtils::getIndexVertexID(100, key)); - ASSERT_EQ(true, IndexKeyUtils::isIndexKey(key)); + auto keys = IndexKeyUtils::vertexIndexKeys(100, 1, 1, "vertex_1_1_1_1", {std::move(values)}); + for (auto& key : keys) { + ASSERT_EQ(1, IndexKeyUtils::getIndexId(key)); + + VertexID vid = "vertex_1_1_1_1"; + vid.append(100 - vid.size(), '\0'); + ASSERT_EQ(vid, IndexKeyUtils::getIndexVertexID(100, key)); + ASSERT_EQ(true, IndexKeyUtils::isIndexKey(key)); + } } TEST(IndexKeyUtilsTest, edgeIndexKeyV1) { auto values = getIndexValues(); - auto key = - IndexKeyUtils::edgeIndexKey(8, 1, 1, getStringId(1), 1, getStringId(2), std::move(values)); + auto key = IndexKeyUtils::edgeIndexKeys( + 8, 1, 1, getStringId(1), 1, getStringId(2), {std::move(values)})[0]; ASSERT_EQ(1, IndexKeyUtils::getIndexId(key)); ASSERT_EQ(getStringId(1), IndexKeyUtils::getIndexSrcId(8, key)); ASSERT_EQ(1, IndexKeyUtils::getIndexRank(8, key)); @@ -176,22 +178,31 @@ TEST(IndexKeyUtilsTest, edgeIndexKeyV1) { TEST(IndexKeyUtilsTest, edgeIndexKeyV2) { VertexID vid = "vertex_1_1_1_1"; auto values = getIndexValues(); - auto key = IndexKeyUtils::edgeIndexKey(100, 1, 1, vid, 1, vid, std::move(values)); - ASSERT_EQ(1, IndexKeyUtils::getIndexId(key)); - vid.append(100 - vid.size(), '\0'); - ASSERT_EQ(vid, IndexKeyUtils::getIndexSrcId(100, key)); - ASSERT_EQ(1, IndexKeyUtils::getIndexRank(100, key)); - ASSERT_EQ(vid, IndexKeyUtils::getIndexDstId(100, key)); - ASSERT_EQ(true, IndexKeyUtils::isIndexKey(key)); + auto keys = IndexKeyUtils::edgeIndexKeys(100, 1, 1, vid, 1, vid, {std::move(values)}); + for (auto& key : keys) { + ASSERT_EQ(1, IndexKeyUtils::getIndexId(key)); + vid.append(100 - vid.size(), '\0'); + ASSERT_EQ(vid, IndexKeyUtils::getIndexSrcId(100, key)); + ASSERT_EQ(1, IndexKeyUtils::getIndexRank(100, key)); + ASSERT_EQ(vid, IndexKeyUtils::getIndexDstId(100, key)); + ASSERT_EQ(true, IndexKeyUtils::isIndexKey(key)); + } - key = IndexKeyUtils::edgeIndexKey(100, 1, 1, vid, -1, vid, std::move(values)); - ASSERT_EQ(-1, IndexKeyUtils::getIndexRank(100, key)); + keys = IndexKeyUtils::edgeIndexKeys(100, 1, 1, vid, -1, vid, {std::move(values)}); + for (auto& key : keys) { + ASSERT_EQ(-1, IndexKeyUtils::getIndexRank(100, key)); + } - key = IndexKeyUtils::edgeIndexKey(100, 1, 1, vid, 9223372036854775807, vid, std::move(values)); - ASSERT_EQ(9223372036854775807, IndexKeyUtils::getIndexRank(100, key)); + keys = + IndexKeyUtils::edgeIndexKeys(100, 1, 1, vid, 9223372036854775807, vid, {std::move(values)}); + for (auto& key : keys) { + ASSERT_EQ(9223372036854775807, IndexKeyUtils::getIndexRank(100, key)); + } - key = IndexKeyUtils::edgeIndexKey(100, 1, 1, vid, 0, vid, std::move(values)); - ASSERT_EQ(0, IndexKeyUtils::getIndexRank(100, key)); + keys = IndexKeyUtils::edgeIndexKeys(100, 1, 1, vid, 0, vid, {std::move(values)}); + for (auto& key : keys) { + ASSERT_EQ(0, IndexKeyUtils::getIndexRank(100, key)); + } } TEST(IndexKeyUtilsTest, nullableValue) { @@ -212,11 +223,12 @@ TEST(IndexKeyUtilsTest, nullableValue) { values.emplace_back(Value(NullType::__NULL__)); cols.emplace_back(nullCol(folly::stringPrintf("col%ld", j), meta::cpp2::PropertyType::BOOL)); } - auto raw = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); + // TODO(jie) Add index key tests for geography + auto raws = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); u_short s = 0xfc00; /* the binary is '11111100 00000000'*/ std::string expected; expected.append(reinterpret_cast(&s), sizeof(u_short)); - auto result = raw.substr(raw.size() - sizeof(u_short), sizeof(u_short)); + auto result = raws[0].substr(raws[0].size() - sizeof(u_short), sizeof(u_short)); ASSERT_EQ(expected, result); } { @@ -227,11 +239,11 @@ TEST(IndexKeyUtilsTest, nullableValue) { for (int64_t j = 1; j <= 2; j++) { cols.emplace_back(nullCol(folly::stringPrintf("col%ld", j), meta::cpp2::PropertyType::BOOL)); } - auto raw = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); + auto raws = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); u_short s = 0x4000; /* the binary is '01000000 00000000'*/ std::string expected; expected.append(reinterpret_cast(&s), sizeof(u_short)); - auto result = raw.substr(raw.size() - sizeof(u_short), sizeof(u_short)); + auto result = raws[0].substr(raws[0].size() - sizeof(u_short), sizeof(u_short)); ASSERT_EQ(expected, result); } { @@ -242,11 +254,11 @@ TEST(IndexKeyUtilsTest, nullableValue) { for (int64_t j = 1; j <= 2; j++) { cols.emplace_back(nullCol(folly::stringPrintf("col%ld", j), meta::cpp2::PropertyType::BOOL)); } - auto raw = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); + auto raws = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); u_short s = 0x0000; /* the binary is '01000000 00000000'*/ std::string expected; expected.append(reinterpret_cast(&s), sizeof(u_short)); - auto result = raw.substr(raw.size() - sizeof(u_short), sizeof(u_short)); + auto result = raws[0].substr(raws[0].size() - sizeof(u_short), sizeof(u_short)); ASSERT_EQ(expected, result); } { @@ -257,11 +269,11 @@ TEST(IndexKeyUtilsTest, nullableValue) { cols.emplace_back(nullCol(folly::stringPrintf("col%ld", i), meta::cpp2::PropertyType::INT64)); } - auto raw = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); + auto raws = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); u_short s = 0xfff0; /* the binary is '11111111 11110000'*/ std::string expected; expected.append(reinterpret_cast(&s), sizeof(u_short)); - auto result = raw.substr(raw.size() - sizeof(u_short), sizeof(u_short)); + auto result = raws[0].substr(raws[0].size() - sizeof(u_short), sizeof(u_short)); ASSERT_EQ(expected, result); } { @@ -295,11 +307,11 @@ TEST(IndexKeyUtilsTest, nullableValue) { cols.emplace_back(nullCol(folly::stringPrintf("col_%ld_%ld", i, j), types[j])); } } - auto raw = IndexKeyUtils::encodeValues(std::move(values), cols); + auto raws = IndexKeyUtils::encodeValues(std::move(values), cols); u_short s = 0xaaa0; /* the binary is '10101010 10100000'*/ std::string expected; expected.append(reinterpret_cast(&s), sizeof(u_short)); - auto result = raw.substr(raw.size() - sizeof(u_short), sizeof(u_short)); + auto result = raws[0].substr(raws[0].size() - sizeof(u_short), sizeof(u_short)); ASSERT_EQ(expected, result); } { @@ -309,11 +321,11 @@ TEST(IndexKeyUtilsTest, nullableValue) { values.emplace_back(Value(NullType::__NULL__)); cols.emplace_back(nullCol(folly::stringPrintf("col%ld", i), meta::cpp2::PropertyType::BOOL)); } - auto raw = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); + auto raws = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); u_short s = 0xff80; /* the binary is '11111111 10000000'*/ std::string expected; expected.append(reinterpret_cast(&s), sizeof(u_short)); - auto result = raw.substr(raw.size() - sizeof(u_short), sizeof(u_short)); + auto result = raws[0].substr(raws[0].size() - sizeof(u_short), sizeof(u_short)); ASSERT_EQ(expected, result); } } @@ -408,9 +420,12 @@ TEST(IndexKeyUtilsTest, getValueFromIndexKeyTest) { std::vector indexKeys; for (auto& row : vertices) { auto values = IndexKeyUtils::encodeValues(std::move(row.second), cols); - ASSERT_EQ(indexValueSize, values.size()); - indexKeys.emplace_back( - IndexKeyUtils::vertexIndexKey(vIdLen, partId, indexId, row.first, std::move(values))); + ASSERT_EQ(indexValueSize, values[0].size()); + auto keys = + IndexKeyUtils::vertexIndexKeys(vIdLen, partId, indexId, row.first, std::move(values)); + for (auto& key : keys) { + indexKeys.emplace_back(key); + } } verifyDecodeIndexKey(false, false, vIdLen, expected, indexKeys, cols); @@ -427,9 +442,13 @@ TEST(IndexKeyUtilsTest, getValueFromIndexKeyTest) { std::vector indexKeys; for (auto& row : edges) { auto values = IndexKeyUtils::encodeValues(std::move(row.second), cols); - ASSERT_EQ(indexValueSize, values.size()); - indexKeys.emplace_back(IndexKeyUtils::edgeIndexKey( - vIdLen, partId, indexId, row.first, 0, row.first, std::move(values))); + ASSERT_EQ(indexValueSize, values[0].size()); + + auto keys = IndexKeyUtils::edgeIndexKeys( + vIdLen, partId, indexId, row.first, 0, row.first, std::move(values)); + for (auto& key : keys) { + indexKeys.emplace_back(key); + } } verifyDecodeIndexKey(true, false, vIdLen, expected, indexKeys, cols); @@ -459,10 +478,12 @@ TEST(IndexKeyUtilsTest, getValueFromIndexKeyTest) { std::vector indexKeys; for (auto& row : vertices) { auto values = IndexKeyUtils::encodeValues(std::move(row.second), cols); - ASSERT_EQ(indexValueSize, values.size()); - auto key = - IndexKeyUtils::vertexIndexKey(vIdLen, partId, indexId, row.first, std::move(values)); - indexKeys.emplace_back(key); + ASSERT_EQ(indexValueSize, values[0].size()); + auto keys = + IndexKeyUtils::vertexIndexKeys(vIdLen, partId, indexId, row.first, std::move(values)); + for (auto& key : keys) { + indexKeys.emplace_back(key); + } } verifyDecodeIndexKey(false, true, vIdLen, expected, indexKeys, cols); } @@ -484,9 +505,12 @@ TEST(IndexKeyUtilsTest, getValueFromIndexKeyTest) { std::vector indexKeys; for (auto& row : edges) { auto values = IndexKeyUtils::encodeValues(std::move(row.second), cols); - ASSERT_EQ(indexValueSize, values.size()); - indexKeys.emplace_back(IndexKeyUtils::edgeIndexKey( - vIdLen, partId, indexId, row.first, 0, row.first, std::move(values))); + ASSERT_EQ(indexValueSize, values[0].size()); + auto keys = IndexKeyUtils::edgeIndexKeys( + vIdLen, partId, indexId, row.first, 0, row.first, std::move(values)); + for (auto& key : keys) { + indexKeys.emplace_back(key); + } } verifyDecodeIndexKey(true, true, vIdLen, expected, indexKeys, cols); diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt index 13eaa0e673e..565c2717f54 100644 --- a/src/daemons/CMakeLists.txt +++ b/src/daemons/CMakeLists.txt @@ -45,6 +45,7 @@ set(storage_meta_deps $ $ $ + $ ) nebula_add_executable( @@ -118,7 +119,6 @@ nebula_add_executable( $ $ $ - $ $ $ $ @@ -128,6 +128,7 @@ nebula_add_executable( $ $ $ + $ ${common_deps} LIBRARIES ${PROXYGEN_LIBRARIES} diff --git a/src/daemons/GraphDaemon.cpp b/src/daemons/GraphDaemon.cpp index 34f5850292a..2efa800ee73 100644 --- a/src/daemons/GraphDaemon.cpp +++ b/src/daemons/GraphDaemon.cpp @@ -106,12 +106,12 @@ int main(int argc, char *argv[]) { } } - // Get the IPv4 address the server will listen on - if (FLAGS_local_ip.empty()) { - LOG(ERROR) << "local_ip is empty, need to config it through config file"; + // Validate the IPv4 address or hostname + status = NetworkUtils::validateHostOrIp(FLAGS_local_ip); + if (!status.ok()) { + LOG(ERROR) << status; return EXIT_FAILURE; } - // TODO: Check the ip is valid nebula::HostAddr localhost{FLAGS_local_ip, FLAGS_port}; // Initialize the global timezone, it's only used for datetime type compute diff --git a/src/daemons/MetaDaemon.cpp b/src/daemons/MetaDaemon.cpp index 83bdb2e9643..0174859311d 100644 --- a/src/daemons/MetaDaemon.cpp +++ b/src/daemons/MetaDaemon.cpp @@ -36,6 +36,7 @@ using nebula::operator<<; using nebula::ProcessUtils; using nebula::Status; using nebula::StatusOr; +using nebula::network::NetworkUtils; using nebula::web::PathParams; DEFINE_string(local_ip, "", "Local ip specified for NetworkUtils::getLocalIP"); @@ -247,10 +248,19 @@ int main(int argc, char* argv[]) { } } - auto hostIdentity = - FLAGS_local_ip == "" ? nebula::network::NetworkUtils::getHostname() : FLAGS_local_ip; - nebula::HostAddr localhost{hostIdentity, FLAGS_port}; - LOG(INFO) << "identify myself as " << localhost; + std::string hostName; + if (FLAGS_local_ip.empty()) { + hostName = nebula::network::NetworkUtils::getHostname(); + } else { + status = NetworkUtils::validateHostOrIp(FLAGS_local_ip); + if (!status.ok()) { + LOG(ERROR) << status; + return EXIT_FAILURE; + } + hostName = FLAGS_local_ip; + } + nebula::HostAddr localhost{hostName, FLAGS_port}; + LOG(INFO) << "localhost = " << localhost; auto peersRet = nebula::network::NetworkUtils::toHosts(FLAGS_meta_server_addrs); if (!peersRet.ok()) { LOG(ERROR) << "Can't get peers address, status:" << peersRet.status(); diff --git a/src/daemons/StorageDaemon.cpp b/src/daemons/StorageDaemon.cpp index 58603620e1a..f1bbf752cd6 100644 --- a/src/daemons/StorageDaemon.cpp +++ b/src/daemons/StorageDaemon.cpp @@ -111,10 +111,19 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } - auto hostName = - FLAGS_local_ip != "" ? FLAGS_local_ip : nebula::network::NetworkUtils::getHostname(); - HostAddr host(hostName, FLAGS_port); - LOG(INFO) << "host = " << host; + std::string hostName; + if (FLAGS_local_ip.empty()) { + hostName = nebula::network::NetworkUtils::getHostname(); + } else { + status = NetworkUtils::validateHostOrIp(FLAGS_local_ip); + if (!status.ok()) { + LOG(ERROR) << status; + return EXIT_FAILURE; + } + hostName = FLAGS_local_ip; + } + nebula::HostAddr localhost{hostName, FLAGS_port}; + LOG(INFO) << "localhost = " << localhost; auto metaAddrsRet = nebula::network::NetworkUtils::toHosts(FLAGS_meta_server_addrs); if (!metaAddrsRet.ok() || metaAddrsRet.value().empty()) { LOG(ERROR) << "Can't get metaServer address, status:" << metaAddrsRet.status() @@ -148,7 +157,7 @@ int main(int argc, char *argv[]) { } gStorageServer = std::make_unique( - host, metaAddrsRet.value(), paths, FLAGS_wal_path, FLAGS_listener_path); + localhost, metaAddrsRet.value(), paths, FLAGS_wal_path, FLAGS_listener_path); if (!gStorageServer->start()) { LOG(ERROR) << "Storage server start failed"; gStorageServer->stop(); diff --git a/src/graph/context/ast/QueryAstContext.h b/src/graph/context/ast/QueryAstContext.h index fed455a2198..b4fb5596fbc 100644 --- a/src/graph/context/ast/QueryAstContext.h +++ b/src/graph/context/ast/QueryAstContext.h @@ -157,6 +157,17 @@ struct FetchEdgesContext final : public AstContext { std::string inputVarName; }; +struct AlterSchemaContext final : public AstContext { + std::vector schemaItems; + meta::cpp2::SchemaProp schemaProps; +}; + +struct CreateSchemaContext final : public AstContext { + bool ifNotExist{false}; + std::string name; + meta::cpp2::Schema schema; +}; + } // namespace graph } // namespace nebula #endif // GRAPH_CONTEXT_AST_QUERYASTCONTEXT_H_ diff --git a/src/graph/optimizer/CMakeLists.txt b/src/graph/optimizer/CMakeLists.txt index 06484cba4d7..361463cd882 100644 --- a/src/graph/optimizer/CMakeLists.txt +++ b/src/graph/optimizer/CMakeLists.txt @@ -32,6 +32,9 @@ nebula_add_library( rule/UnionAllIndexScanBaseRule.cpp rule/UnionAllTagIndexScanRule.cpp rule/UnionAllEdgeIndexScanRule.cpp + rule/GeoPredicateIndexScanBaseRule.cpp + rule/GeoPredicateTagIndexScanRule.cpp + rule/GeoPredicateEdgeIndexScanRule.cpp rule/IndexFullScanBaseRule.cpp rule/TagIndexFullScanRule.cpp rule/EdgeIndexFullScanRule.cpp diff --git a/src/graph/optimizer/rule/GeoPredicateEdgeIndexScanRule.cpp b/src/graph/optimizer/rule/GeoPredicateEdgeIndexScanRule.cpp new file mode 100644 index 00000000000..758cc457016 --- /dev/null +++ b/src/graph/optimizer/rule/GeoPredicateEdgeIndexScanRule.cpp @@ -0,0 +1,32 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "graph/optimizer/rule/GeoPredicateEdgeIndexScanRule.h" + +using Kind = nebula::graph::PlanNode::Kind; + +namespace nebula { +namespace opt { + +std::unique_ptr GeoPredicateEdgeIndexScanRule::kInstance = + std::unique_ptr(new GeoPredicateEdgeIndexScanRule()); + +GeoPredicateEdgeIndexScanRule::GeoPredicateEdgeIndexScanRule() { + RuleSet::DefaultRules().addRule(this); +} + +const Pattern& GeoPredicateEdgeIndexScanRule::pattern() const { + static Pattern pattern = + Pattern::create(Kind::kFilter, {Pattern::create(Kind::kEdgeIndexFullScan)}); + return pattern; +} + +std::string GeoPredicateEdgeIndexScanRule::toString() const { + return "GeoPredicateEdgeIndexScanRule"; +} + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/GeoPredicateEdgeIndexScanRule.h b/src/graph/optimizer/rule/GeoPredicateEdgeIndexScanRule.h new file mode 100644 index 00000000000..91a1dd28adc --- /dev/null +++ b/src/graph/optimizer/rule/GeoPredicateEdgeIndexScanRule.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#pragma once + +#include "graph/optimizer/rule/GeoPredicateIndexScanBaseRule.h" + +namespace nebula { +namespace opt { + +class GeoPredicateEdgeIndexScanRule final : public GeoPredicateIndexScanBaseRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + GeoPredicateEdgeIndexScanRule(); + + static std::unique_ptr kInstance; +}; + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp b/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp new file mode 100644 index 00000000000..5f7eacd82d9 --- /dev/null +++ b/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp @@ -0,0 +1,142 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "graph/optimizer/rule/GeoPredicateIndexScanBaseRule.h" + +#include "common/expression/Expression.h" +#include "common/expression/LogicalExpression.h" +#include "common/geo/GeoIndex.h" +#include "graph/optimizer/OptContext.h" +#include "graph/optimizer/OptGroup.h" +#include "graph/optimizer/OptRule.h" +#include "graph/optimizer/OptimizerUtils.h" +#include "graph/planner/plan/PlanNode.h" +#include "graph/planner/plan/Query.h" +#include "graph/planner/plan/Scan.h" +#include "graph/util/ExpressionUtils.h" +#include "interface/gen-cpp2/storage_types.h" + +using nebula::graph::Filter; +using nebula::graph::IndexScan; +using nebula::graph::OptimizerUtils; +using nebula::graph::TagIndexFullScan; +using nebula::storage::cpp2::IndexColumnHint; +using nebula::storage::cpp2::IndexQueryContext; + +using Kind = nebula::graph::PlanNode::Kind; +using ExprKind = nebula::Expression::Kind; +using TransformResult = nebula::opt::OptRule::TransformResult; + +namespace nebula { +namespace opt { + +bool GeoPredicateIndexScanBaseRule::match(OptContext* ctx, const MatchedResult& matched) const { + if (!OptRule::match(ctx, matched)) { + return false; + } + auto filter = static_cast(matched.planNode()); + auto scan = static_cast(matched.planNode({0, 0})); + for (auto& ictx : scan->queryContext()) { + if (ictx.column_hints_ref().is_set()) { + return false; + } + } + auto condition = filter->condition(); + return graph::ExpressionUtils::isGeoIndexAcceleratedPredicate(condition); +} + +StatusOr GeoPredicateIndexScanBaseRule::transform( + OptContext* ctx, const MatchedResult& matched) const { + auto filter = static_cast(matched.planNode()); + auto node = matched.planNode({0, 0}); + auto scan = static_cast(node); + + auto metaClient = ctx->qctx()->getMetaClient(); + auto status = node->kind() == graph::PlanNode::Kind::kTagIndexFullScan + ? metaClient->getTagIndexesFromCache(scan->space()) + : metaClient->getEdgeIndexesFromCache(scan->space()); + + NG_RETURN_IF_ERROR(status); + auto indexItems = std::move(status).value(); + + OptimizerUtils::eraseInvalidIndexItems(scan->schemaId(), &indexItems); + + if (indexItems.empty()) { + return TransformResult::noTransform(); + } + + auto condition = filter->condition(); + DCHECK(graph::ExpressionUtils::isGeoIndexAcceleratedPredicate(condition)); + + auto* geoPredicate = static_cast(condition); + DCHECK_GE(geoPredicate->args()->numArgs(), 2); + std::string geoPredicateName = geoPredicate->name(); + folly::toLowerAscii(geoPredicateName); + auto* first = geoPredicate->args()->args()[0]; + auto* second = geoPredicate->args()->args()[1]; + DCHECK(first->kind() == Expression::Kind::kTagProperty || + first->kind() == Expression::Kind::kEdgeProperty); + DCHECK(second->kind() == Expression::Kind::kConstant); + const auto& secondVal = static_cast(second)->value(); + DCHECK(secondVal.type() == Value::Type::GEOGRAPHY); + const auto& geog = secondVal.getGeography(); + + // TODO(jie): Get index params from meta to construct RegionCoverParams + geo::RegionCoverParams rc; + // TODO(jie): Get schema meta to know if it's point only + geo::GeoIndex geoIndex(rc, false); + std::vector scanRanges; + if (geoPredicateName == "st_intersects") { + scanRanges = geoIndex.intersects(geog); + } else if (geoPredicateName == "st_covers") { + scanRanges = geoIndex.coveredBy(geog); + } else if (geoPredicateName == "st_coveredby") { + scanRanges = geoIndex.covers(geog); + } else if (geoPredicateName == "st_dwithin") { + DCHECK_GE(geoPredicate->args()->numArgs(), 3); + auto* third = geoPredicate->args()->args()[2]; + DCHECK_EQ(third->kind(), Expression::Kind::kConstant); + const auto& thirdVal = static_cast(third)->value(); + DCHECK_EQ(thirdVal.type(), Value::Type::FLOAT); + double distanceInMeters = thirdVal.getFloat(); + scanRanges = geoIndex.dWithin(geog, distanceInMeters); + } + std::vector idxCtxs; + idxCtxs.reserve(scanRanges.size()); + auto indexItem = indexItems.back(); + const auto& fields = indexItem->get_fields(); + DCHECK_EQ(fields.size(), 1); // geo field + auto fieldName = fields.back().get_name(); + for (auto& scanRange : scanRanges) { + IndexQueryContext ictx; + auto indexColumnHint = scanRange.toIndexColumnHint(); + indexColumnHint.set_column_name(fieldName); + ictx.set_filter(condition->encode()); + ictx.set_index_id(indexItem->get_index_id()); + ictx.set_column_hints({indexColumnHint}); + idxCtxs.emplace_back(std::move(ictx)); + } + + auto scanNode = IndexScan::make(ctx->qctx(), nullptr); + OptimizerUtils::copyIndexScanData(scan, scanNode); + scanNode->setIndexQueryContext(std::move(idxCtxs)); + // TODO(jie): geo predicate's caculation is a little heavy, + // which is not suitable to push down to the storage + scanNode->setOutputVar(filter->outputVar()); + scanNode->setColNames(filter->colNames()); + auto filterGroup = matched.node->group(); + auto optScanNode = OptGroupNode::create(ctx, scanNode, filterGroup); + for (auto group : matched.dependencies[0].node->dependencies()) { + optScanNode->dependsOn(group); + } + TransformResult result; + result.newGroupNodes.emplace_back(optScanNode); + result.eraseCurr = true; + return result; +} + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.h b/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.h new file mode 100644 index 00000000000..f5abfe62592 --- /dev/null +++ b/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.h @@ -0,0 +1,21 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#pragma once + +#include "graph/optimizer/OptRule.h" + +namespace nebula { +namespace opt { + +class GeoPredicateIndexScanBaseRule : public OptRule { + public: + bool match(OptContext *ctx, const MatchedResult &matched) const override; + StatusOr transform(OptContext *ctx, const MatchedResult &matched) const override; +}; + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/GeoPredicateTagIndexScanRule.cpp b/src/graph/optimizer/rule/GeoPredicateTagIndexScanRule.cpp new file mode 100644 index 00000000000..c115e48ce5b --- /dev/null +++ b/src/graph/optimizer/rule/GeoPredicateTagIndexScanRule.cpp @@ -0,0 +1,32 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "graph/optimizer/rule/GeoPredicateTagIndexScanRule.h" + +using Kind = nebula::graph::PlanNode::Kind; + +namespace nebula { +namespace opt { + +std::unique_ptr GeoPredicateTagIndexScanRule::kInstance = + std::unique_ptr(new GeoPredicateTagIndexScanRule()); + +GeoPredicateTagIndexScanRule::GeoPredicateTagIndexScanRule() { + RuleSet::DefaultRules().addRule(this); +} + +const Pattern& GeoPredicateTagIndexScanRule::pattern() const { + static Pattern pattern = + Pattern::create(Kind::kFilter, {Pattern::create(Kind::kTagIndexFullScan)}); + return pattern; +} + +std::string GeoPredicateTagIndexScanRule::toString() const { + return "GeoPredicateTagIndexScanRule"; +} + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/GeoPredicateTagIndexScanRule.h b/src/graph/optimizer/rule/GeoPredicateTagIndexScanRule.h new file mode 100644 index 00000000000..e2dfc82cf14 --- /dev/null +++ b/src/graph/optimizer/rule/GeoPredicateTagIndexScanRule.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#pragma once + +#include "graph/optimizer/rule/GeoPredicateIndexScanBaseRule.h" + +namespace nebula { +namespace opt { + +class GeoPredicateTagIndexScanRule final : public GeoPredicateIndexScanBaseRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + GeoPredicateTagIndexScanRule(); + + static std::unique_ptr kInstance; +}; + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/OptimizeEdgeIndexScanByFilterRule.cpp b/src/graph/optimizer/rule/OptimizeEdgeIndexScanByFilterRule.cpp index 42021d4422d..7433c9af986 100644 --- a/src/graph/optimizer/rule/OptimizeEdgeIndexScanByFilterRule.cpp +++ b/src/graph/optimizer/rule/OptimizeEdgeIndexScanByFilterRule.cpp @@ -70,6 +70,13 @@ bool OptimizeEdgeIndexScanByFilterRule::match(OptContext* ctx, const MatchedResu auto condition = filter->condition(); if (condition->isRelExpr()) { auto relExpr = static_cast(condition); + // If the container in the IN expr has only 1 element, it will be converted to an relEQ + // expr. If more than 1 element found in the container, UnionAllIndexScanBaseRule will be + // applied. + if (relExpr->kind() == ExprKind::kRelIn && relExpr->right()->isContainerExpr()) { + auto ContainerOperands = graph::ExpressionUtils::getContainerExprOperands(relExpr->right()); + return ContainerOperands.size() == 1; + } return relExpr->left()->kind() == ExprKind::kEdgeProperty && relExpr->right()->kind() == ExprKind::kConstant; } @@ -103,13 +110,43 @@ StatusOr OptimizeEdgeIndexScanByFilterRule::transform( OptimizerUtils::eraseInvalidIndexItems(scan->schemaId(), &indexItems); + auto condition = filter->condition(); + auto conditionType = condition->kind(); + Expression* transformedExpr = condition->clone(); + + // Stand alone IN expr with only 1 element in the list, no need to check index + if (conditionType == ExprKind::kRelIn) { + transformedExpr = graph::ExpressionUtils::rewriteInExpr(condition); + DCHECK(transformedExpr->kind() == ExprKind::kRelEQ); + } + + // case2: logical AND expr + if (condition->kind() == ExprKind::kLogicalAnd) { + for (auto& operand : static_cast(condition)->operands()) { + if (operand->kind() == ExprKind::kRelIn) { + auto inExpr = static_cast(operand); + // Do not apply this rule if the IN expr has a valid index or it has only 1 element in the + // list + if (static_cast(inExpr->right())->size() > 1) { + return TransformResult::noTransform(); + } else { + transformedExpr = graph::ExpressionUtils::rewriteInExpr(condition); + } + if (OptimizerUtils::relExprHasIndex(inExpr, indexItems)) { + return TransformResult::noTransform(); + } + } + } + } + IndexQueryContext ictx; bool isPrefixScan = false; - if (!OptimizerUtils::findOptimalIndex(filter->condition(), indexItems, &isPrefixScan, &ictx)) { + if (!OptimizerUtils::findOptimalIndex(transformedExpr, indexItems, &isPrefixScan, &ictx)) { return TransformResult::noTransform(); } + std::vector idxCtxs = {ictx}; - EdgeIndexScan* scanNode = makeEdgeIndexScan(ctx->qctx(), scan, isPrefixScan); + auto scanNode = makeEdgeIndexScan(ctx->qctx(), scan, isPrefixScan); scanNode->setIndexQueryContext(std::move(idxCtxs)); scanNode->setOutputVar(filter->outputVar()); scanNode->setColNames(filter->colNames()); diff --git a/src/graph/optimizer/rule/UnionAllIndexScanBaseRule.cpp b/src/graph/optimizer/rule/UnionAllIndexScanBaseRule.cpp index 2333ba672f4..ee32b926897 100644 --- a/src/graph/optimizer/rule/UnionAllIndexScanBaseRule.cpp +++ b/src/graph/optimizer/rule/UnionAllIndexScanBaseRule.cpp @@ -32,7 +32,7 @@ namespace nebula { namespace opt { // The matched expression should be either a OR expression or an expression that could be -// rewrote to a OR expression. There are 3 senarios. +// rewrote to a OR expression. There are 3 scenarios. // // 1. OR expr. If OR expr has an IN expr operand that has a valid index, expand it to OR expr. // @@ -162,8 +162,7 @@ StatusOr UnionAllIndexScanBaseRule::transform(OptContext* ctx, break; } - DCHECK(transformedExpr->kind() == ExprKind::kLogicalOr || - transformedExpr->kind() == ExprKind::kRelEQ); + DCHECK(transformedExpr->kind() == ExprKind::kLogicalOr); std::vector idxCtxs; auto logicalExpr = static_cast(transformedExpr); for (auto operand : logicalExpr->operands()) { diff --git a/src/graph/optimizer/test/CMakeLists.txt b/src/graph/optimizer/test/CMakeLists.txt index 1d99b18f5cb..c9a3d8245b1 100644 --- a/src/graph/optimizer/test/CMakeLists.txt +++ b/src/graph/optimizer/test/CMakeLists.txt @@ -46,6 +46,7 @@ set(OPTIMIZER_TEST_LIB $ $ $ + $ ) nebula_add_test( diff --git a/src/graph/planner/CMakeLists.txt b/src/graph/planner/CMakeLists.txt index 47d76119229..5f822cafd73 100644 --- a/src/graph/planner/CMakeLists.txt +++ b/src/graph/planner/CMakeLists.txt @@ -42,4 +42,5 @@ nebula_add_library( ngql/LookupPlanner.cpp ngql/FetchVerticesPlanner.cpp ngql/FetchEdgesPlanner.cpp + ngql/MaintainPlanner.cpp ) diff --git a/src/graph/planner/PlannersRegister.cpp b/src/graph/planner/PlannersRegister.cpp index 3cdea03b920..15b3d4064d8 100644 --- a/src/graph/planner/PlannersRegister.cpp +++ b/src/graph/planner/PlannersRegister.cpp @@ -17,6 +17,7 @@ #include "graph/planner/ngql/FetchVerticesPlanner.h" #include "graph/planner/ngql/GoPlanner.h" #include "graph/planner/ngql/LookupPlanner.h" +#include "graph/planner/ngql/MaintainPlanner.h" #include "graph/planner/ngql/PathPlanner.h" #include "graph/planner/ngql/SubgraphPlanner.h" @@ -24,10 +25,30 @@ namespace nebula { namespace graph { void PlannersRegister::registPlanners() { + registDDL(); registSequential(); registMatch(); } +void PlannersRegister::registDDL() { + { + auto& planners = Planner::plannersMap()[Sentence::Kind::kAlterTag]; + planners.emplace_back(&AlterTagPlanner::match, &AlterTagPlanner::make); + } + { + auto& planners = Planner::plannersMap()[Sentence::Kind::kAlterEdge]; + planners.emplace_back(&AlterEdgePlanner::match, &AlterEdgePlanner::make); + } + { + auto& planners = Planner::plannersMap()[Sentence::Kind::kCreateTag]; + planners.emplace_back(&CreateTagPlanner::match, &CreateTagPlanner::make); + } + { + auto& planners = Planner::plannersMap()[Sentence::Kind::kCreateEdge]; + planners.emplace_back(&CreateEdgePlanner::match, &CreateEdgePlanner::make); + } +} + void PlannersRegister::registSequential() { { auto& planners = Planner::plannersMap()[Sentence::Kind::kSequential]; diff --git a/src/graph/planner/PlannersRegister.h b/src/graph/planner/PlannersRegister.h index 995a4a3615e..bc89cac3870 100644 --- a/src/graph/planner/PlannersRegister.h +++ b/src/graph/planner/PlannersRegister.h @@ -18,6 +18,7 @@ class PlannersRegister final { static void registPlanners(); private: + static void registDDL(); static void registSequential(); static void registMatch(); }; diff --git a/src/graph/planner/ngql/GoPlanner.cpp b/src/graph/planner/ngql/GoPlanner.cpp index fc0a38ddda3..aa747c2063d 100644 --- a/src/graph/planner/ngql/GoPlanner.cpp +++ b/src/graph/planner/ngql/GoPlanner.cpp @@ -504,7 +504,7 @@ SubPlan GoPlanner::mToNStepsPlan(SubPlan& startVidPlan) { } const auto& projectInput = - (joinInput || joinDst) ? loopBody->outputVar() : sampleLimit->outputVar(); + (loopBody != getDst) ? loopBody->outputVar() : sampleLimit->outputVar(); loopBody = Project::make(qctx, loopBody, goCtx_->yieldExpr); loopBody->setInputVar(projectInput); loopBody->setColNames(std::move(goCtx_->colNames)); diff --git a/src/graph/planner/ngql/MaintainPlanner.cpp b/src/graph/planner/ngql/MaintainPlanner.cpp new file mode 100644 index 00000000000..2d3e92debe6 --- /dev/null +++ b/src/graph/planner/ngql/MaintainPlanner.cpp @@ -0,0 +1,66 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "graph/planner/ngql/MaintainPlanner.h" + +#include "graph/context/ast/QueryAstContext.h" +#include "graph/planner/plan/Maintain.h" + +namespace nebula { +namespace graph { + +StatusOr CreateTagPlanner::transform(AstContext* astCtx) { + auto createCtx = static_cast(astCtx); + SubPlan plan; + plan.root = plan.tail = CreateTag::make(createCtx->qctx, + nullptr, + std::move(createCtx->name), + std::move(createCtx->schema), + createCtx->ifNotExist); + return plan; +} + +StatusOr CreateEdgePlanner::transform(AstContext* astCtx) { + auto createCtx = static_cast(astCtx); + SubPlan plan; + plan.root = plan.tail = CreateEdge::make(createCtx->qctx, + nullptr, + std::move(createCtx->name), + std::move(createCtx->schema), + createCtx->ifNotExist); + return plan; +} + +StatusOr AlterTagPlanner::transform(AstContext* astCtx) { + auto alterCtx = static_cast(astCtx); + auto qctx = alterCtx->qctx; + auto name = *static_cast(alterCtx->sentence)->name(); + SubPlan plan; + plan.root = plan.tail = AlterTag::make(qctx, + nullptr, + alterCtx->space.id, + std::move(name), + std::move(alterCtx->schemaItems), + std::move(alterCtx->schemaProps)); + return plan; +} + +StatusOr AlterEdgePlanner::transform(AstContext* astCtx) { + auto alterCtx = static_cast(astCtx); + auto qctx = alterCtx->qctx; + auto name = *static_cast(alterCtx->sentence)->name(); + SubPlan plan; + plan.root = plan.tail = AlterEdge::make(qctx, + nullptr, + alterCtx->space.id, + std::move(name), + std::move(alterCtx->schemaItems), + std::move(alterCtx->schemaProps)); + return plan; +} + +} // namespace graph +} // namespace nebula diff --git a/src/graph/planner/ngql/MaintainPlanner.h b/src/graph/planner/ngql/MaintainPlanner.h new file mode 100644 index 00000000000..eb8a8953652 --- /dev/null +++ b/src/graph/planner/ngql/MaintainPlanner.h @@ -0,0 +1,62 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#pragma once + +#include "common/base/Base.h" +#include "graph/planner/Planner.h" +#include "graph/planner/plan/PlanNode.h" + +namespace nebula { +namespace graph { + +struct AstContext; + +class CreateTagPlanner final : public Planner { + public: + static std::unique_ptr make() { + return std::unique_ptr(new CreateTagPlanner()); + } + static bool match(AstContext* astCtx) { + return astCtx->sentence->kind() == Sentence::Kind::kCreateTag; + } + StatusOr transform(AstContext* astCtx) override; +}; + +class CreateEdgePlanner final : public Planner { + public: + static std::unique_ptr make() { + return std::unique_ptr(new CreateEdgePlanner()); + } + static bool match(AstContext* astCtx) { + return astCtx->sentence->kind() == Sentence::Kind::kCreateEdge; + } + StatusOr transform(AstContext* astCtx) override; +}; + +class AlterTagPlanner final : public Planner { + public: + static std::unique_ptr make() { + return std::unique_ptr(new AlterTagPlanner()); + } + static bool match(AstContext* astCtx) { + return astCtx->sentence->kind() == Sentence::Kind::kAlterTag; + } + StatusOr transform(AstContext* astCtx) override; +}; + +class AlterEdgePlanner final : public Planner { + public: + static std::unique_ptr make() { + return std::unique_ptr(new AlterEdgePlanner()); + } + static bool match(AstContext* astCtx) { + return astCtx->sentence->kind() == Sentence::Kind::kAlterEdge; + } + StatusOr transform(AstContext* astCtx) override; +}; + +} // namespace graph +} // namespace nebula diff --git a/src/graph/util/ExpressionUtils.cpp b/src/graph/util/ExpressionUtils.cpp index 932a638d561..28bb5d3ff42 100644 --- a/src/graph/util/ExpressionUtils.cpp +++ b/src/graph/util/ExpressionUtils.cpp @@ -899,5 +899,32 @@ Expression *ExpressionUtils::equalCondition(ObjectPool *pool, return RelationalExpression::makeEQ( pool, VariableExpression::make(pool, var), ConstantExpression::make(pool, value)); } + +bool ExpressionUtils::isGeoIndexAcceleratedPredicate(const Expression *expr) { + static std::unordered_set geoIndexAcceleratedPredicates{ + "st_intersects", "st_covers", "st_coveredby", "st_dwithin"}; + + if (expr->isRelExpr()) { + auto *relExpr = static_cast(expr); + auto isSt_Distance = [](const Expression *e) { + if (e->kind() == Expression::Kind::kFunctionCall) { + auto *funcExpr = static_cast(e); + return boost::algorithm::iequals(funcExpr->name(), "st_distance"); + } + return false; + }; + return isSt_Distance(relExpr->left()) || isSt_Distance(relExpr->right()); + } else if (expr->kind() == Expression::Kind::kFunctionCall) { + auto *funcExpr = static_cast(expr); + std::string funcName = funcExpr->name(); + folly::toLowerAscii(funcName); + if (geoIndexAcceleratedPredicates.count(funcName) != 0) { + return true; + } + } + + return false; +} + } // namespace graph } // namespace nebula diff --git a/src/graph/util/ExpressionUtils.h b/src/graph/util/ExpressionUtils.h index a4faeaf5777..7af0522a09b 100644 --- a/src/graph/util/ExpressionUtils.h +++ b/src/graph/util/ExpressionUtils.h @@ -156,6 +156,9 @@ class ExpressionUtils { // var == value static Expression* equalCondition(ObjectPool* pool, const std::string& var, const Value& value); + + // TODO(jie) Move it to a better place + static bool isGeoIndexAcceleratedPredicate(const Expression* expr); }; } // namespace graph diff --git a/src/graph/util/SchemaUtil.cpp b/src/graph/util/SchemaUtil.cpp index ccbcb9ca028..b3d054b711d 100644 --- a/src/graph/util/SchemaUtil.cpp +++ b/src/graph/util/SchemaUtil.cpp @@ -67,7 +67,8 @@ std::shared_ptr SchemaUtil::generateSchemaProv col.get_type().get_type(), col.type.type_length_ref().value_or(0), col.nullable_ref().value_or(false), - hasDef ? defaultValueExpr : nullptr); + hasDef ? defaultValueExpr : nullptr, + col.type.geo_shape_ref().value_or(meta::cpp2::GeoShape::ANY)); } return schemaPtr; } @@ -259,6 +260,12 @@ std::string SchemaUtil::typeToString(const meta::cpp2::ColumnTypeDef &col) { auto type = apache::thrift::util::enumNameSafe(col.get_type()); if (col.get_type() == meta::cpp2::PropertyType::FIXED_STRING) { return folly::stringPrintf("%s(%d)", type.c_str(), *col.get_type_length()); + } else if (col.get_type() == meta::cpp2::PropertyType::GEOGRAPHY) { + auto geoShape = *col.get_geo_shape(); + if (geoShape == meta::cpp2::GeoShape::ANY) { + return type; + } + return folly::sformat("{}({})", type, apache::thrift::util::enumNameSafe(geoShape)); } return type; } diff --git a/src/graph/validator/FetchEdgesValidator.cpp b/src/graph/validator/FetchEdgesValidator.cpp index 88bd8c33cad..0dc826c9650 100644 --- a/src/graph/validator/FetchEdgesValidator.cpp +++ b/src/graph/validator/FetchEdgesValidator.cpp @@ -105,7 +105,7 @@ Status FetchEdgesValidator::validateEdgeKey() { auto keys = sentence->keys()->keys(); edgeKeys.rows.reserve(keys.size()); for (const auto &key : keys) { - if (!evaluableExpr(key->srcid())) { + if (!ExpressionUtils::isEvaluableExpr(key->srcid())) { return Status::SemanticError("`%s' is not evaluable.", key->srcid()->toString().c_str()); } auto src = key->srcid()->eval(ctx); @@ -116,7 +116,7 @@ Status FetchEdgesValidator::validateEdgeKey() { } auto ranking = key->rank(); - if (!evaluableExpr(key->dstid())) { + if (!ExpressionUtils::isEvaluableExpr(key->dstid())) { return Status::SemanticError("`%s' is not evaluable.", key->dstid()->toString().c_str()); } auto dst = key->dstid()->eval(ctx); diff --git a/src/graph/validator/FetchVerticesValidator.cpp b/src/graph/validator/FetchVerticesValidator.cpp index 114c983a281..3dbcd19eaa5 100644 --- a/src/graph/validator/FetchVerticesValidator.cpp +++ b/src/graph/validator/FetchVerticesValidator.cpp @@ -26,21 +26,6 @@ Status FetchVerticesValidator::validateImpl() { return Status::OK(); } -Expression *FetchVerticesValidator::rewriteIDVertex2Vid(const Expression *expr) { - auto *pool = qctx_->objPool(); - auto matcher = [](const Expression *e) -> bool { - std::string lowerStr = e->toString(); - folly::toLowerAscii(lowerStr); - return e->kind() == Expression::Kind::kFunctionCall && lowerStr == "id(vertex)"; - }; - auto rewriter = [pool](const Expression *e) -> Expression * { - UNUSED(e); - return InputPropertyExpression::make(pool, nebula::kVid); - }; - - return RewriteVisitor::transform(expr, std::move(matcher), std::move(rewriter)); -} - Status FetchVerticesValidator::validateTag(const NameLabelList *nameLabels) { if (nameLabels == nullptr) { // all tag @@ -89,13 +74,6 @@ Status FetchVerticesValidator::validateYield(YieldClause *yield) { } auto &exprProps = fetchCtx_->exprProps; - for (const auto &col : yield->columns()) { - if (col->expr()->kind() == Expression::Kind::kVertex) { - extractVertexProp(exprProps); - break; - } - } - for (auto col : yield->columns()) { if (ExpressionUtils::hasAny(col->expr(), {Expression::Kind::kEdge, Expression::Kind::kPathBuild})) { @@ -103,13 +81,17 @@ Status FetchVerticesValidator::validateYield(YieldClause *yield) { } col->setExpr(ExpressionUtils::rewriteLabelAttr2TagProp(col->expr())); NG_RETURN_IF_ERROR(ValidateUtil::invalidLabelIdentifiers(col->expr())); + auto colExpr = col->expr(); auto typeStatus = deduceExprType(colExpr); NG_RETURN_IF_ERROR(typeStatus); outputs_.emplace_back(col->name(), typeStatus.value()); - if (colExpr->kind() == Expression::Kind::kFunctionCall) { + if (colExpr->toString() == "id(VERTEX)") { col->setAlias(col->name()); - col->setExpr(rewriteIDVertex2Vid(colExpr)); + col->setExpr(InputPropertyExpression::make(pool, nebula::kVid)); + } + if (ExpressionUtils::hasAny(colExpr, {Expression::Kind::kVertex})) { + extractVertexProp(exprProps); } newCols->addColumn(col->clone().release()); diff --git a/src/graph/validator/FetchVerticesValidator.h b/src/graph/validator/FetchVerticesValidator.h index 608aa39ffd7..fda754f393d 100644 --- a/src/graph/validator/FetchVerticesValidator.h +++ b/src/graph/validator/FetchVerticesValidator.h @@ -30,8 +30,6 @@ class FetchVerticesValidator final : public Validator { void extractVertexProp(ExpressionProps& exprProps); - Expression* rewriteIDVertex2Vid(const Expression* expr); - private: std::map> tagsSchema_; diff --git a/src/graph/validator/GoValidator.cpp b/src/graph/validator/GoValidator.cpp index 571278a1706..5527e38d4a9 100644 --- a/src/graph/validator/GoValidator.cpp +++ b/src/graph/validator/GoValidator.cpp @@ -142,9 +142,9 @@ Status GoValidator::validateYield(YieldClause* yield) { return Status::SemanticError("`%s' is not support in go sentence.", col->toString().c_str()); } - auto* vertexExpr = ExpressionUtils::findAny(col->expr(), {Expression::Kind::kVertex}); - if (vertexExpr != nullptr) { - const auto& colName = static_cast(vertexExpr)->name(); + const auto& vertexExprs = ExpressionUtils::collectAll(col->expr(), {Expression::Kind::kVertex}); + for (const auto* expr : vertexExprs) { + const auto& colName = static_cast(expr)->name(); if (colName == SRC_VERTEX) { NG_RETURN_IF_ERROR(extractVertexProp(exprProps, true)); } else if (colName == DST_VERTEX) { @@ -266,36 +266,37 @@ Status GoValidator::buildColumns() { goCtx_->filter = rewrite2VarProp(newFilter); } - std::unordered_map existExpr; + std::unordered_set existExpr; auto* newYieldExpr = pool->add(new YieldColumns()); for (auto* col : goCtx_->yieldExpr->columns()) { - auto* vertexExpr = ExpressionUtils::findAny(col->expr(), {Expression::Kind::kVertex}); - if (vertexExpr != nullptr) { - const auto& colName = static_cast(vertexExpr)->name(); - if (colName == SRC_VERTEX && existExpr.count(SRC_VERTEX) == 0) { - goCtx_->srcEdgePropsExpr->addColumn( - new YieldColumn(VertexExpression::make(pool), SRC_VERTEX)); - existExpr[SRC_VERTEX] = true; - } - if (colName == DST_VERTEX && existExpr.count(DST_VERTEX) == 0) { - goCtx_->dstPropsExpr->addColumn(new YieldColumn(VertexExpression::make(pool), DST_VERTEX)); - existExpr[DST_VERTEX] = true; + const auto& vertexExprs = ExpressionUtils::collectAll(col->expr(), {Expression::Kind::kVertex}); + auto existEdge = ExpressionUtils::hasAny(col->expr(), {Expression::Kind::kEdge}); + if (!existEdge && vertexExprs.empty()) { + extractPropExprs(col->expr()); + newYieldExpr->addColumn(new YieldColumn(rewrite2VarProp(col->expr()), col->alias())); + } else { + if (existEdge) { + if (existExpr.emplace(COLNAME_EDGE).second) { + goCtx_->srcEdgePropsExpr->addColumn( + new YieldColumn(EdgeExpression::make(pool), COLNAME_EDGE)); + } } - newYieldExpr->addColumn(col->clone().release()); - } else if (ExpressionUtils::hasAny(col->expr(), {Expression::Kind::kEdge})) { - if (existExpr.count(COLNAME_EDGE) == 0) { - goCtx_->srcEdgePropsExpr->addColumn( - new YieldColumn(EdgeExpression::make(pool), COLNAME_EDGE)); - existExpr[COLNAME_EDGE] = true; + for (const auto* expr : vertexExprs) { + const auto& colName = static_cast(expr)->name(); + if (existExpr.emplace(colName).second) { + if (colName == SRC_VERTEX) { + goCtx_->srcEdgePropsExpr->addColumn( + new YieldColumn(VertexExpression::make(pool), SRC_VERTEX)); + } else { + goCtx_->dstPropsExpr->addColumn( + new YieldColumn(VertexExpression::make(pool), DST_VERTEX)); + } + } } newYieldExpr->addColumn(col->clone().release()); - } else { - extractPropExprs(col->expr()); - newYieldExpr->addColumn(new YieldColumn(rewrite2VarProp(col->expr()), col->alias())); } } goCtx_->yieldExpr = newYieldExpr; - return Status::OK(); } diff --git a/src/graph/validator/GroupByValidator.cpp b/src/graph/validator/GroupByValidator.cpp index 379d7a5ff82..42cf15151de 100644 --- a/src/graph/validator/GroupByValidator.cpp +++ b/src/graph/validator/GroupByValidator.cpp @@ -174,7 +174,7 @@ Status GroupByValidator::groupClauseSemanticCheck() { return false; }; for (auto* expr : yieldCols_) { - if (evaluableExpr(expr)) { + if (ExpressionUtils::isEvaluableExpr(expr)) { continue; } FindVisitor visitor(finder); diff --git a/src/graph/validator/LookupValidator.cpp b/src/graph/validator/LookupValidator.cpp index 8d4f421d6ba..5c8a4567db0 100644 --- a/src/graph/validator/LookupValidator.cpp +++ b/src/graph/validator/LookupValidator.cpp @@ -260,7 +260,10 @@ StatusOr LookupValidator::handleLogicalExprOperands(LogicalExpressi StatusOr LookupValidator::checkFilter(Expression* expr) { // TODO: Support IN expression push down - if (expr->isRelExpr()) { + if (ExpressionUtils::isGeoIndexAcceleratedPredicate(expr)) { + NG_RETURN_IF_ERROR(checkGeoPredicate(expr)); + return rewriteGeoPredicate(expr); + } else if (expr->isRelExpr()) { // Only starts with can be pushed down as a range scan, so forbid other string-related relExpr if (expr->kind() == ExprKind::kRelREG || expr->kind() == ExprKind::kContains || expr->kind() == ExprKind::kNotContains || expr->kind() == ExprKind::kEndsWith || @@ -299,9 +302,47 @@ Status LookupValidator::checkRelExpr(RelationalExpression* expr) { if (left->kind() == ExprKind::kLabelAttribute || right->kind() == ExprKind::kLabelAttribute) { return Status::OK(); } + return Status::SemanticError("Expression %s not supported yet", expr->toString().c_str()); } +Status LookupValidator::checkGeoPredicate(const Expression* expr) const { + auto checkFunc = [](const FunctionCallExpression* funcExpr) -> Status { + if (funcExpr->args()->numArgs() < 2) { + return Status::SemanticError("Expression %s has not enough arguments", + funcExpr->toString().c_str()); + } + auto* first = funcExpr->args()->args()[0]; + auto* second = funcExpr->args()->args()[1]; + + if (first->kind() == ExprKind::kLabelAttribute && second->kind() == ExprKind::kLabelAttribute) { + return Status::SemanticError("Expression %s not supported yet", funcExpr->toString().c_str()); + } + if (first->kind() == ExprKind::kLabelAttribute || second->kind() == ExprKind::kLabelAttribute) { + return Status::OK(); + } + + return Status::SemanticError("Expression %s not supported yet", funcExpr->toString().c_str()); + }; + if (expr->isRelExpr()) { + auto* relExpr = static_cast(expr); + auto* left = relExpr->left(); + auto* right = relExpr->right(); + if (left->kind() == Expression::Kind::kFunctionCall) { + NG_RETURN_IF_ERROR(checkFunc(static_cast(left))); + } + if (right->kind() == Expression::Kind::kFunctionCall) { + NG_RETURN_IF_ERROR(checkFunc(static_cast(right))); + } + } else if (expr->kind() == Expression::Kind::kFunctionCall) { + NG_RETURN_IF_ERROR(checkFunc(static_cast(expr))); + } else { + return Status::SemanticError("Expression %s not supported yet", expr->toString().c_str()); + } + + return Status::OK(); +} + StatusOr LookupValidator::rewriteRelExpr(RelationalExpression* expr) { // swap LHS and RHS of relExpr if LabelAttributeExpr in on the right, // so that LabelAttributeExpr is always on the left @@ -336,11 +377,77 @@ StatusOr LookupValidator::rewriteRelExpr(RelationalExpression* expr return expr; } +StatusOr LookupValidator::rewriteGeoPredicate(Expression* expr) { + // swap LHS and RHS of relExpr if LabelAttributeExpr in on the right, + // so that LabelAttributeExpr is always on the left + if (expr->isRelExpr()) { + if (static_cast(expr)->right()->kind() == + Expression::Kind::kFunctionCall) { + expr = reverseGeoPredicate(expr); + } + auto* relExpr = static_cast(expr); + auto* left = relExpr->left(); + auto* right = relExpr->right(); + DCHECK(left->kind() == Expression::Kind::kFunctionCall); + auto* funcExpr = static_cast(left); + DCHECK_EQ(boost::to_lower_copy(funcExpr->name()), "st_distance"); + // `ST_Distance(g1, g1) < distanceInMeters` is equivalent to `ST_DWithin(g1, g2, + // distanceInMeters, true), `ST_Distance(g1, g1) <= distanceInMeters` is equivalent to + // `ST_DWithin(g1, g2, distanceInMeters, false) + if (relExpr->kind() == Expression::Kind::kRelLT || + relExpr->kind() == Expression::Kind::kRelLE) { + auto* newArgList = ArgumentList::make(qctx_->objPool(), 3); + if (funcExpr->args()->numArgs() != 2) { + return Status::SemanticError("Expression %s has wrong number arguments", + funcExpr->toString().c_str()); + } + newArgList->addArgument(funcExpr->args()->args()[0]->clone()); + newArgList->addArgument(funcExpr->args()->args()[1]->clone()); + newArgList->addArgument(right->clone()); // distanceInMeters + bool exclusive = relExpr->kind() == Expression::Kind::kRelLT; + newArgList->addArgument(ConstantExpression::make(qctx_->objPool(), exclusive)); + expr = FunctionCallExpression::make(qctx_->objPool(), "st_dwithin", newArgList); + } else { + return Status::SemanticError("Expression %s not supported yet", expr->toString().c_str()); + } + } + + DCHECK(expr->kind() == Expression::Kind::kFunctionCall); + if (static_cast(expr)->args()->args()[1]->kind() == + ExprKind::kLabelAttribute) { + expr = reverseGeoPredicate(expr); + } + + auto* geoFuncExpr = static_cast(expr); + auto* first = geoFuncExpr->args()->args()[0]; + auto* la = static_cast(first); + if (la->left()->name() != sentence()->from()) { + return Status::SemanticError("Schema name error: %s", la->left()->name().c_str()); + } + + // fold constant expression + auto foldRes = ExpressionUtils::foldConstantExpr(expr); + NG_RETURN_IF_ERROR(foldRes); + geoFuncExpr = static_cast(foldRes.value()); + + // Check schema and value type + std::string prop = la->right()->value().getStr(); + auto c = checkConstExpr(geoFuncExpr->args()->args()[1], prop, Expression::Kind::kFunctionCall); + NG_RETURN_IF_ERROR(c); + geoFuncExpr->args()->setArg(1, std::move(c).value()); + + // rewrite PropertyExpression + auto propExpr = lookupCtx_->isEdge ? ExpressionUtils::rewriteLabelAttr2EdgeProp(la) + : ExpressionUtils::rewriteLabelAttr2TagProp(la); + geoFuncExpr->args()->setArg(0, propExpr); + return geoFuncExpr; +} + StatusOr LookupValidator::checkConstExpr(Expression* expr, const std::string& prop, const ExprKind kind) { auto* pool = expr->getObjPool(); - if (!evaluableExpr(expr)) { + if (!ExpressionUtils::isEvaluableExpr(expr)) { return Status::SemanticError("'%s' is not an evaluable expression.", expr->toString().c_str()); } auto schemaMgr = qctx_->schemaMng(); @@ -437,6 +544,35 @@ Expression* LookupValidator::reverseRelKind(RelationalExpression* expr) { return RelationalExpression::makeKind(pool, reversedKind, right->clone(), left->clone()); } +Expression* LookupValidator::reverseGeoPredicate(Expression* expr) { + if (expr->isRelExpr()) { + auto* relExpr = static_cast(expr); + return reverseRelKind(relExpr); + } else { + DCHECK(expr->kind() == Expression::Kind::kFunctionCall); + auto* funcExpr = static_cast(expr); + auto name = funcExpr->name(); + folly::toLowerAscii(name); + auto newName = name; + + if (name == "st_covers") { + newName = "st_coveredby"; + } else if (name == "st_coveredby") { + newName = "st_covers"; + } else if (name == "st_intersects") { + } else if (name == "st_dwithin") { + } + + auto* newArgList = ArgumentList::make(qctx_->objPool(), funcExpr->args()->numArgs()); + for (auto& arg : funcExpr->args()->args()) { + newArgList->addArgument(arg->clone()); + } + newArgList->setArg(0, funcExpr->args()->args()[1]); + newArgList->setArg(1, funcExpr->args()->args()[0]); + return FunctionCallExpression::make(qctx_->objPool(), newName, newArgList); + } +} + Status LookupValidator::getSchemaProvider(shared_ptr* provider) const { auto from = sentence()->from(); auto schemaMgr = qctx_->schemaMng(); diff --git a/src/graph/validator/LookupValidator.h b/src/graph/validator/LookupValidator.h index a0a1fd859d0..ce6f6a7291a 100644 --- a/src/graph/validator/LookupValidator.h +++ b/src/graph/validator/LookupValidator.h @@ -39,12 +39,15 @@ class LookupValidator final : public Validator { StatusOr checkFilter(Expression* expr); Status checkRelExpr(RelationalExpression* expr); + Status checkGeoPredicate(const Expression* expr) const; StatusOr checkTSExpr(Expression* expr); StatusOr checkConstExpr(Expression* expr, const std::string& prop, const Expression::Kind kind); StatusOr rewriteRelExpr(RelationalExpression* expr); + StatusOr rewriteGeoPredicate(Expression* expr); Expression* reverseRelKind(RelationalExpression* expr); + Expression* reverseGeoPredicate(Expression* expr); const LookupSentence* sentence() const; int32_t schemaId() const; diff --git a/src/graph/validator/MaintainValidator.cpp b/src/graph/validator/MaintainValidator.cpp index ca04be664ee..a778c841e28 100644 --- a/src/graph/validator/MaintainValidator.cpp +++ b/src/graph/validator/MaintainValidator.cpp @@ -22,33 +22,28 @@ namespace nebula { namespace graph { -Status SchemaValidator::validateColumns(const std::vector &columnSpecs, - meta::cpp2::Schema &schema) { - auto status = Status::OK(); - std::unordered_set nameSet; +static Status validateColumns(const std::vector &columnSpecs, + meta::cpp2::Schema &schema) { for (auto &spec : columnSpecs) { - if (nameSet.find(*spec->name()) != nameSet.end()) { - return Status::SemanticError("Duplicate column name `%s'", spec->name()->c_str()); - } - nameSet.emplace(*spec->name()); meta::cpp2::ColumnDef column; auto type = spec->type(); column.set_name(*spec->name()); column.type.set_type(type); if (meta::cpp2::PropertyType::FIXED_STRING == type) { column.type.set_type_length(spec->typeLen()); + } else if (meta::cpp2::PropertyType::GEOGRAPHY == type) { + column.type.set_geo_shape(spec->geoShape()); } for (const auto &property : spec->properties()->properties()) { if (property->isNullable()) { column.set_nullable(property->nullable()); } else if (property->isDefaultValue()) { - if (!evaluableExpr(property->defaultValue())) { + if (!ExpressionUtils::isEvaluableExpr(property->defaultValue())) { return Status::SemanticError("Wrong default value experssion `%s'", property->defaultValue()->toString().c_str()); } auto *defaultValueExpr = property->defaultValue(); - // some expression is evaluable but not pure so only fold instead of - // eval here + // some expression is evaluable but not pure so only fold instead of eval here auto foldRes = ExpressionUtils::foldConstantExpr(defaultValueExpr); NG_RETURN_IF_ERROR(foldRes); column.set_default_value(foldRes.value()->encode()); @@ -61,65 +56,133 @@ Status SchemaValidator::validateColumns(const std::vector } schema.columns_ref().value().emplace_back(std::move(column)); } + return Status::OK(); +} + +static StatusOr> validateSchemaOpts( + const std::vector &schemaOpts) { + std::vector schemaItems; + std::unordered_set uniqueColName; + for (const auto &schemaOpt : schemaOpts) { + meta::cpp2::AlterSchemaItem schemaItem; + auto opType = schemaOpt->toType(); + schemaItem.set_op(opType); + meta::cpp2::Schema schema; + + if (opType == meta::cpp2::AlterSchemaOp::DROP) { + const auto &colNames = schemaOpt->columnNames(); + for (auto &colName : colNames) { + if (!uniqueColName.emplace(*colName).second) { + return Status::SemanticError("Duplicate column name `%s'", colName->c_str()); + } + meta::cpp2::ColumnDef column; + column.name = *colName; + schema.columns_ref().value().emplace_back(std::move(column)); + } + } else { + const auto &specs = schemaOpt->columnSpecs(); + for (auto &spec : specs) { + if (!uniqueColName.emplace(*spec->name()).second) { + return Status::SemanticError("Duplicate column name `%s'", spec->name()->c_str()); + } + } + NG_LOG_AND_RETURN_IF_ERROR(validateColumns(specs, schema)); + } + schemaItem.set_schema(std::move(schema)); + schemaItems.emplace_back(std::move(schemaItem)); + } + return schemaItems; +} + +static StatusOr validateSchemaProps( + const std::vector &schemaProps) { + meta::cpp2::SchemaProp schemaProp; + for (const auto &prop : schemaProps) { + auto propType = prop->getPropType(); + switch (propType) { + case SchemaPropItem::TTL_DURATION: { + auto ttlDur = prop->getTtlDuration(); + NG_RETURN_IF_ERROR(ttlDur); + schemaProp.set_ttl_duration(ttlDur.value()); + break; + } + case SchemaPropItem::TTL_COL: { + // Check the legality of the column in meta + auto ttlCol = prop->getTtlCol(); + NG_RETURN_IF_ERROR(ttlCol); + schemaProp.set_ttl_col(ttlCol.value()); + break; + } + case SchemaPropItem::COMMENT: { + // Check the legality of the column in meta + auto comment = prop->getComment(); + NG_RETURN_IF_ERROR(comment); + schemaProp.set_comment(comment.value()); + break; + } + default: { + return Status::SemanticError("Property type not support"); + } + } + } + return schemaProp; +} + +static Status checkColName(const std::vector specs) { + std::unordered_set uniqueColName; + for (const auto &spec : specs) { + auto name = *spec->name(); + if (!uniqueColName.emplace(name).second) { + return Status::SemanticError("Duplicate column name `%s'", name.c_str()); + } + } return Status::OK(); } Status CreateTagValidator::validateImpl() { + createCtx_ = getContext(); auto sentence = static_cast(sentence_); - name_ = *sentence->name(); - ifNotExist_ = sentence->isIfNotExist(); - + createCtx_->ifNotExist = sentence->isIfNotExist(); + auto name = *sentence->name(); // Check the validateContext has the same name schema - auto pro = vctx_->getSchema(name_); + auto pro = vctx_->getSchema(name); if (pro != nullptr) { - return Status::SemanticError("Has the same name `%s' in the SequentialSentences", - name_.c_str()); + return Status::SemanticError("Has the same name `%s' in the SequentialSentences", name.c_str()); } - NG_RETURN_IF_ERROR(validateColumns(sentence->columnSpecs(), schema_)); - NG_RETURN_IF_ERROR(SchemaUtil::validateProps(sentence->getSchemaProps(), schema_)); + meta::cpp2::Schema schema; + NG_RETURN_IF_ERROR(checkColName(sentence->columnSpecs())); + NG_RETURN_IF_ERROR(validateColumns(sentence->columnSpecs(), schema)); + NG_RETURN_IF_ERROR(SchemaUtil::validateProps(sentence->getSchemaProps(), schema)); // Save the schema in validateContext auto pool = qctx_->objPool(); - auto schemaPro = SchemaUtil::generateSchemaProvider(pool, 0, schema_); - vctx_->addSchema(name_, schemaPro); - return Status::OK(); -} - -Status CreateTagValidator::toPlan() { - auto *plan = qctx_->plan(); - auto doNode = - CreateTag::make(qctx_, plan->root(), std::move(name_), std::move(schema_), ifNotExist_); - root_ = doNode; - tail_ = root_; + auto schemaPro = SchemaUtil::generateSchemaProvider(pool, 0, schema); + vctx_->addSchema(name, schemaPro); + createCtx_->name = std::move(name); + createCtx_->schema = std::move(schema); return Status::OK(); } Status CreateEdgeValidator::validateImpl() { + createCtx_ = getContext(); auto sentence = static_cast(sentence_); - auto status = Status::OK(); - name_ = *sentence->name(); - ifNotExist_ = sentence->isIfNotExist(); + createCtx_->ifNotExist = sentence->isIfNotExist(); + auto name = *sentence->name(); // Check the validateContext has the same name schema - auto pro = vctx_->getSchema(name_); + auto pro = vctx_->getSchema(name); if (pro != nullptr) { - return Status::SemanticError("Has the same name `%s' in the SequentialSentences", - name_.c_str()); + return Status::SemanticError("Has the same name `%s' in the SequentialSentences", name.c_str()); } - NG_RETURN_IF_ERROR(validateColumns(sentence->columnSpecs(), schema_)); - NG_RETURN_IF_ERROR(SchemaUtil::validateProps(sentence->getSchemaProps(), schema_)); + meta::cpp2::Schema schema; + NG_RETURN_IF_ERROR(checkColName(sentence->columnSpecs())); + NG_RETURN_IF_ERROR(validateColumns(sentence->columnSpecs(), schema)); + NG_RETURN_IF_ERROR(SchemaUtil::validateProps(sentence->getSchemaProps(), schema)); // Save the schema in validateContext auto pool = qctx_->objPool(); - auto schemaPro = SchemaUtil::generateSchemaProvider(pool, 0, schema_); - vctx_->addSchema(name_, schemaPro); - return Status::OK(); -} - -Status CreateEdgeValidator::toPlan() { - auto *plan = qctx_->plan(); - auto doNode = - CreateEdge::make(qctx_, plan->root(), std::move(name_), std::move(schema_), ifNotExist_); - root_ = doNode; - tail_ = root_; + auto schemaPro = SchemaUtil::generateSchemaProvider(pool, 0, schema); + vctx_->addSchema(name, schemaPro); + createCtx_->name = std::move(name); + createCtx_->schema = std::move(schema); return Status::OK(); } @@ -145,93 +208,27 @@ Status DescEdgeValidator::toPlan() { return Status::OK(); } -Status AlterValidator::alterSchema(const std::vector &schemaOpts, - const std::vector &schemaProps) { - for (auto &schemaOpt : schemaOpts) { - meta::cpp2::AlterSchemaItem schemaItem; - auto opType = schemaOpt->toType(); - schemaItem.set_op(opType); - meta::cpp2::Schema schema; - if (opType == meta::cpp2::AlterSchemaOp::DROP) { - const auto &colNames = schemaOpt->columnNames(); - for (auto &colName : colNames) { - meta::cpp2::ColumnDef column; - column.name = *colName; - schema.columns_ref().value().emplace_back(std::move(column)); - } - } else { - const auto &specs = schemaOpt->columnSpecs(); - NG_LOG_AND_RETURN_IF_ERROR(validateColumns(specs, schema)); - } - - schemaItem.set_schema(std::move(schema)); - schemaItems_.emplace_back(std::move(schemaItem)); - } - - for (auto &schemaProp : schemaProps) { - auto propType = schemaProp->getPropType(); - StatusOr retInt; - StatusOr retStr; - int ttlDuration; - switch (propType) { - case SchemaPropItem::TTL_DURATION: - retInt = schemaProp->getTtlDuration(); - NG_RETURN_IF_ERROR(retInt); - ttlDuration = retInt.value(); - schemaProp_.set_ttl_duration(ttlDuration); - break; - case SchemaPropItem::TTL_COL: - // Check the legality of the column in meta - retStr = schemaProp->getTtlCol(); - NG_RETURN_IF_ERROR(retStr); - schemaProp_.set_ttl_col(retStr.value()); - break; - case SchemaPropItem::COMMENT: - // Check the legality of the column in meta - retStr = schemaProp->getComment(); - NG_RETURN_IF_ERROR(retStr); - schemaProp_.set_comment(retStr.value()); - break; - default: - return Status::SemanticError("Property type not support"); - } - } - return Status::OK(); -} - Status AlterTagValidator::validateImpl() { + alterCtx_ = getContext(); auto sentence = static_cast(sentence_); - name_ = *sentence->name(); - return alterSchema(sentence->getSchemaOpts(), sentence->getSchemaProps()); -} - -Status AlterTagValidator::toPlan() { - auto *doNode = AlterTag::make(qctx_, - nullptr, - vctx_->whichSpace().id, - std::move(name_), - std::move(schemaItems_), - std::move(schemaProp_)); - root_ = doNode; - tail_ = root_; + auto schemaItems = validateSchemaOpts(sentence->getSchemaOpts()); + NG_RETURN_IF_ERROR(schemaItems); + alterCtx_->schemaItems = std::move(schemaItems.value()); + auto schemaProps = validateSchemaProps(sentence->getSchemaProps()); + NG_RETURN_IF_ERROR(schemaProps); + alterCtx_->schemaProps = std::move(schemaProps.value()); return Status::OK(); } Status AlterEdgeValidator::validateImpl() { + alterCtx_ = getContext(); auto sentence = static_cast(sentence_); - name_ = *sentence->name(); - return alterSchema(sentence->getSchemaOpts(), sentence->getSchemaProps()); -} - -Status AlterEdgeValidator::toPlan() { - auto *doNode = AlterEdge::make(qctx_, - nullptr, - vctx_->whichSpace().id, - std::move(name_), - std::move(schemaItems_), - std::move(schemaProp_)); - root_ = doNode; - tail_ = root_; + auto schemaItems = validateSchemaOpts(sentence->getSchemaOpts()); + NG_RETURN_IF_ERROR(schemaItems); + alterCtx_->schemaItems = std::move(schemaItems.value()); + auto schemaProps = validateSchemaProps(sentence->getSchemaProps()); + NG_RETURN_IF_ERROR(schemaProps); + alterCtx_->schemaProps = std::move(schemaProps.value()); return Status::OK(); } diff --git a/src/graph/validator/MaintainValidator.h b/src/graph/validator/MaintainValidator.h index 3c0dc912f8f..7b02a0a9fb9 100644 --- a/src/graph/validator/MaintainValidator.h +++ b/src/graph/validator/MaintainValidator.h @@ -8,50 +8,35 @@ #define GRAPH_VALIDATOR_MAINTAINVALIDATOR_H_ #include "clients/meta/MetaClient.h" +#include "graph/context/ast/QueryAstContext.h" #include "graph/validator/Validator.h" #include "parser/AdminSentences.h" #include "parser/MaintainSentences.h" namespace nebula { namespace graph { -class SchemaValidator : public Validator { +class CreateTagValidator final : public Validator { public: - SchemaValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) {} + CreateTagValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) {} - protected: - Status validateColumns(const std::vector& columnSpecs, - meta::cpp2::Schema& schema); - - protected: - std::string name_; -}; - -class CreateTagValidator final : public SchemaValidator { - public: - CreateTagValidator(Sentence* sentence, QueryContext* context) - : SchemaValidator(sentence, context) {} + AstContext* getAstContext() override { return createCtx_.get(); } private: Status validateImpl() override; - Status toPlan() override; - private: - meta::cpp2::Schema schema_; - bool ifNotExist_; + std::unique_ptr createCtx_; }; -class CreateEdgeValidator final : public SchemaValidator { +class CreateEdgeValidator final : public Validator { public: - CreateEdgeValidator(Sentence* sentence, QueryContext* context) - : SchemaValidator(sentence, context) {} + CreateEdgeValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) {} + + AstContext* getAstContext() override { return createCtx_.get(); } private: Status validateImpl() override; - Status toPlan() override; - private: - meta::cpp2::Schema schema_; - bool ifNotExist_; + std::unique_ptr createCtx_; }; class DescTagValidator final : public Validator { @@ -96,39 +81,28 @@ class ShowCreateEdgeValidator final : public Validator { Status toPlan() override; }; -class AlterValidator : public SchemaValidator { +class AlterTagValidator final : public Validator { public: - AlterValidator(Sentence* sentence, QueryContext* context) : SchemaValidator(sentence, context) {} + AlterTagValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) {} - protected: - Status alterSchema(const std::vector& schemaOpts, - const std::vector& schemaProps); - - protected: - std::vector schemaItems_; - meta::cpp2::SchemaProp schemaProp_; -}; - -class AlterTagValidator final : public AlterValidator { - public: - AlterTagValidator(Sentence* sentence, QueryContext* context) - : AlterValidator(sentence, context) {} + AstContext* getAstContext() override { return alterCtx_.get(); } private: Status validateImpl() override; - Status toPlan() override; + std::unique_ptr alterCtx_; }; -class AlterEdgeValidator final : public AlterValidator { +class AlterEdgeValidator final : public Validator { public: - AlterEdgeValidator(Sentence* sentence, QueryContext* context) - : AlterValidator(sentence, context) {} + AlterEdgeValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) {} + + AstContext* getAstContext() override { return alterCtx_.get(); } private: Status validateImpl() override; - Status toPlan() override; + std::unique_ptr alterCtx_; }; class ShowTagsValidator final : public Validator { diff --git a/src/graph/validator/MatchValidator.cpp b/src/graph/validator/MatchValidator.cpp index aae111617b9..f090eacc72f 100644 --- a/src/graph/validator/MatchValidator.cpp +++ b/src/graph/validator/MatchValidator.cpp @@ -618,7 +618,7 @@ Status MatchValidator::validatePagination(const Expression *skipExpr, int64_t skip = 0; int64_t limit = std::numeric_limits::max(); if (skipExpr != nullptr) { - if (!evaluableExpr(skipExpr)) { + if (!ExpressionUtils::isEvaluableExpr(skipExpr)) { return Status::SemanticError("SKIP should be instantly evaluable"); } QueryExpressionContext ctx; @@ -633,7 +633,7 @@ Status MatchValidator::validatePagination(const Expression *skipExpr, } if (limitExpr != nullptr) { - if (!evaluableExpr(limitExpr)) { + if (!ExpressionUtils::isEvaluableExpr(limitExpr)) { return Status::SemanticError("SKIP should be instantly evaluable"); } QueryExpressionContext ctx; diff --git a/src/graph/validator/MutateValidator.cpp b/src/graph/validator/MutateValidator.cpp index 05643548fc3..170569e7667 100644 --- a/src/graph/validator/MutateValidator.cpp +++ b/src/graph/validator/MutateValidator.cpp @@ -89,7 +89,7 @@ Status InsertVerticesValidator::prepareVertices() { if (propSize_ != row->values().size()) { return Status::SemanticError("Column count doesn't match value count."); } - if (!evaluableExpr(row->id())) { + if (!ExpressionUtils::isEvaluableExpr(row->id())) { LOG(ERROR) << "Wrong vid expression `" << row->id()->toString() << "\""; return Status::SemanticError("Wrong vid expression `%s'", row->id()->toString().c_str()); } @@ -99,7 +99,7 @@ Status InsertVerticesValidator::prepareVertices() { // check value expr for (auto &value : row->values()) { - if (!evaluableExpr(value)) { + if (!ExpressionUtils::isEvaluableExpr(value)) { LOG(ERROR) << "Insert wrong value: `" << value->toString() << "'."; return Status::SemanticError("Insert wrong value: `%s'.", value->toString().c_str()); } @@ -203,13 +203,13 @@ Status InsertEdgesValidator::prepareEdges() { if (propNames_.size() != row->values().size()) { return Status::SemanticError("Column count doesn't match value count."); } - if (!evaluableExpr(row->srcid())) { + if (!ExpressionUtils::isEvaluableExpr(row->srcid())) { LOG(ERROR) << "Wrong src vid expression `" << row->srcid()->toString() << "\""; return Status::SemanticError("Wrong src vid expression `%s'", row->srcid()->toString().c_str()); } - if (!evaluableExpr(row->dstid())) { + if (!ExpressionUtils::isEvaluableExpr(row->dstid())) { LOG(ERROR) << "Wrong dst vid expression `" << row->dstid()->toString() << "\""; return Status::SemanticError("Wrong dst vid expression `%s'", row->dstid()->toString().c_str()); @@ -226,7 +226,7 @@ Status InsertEdgesValidator::prepareEdges() { // check value expr for (auto &value : row->values()) { - if (!evaluableExpr(value)) { + if (!ExpressionUtils::isEvaluableExpr(value)) { LOG(ERROR) << "Insert wrong value: `" << value->toString() << "'."; return Status::SemanticError("Insert wrong value: `%s'.", value->toString().c_str()); } diff --git a/src/graph/validator/Validator.cpp b/src/graph/validator/Validator.cpp index 761c0d691f7..48963d4ba4e 100644 --- a/src/graph/validator/Validator.cpp +++ b/src/graph/validator/Validator.cpp @@ -373,12 +373,6 @@ Status Validator::deduceProps(const Expression* expr, ExpressionProps& exprProps return std::move(visitor).status(); } -bool Validator::evaluableExpr(const Expression* expr) const { - EvaluableExprVisitor visitor; - const_cast(expr)->accept(&visitor); - return visitor.ok(); -} - Status Validator::toPlan() { auto* astCtx = getAstContext(); if (astCtx != nullptr) { @@ -458,7 +452,7 @@ Status Validator::validateStarts(const VerticesClause* clause, Starts& starts) { auto vidList = clause->vidList(); QueryExpressionContext ctx; for (auto* expr : vidList) { - if (!evaluableExpr(expr)) { + if (!ExpressionUtils::isEvaluableExpr(expr)) { return Status::SemanticError("`%s' is not an evaluable expression.", expr->toString().c_str()); } diff --git a/src/graph/validator/Validator.h b/src/graph/validator/Validator.h index 4d1320ff14b..381b1db65fe 100644 --- a/src/graph/validator/Validator.h +++ b/src/graph/validator/Validator.h @@ -110,8 +110,6 @@ class Validator { Status deduceProps(const Expression* expr, ExpressionProps& exprProps); - bool evaluableExpr(const Expression* expr) const; - static StatusOr checkPropNonexistOrDuplicate(const ColsDef& cols, folly::StringPiece prop, const std::string& validator); diff --git a/src/kvstore/test/CMakeLists.txt b/src/kvstore/test/CMakeLists.txt index 6f3db67d2cd..aa039b23854 100644 --- a/src/kvstore/test/CMakeLists.txt +++ b/src/kvstore/test/CMakeLists.txt @@ -33,6 +33,7 @@ set(KVSTORE_TEST_LIBS $ $ $ + $ ) nebula_add_test( diff --git a/src/kvstore/test/PartTest.cpp b/src/kvstore/test/PartTest.cpp index e27e1a81767..ab28851f0b4 100644 --- a/src/kvstore/test/PartTest.cpp +++ b/src/kvstore/test/PartTest.cpp @@ -159,9 +159,11 @@ TEST(PartTest, PartCleanTest) { } IndexID indexId = 5; for (int i = 0; i < 10; i++) { - auto key = - IndexKeyUtils::vertexIndexKey(kDefaultVIdLen, partId, indexId, std::to_string(i), "123"); - data.emplace_back(key, folly::stringPrintf("val%d", i)); + auto keys = IndexKeyUtils::vertexIndexKeys( + kDefaultVIdLen, partId, indexId, std::to_string(i), {"123"}); + for (auto& key : keys) { + data.emplace_back(key, folly::stringPrintf("val%d", i)); + } } data.emplace_back(NebulaKeyUtils::systemCommitKey(partId), "123"); diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index ea9f436d611..90c4835dd0b 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -162,6 +162,7 @@ set(meta_test_deps $ $ $ + $ ) nebula_add_subdirectory(http) diff --git a/src/meta/processors/index/CreateEdgeIndexProcessor.cpp b/src/meta/processors/index/CreateEdgeIndexProcessor.cpp index 21ff62be820..9fa97cda12d 100644 --- a/src/meta/processors/index/CreateEdgeIndexProcessor.cpp +++ b/src/meta/processors/index/CreateEdgeIndexProcessor.cpp @@ -156,6 +156,12 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) { handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); onFinished(); return; + } else if (col.type.get_type() == meta::cpp2::PropertyType::GEOGRAPHY && fields.size() > 1) { + // TODO(jie): Support joint index for geography + LOG(ERROR) << "Only support to create index on a single geography column currently"; + handleErrorCode(nebula::cpp2::ErrorCode::E_UNSUPPORTED); + onFinished(); + return; } columns.emplace_back(col); } diff --git a/src/meta/processors/index/CreateTagIndexProcessor.cpp b/src/meta/processors/index/CreateTagIndexProcessor.cpp index 0e7cdf5b768..2f6a1c0f624 100644 --- a/src/meta/processors/index/CreateTagIndexProcessor.cpp +++ b/src/meta/processors/index/CreateTagIndexProcessor.cpp @@ -154,6 +154,12 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) { handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); onFinished(); return; + } else if (col.type.get_type() == meta::cpp2::PropertyType::GEOGRAPHY && fields.size() > 1) { + // TODO(jie): Support joint index for geography + LOG(ERROR) << "Only support to create index on a single geography column currently"; + handleErrorCode(nebula::cpp2::ErrorCode::E_UNSUPPORTED); + onFinished(); + return; } columns.emplace_back(col); } diff --git a/src/mock/MockData.cpp b/src/mock/MockData.cpp index 87e84b15b54..c9f8356ce84 100644 --- a/src/mock/MockData.cpp +++ b/src/mock/MockData.cpp @@ -736,9 +736,11 @@ std::vector> MockData::mockPlayerIndexKeys(b values.append(encodeFixedStr(name, 20)); values.append(IndexKeyUtils::encodeValue(player.age_)); values.append(IndexKeyUtils::encodeValue(player.playing_)); - auto key = IndexKeyUtils::vertexIndexKey(32, part, 1, name, std::move(values)); - auto pair = std::make_pair(part, std::move(key)); - keys.emplace_back(std::move(pair)); + auto indexKeys = IndexKeyUtils::vertexIndexKeys(32, part, 1, name, {std::move(values)}); + for (auto& indexKey : indexKeys) { + auto pair = std::make_pair(part, std::move(indexKey)); + keys.emplace_back(std::move(pair)); + } } return keys; } @@ -839,10 +841,12 @@ std::vector> MockData::mockServeIndexKeys() values.append(encodeFixedStr(serve.playerName_, 20)); values.append(encodeFixedStr(serve.teamName_, 20)); values.append(IndexKeyUtils::encodeValue(serve.startYear_)); - auto key = IndexKeyUtils::edgeIndexKey( - 32, part, 101, serve.playerName_, serve.startYear_, serve.teamName_, std::move(values)); - auto pair = std::make_pair(part, std::move(key)); - keys.emplace_back(std::move(pair)); + auto idxKeys = IndexKeyUtils::edgeIndexKeys( + 32, part, 101, serve.playerName_, serve.startYear_, serve.teamName_, {std::move(values)}); + for (auto& idxkey : idxKeys) { + auto pair = std::make_pair(part, std::move(idxkey)); + keys.emplace_back(std::move(pair)); + } } return keys; } diff --git a/src/parser/test/ParserTest.cpp b/src/parser/test/ParserTest.cpp index 8ccc3490942..4d647db9960 100644 --- a/src/parser/test/ParserTest.cpp +++ b/src/parser/test/ParserTest.cpp @@ -91,6 +91,27 @@ TEST_F(ParserTest, TestSchemaCreation) { auto result = parse(query); ASSERT_TRUE(result.ok()) << result.status(); } + // Geo spatial + { + std::string query = "CREATE TAG any_shape(geo geography)"; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = "CREATE TAG any_shape(geo geography(point))"; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = "CREATE TAG any_shape(geo geography(linestring))"; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = "CREATE TAG any_shape(geo geography(polygon))"; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } } TEST_F(ParserTest, Go) { diff --git a/src/storage/admin/RebuildEdgeIndexTask.cpp b/src/storage/admin/RebuildEdgeIndexTask.cpp index f8ceae17e7e..31006c99929 100644 --- a/src/storage/admin/RebuildEdgeIndexTask.cpp +++ b/src/storage/admin/RebuildEdgeIndexTask.cpp @@ -142,15 +142,17 @@ nebula::cpp2::ErrorCode RebuildEdgeIndexTask::buildIndexGlobal(GraphSpaceID spac LOG(WARNING) << "Collect index value failed"; continue; } - auto indexKey = IndexKeyUtils::edgeIndexKey(vidSize, - part, - item->get_index_id(), - source.toString(), - ranking, - destination.toString(), - std::move(valuesRet).value()); - batchSize += indexKey.size() + indexVal.size(); - data.emplace_back(std::move(indexKey), indexVal); + auto indexKeys = IndexKeyUtils::edgeIndexKeys(vidSize, + part, + item->get_index_id(), + source.toString(), + ranking, + destination.toString(), + std::move(valuesRet).value()); + for (auto& indexKey : indexKeys) { + batchSize += indexKey.size() + indexVal.size(); + data.emplace_back(std::move(indexKey), indexVal); + } } } iter->next(); diff --git a/src/storage/admin/RebuildTagIndexTask.cpp b/src/storage/admin/RebuildTagIndexTask.cpp index 3bcdd4879da..006571fb021 100644 --- a/src/storage/admin/RebuildTagIndexTask.cpp +++ b/src/storage/admin/RebuildTagIndexTask.cpp @@ -133,10 +133,12 @@ nebula::cpp2::ErrorCode RebuildTagIndexTask::buildIndexGlobal(GraphSpaceID space LOG(WARNING) << "Collect index value failed"; continue; } - auto indexKey = IndexKeyUtils::vertexIndexKey( + auto indexKeys = IndexKeyUtils::vertexIndexKeys( vidSize, part, item->get_index_id(), vertex.toString(), std::move(valuesRet).value()); - batchSize += indexKey.size() + indexVal.size(); - data.emplace_back(std::move(indexKey), indexVal); + for (auto& indexKey : indexKeys) { + batchSize += indexKey.size() + indexVal.size(); + data.emplace_back(std::move(indexKey), indexVal); + } } } iter->next(); diff --git a/src/storage/exec/UpdateNode.h b/src/storage/exec/UpdateNode.h index 01c4ac60b50..f36e284c082 100644 --- a/src/storage/exec/UpdateNode.h +++ b/src/storage/exec/UpdateNode.h @@ -362,17 +362,21 @@ class UpdateTagNode : public UpdateNode { LOG(ERROR) << "Bad format row"; return folly::none; } - auto oi = indexKey(partId, vId, reader_, index); - if (!oi.empty()) { + auto ois = indexKeys(partId, vId, reader_, index); + if (!ois.empty()) { auto iState = context_->env()->getIndexState(context_->spaceId(), partId); if (context_->env()->checkRebuilding(iState)) { auto deleteOpKey = OperationKeyUtils::deleteOperationKey(partId); - batchHolder->put(std::move(deleteOpKey), std::move(oi)); + for (auto& oi : ois) { + batchHolder->put(std::string(deleteOpKey), std::move(oi)); + } } else if (context_->env()->checkIndexLocked(iState)) { LOG(ERROR) << "The index has been locked: " << index->get_index_name(); return folly::none; } else { - batchHolder->remove(std::move(oi)); + for (auto& oi : ois) { + batchHolder->remove(std::move(oi)); + } } } } @@ -386,19 +390,23 @@ class UpdateTagNode : public UpdateNode { LOG(ERROR) << "Bad format row"; return folly::none; } - auto ni = indexKey(partId, vId, nReader.get(), index); - if (!ni.empty()) { + auto nis = indexKeys(partId, vId, nReader.get(), index); + if (!nis.empty()) { auto v = CommonUtils::ttlValue(schema_, nReader.get()); auto niv = v.ok() ? IndexKeyUtils::indexVal(std::move(v).value()) : ""; auto indexState = context_->env()->getIndexState(context_->spaceId(), partId); if (context_->env()->checkRebuilding(indexState)) { - auto modifyKey = OperationKeyUtils::modifyOperationKey(partId, std::move(ni)); - batchHolder->put(std::move(modifyKey), std::move(niv)); + for (auto& ni : nis) { + auto modifyKey = OperationKeyUtils::modifyOperationKey(partId, std::move(ni)); + batchHolder->put(std::move(modifyKey), std::string(niv)); + } } else if (context_->env()->checkIndexLocked(indexState)) { LOG(ERROR) << "The index has been locked: " << index->get_index_name(); return folly::none; } else { - batchHolder->put(std::move(ni), std::move(niv)); + for (auto& ni : nis) { + batchHolder->put(std::move(ni), std::string(niv)); + } } } } @@ -409,15 +417,15 @@ class UpdateTagNode : public UpdateNode { return encodeBatchValue(batchHolder->getBatch()); } - std::string indexKey(PartitionID partId, - const VertexID& vId, - RowReader* reader, - std::shared_ptr index) { + std::vector indexKeys(PartitionID partId, + const VertexID& vId, + RowReader* reader, + std::shared_ptr index) { auto values = IndexKeyUtils::collectIndexValues(reader, index->get_fields()); if (!values.ok()) { - return ""; + return {}; } - return IndexKeyUtils::vertexIndexKey( + return IndexKeyUtils::vertexIndexKeys( context_->vIdLen(), partId, index->get_index_id(), vId, std::move(values).value()); } @@ -677,17 +685,21 @@ class UpdateEdgeNode : public UpdateNode { LOG(ERROR) << "Bad format row"; return folly::none; } - auto oi = indexKey(partId, reader_, edgeKey, index); - if (!oi.empty()) { + auto ois = indexKeys(partId, reader_, edgeKey, index); + if (!ois.empty()) { auto iState = context_->env()->getIndexState(context_->spaceId(), partId); if (context_->env()->checkRebuilding(iState)) { auto deleteOpKey = OperationKeyUtils::deleteOperationKey(partId); - batchHolder->put(std::move(deleteOpKey), std::move(oi)); + for (auto& oi : ois) { + batchHolder->put(std::string(deleteOpKey), std::move(oi)); + } } else if (context_->env()->checkIndexLocked(iState)) { LOG(ERROR) << "The index has been locked: " << index->get_index_name(); return folly::none; } else { - batchHolder->remove(std::move(oi)); + for (auto& oi : ois) { + batchHolder->remove(std::move(oi)); + } } } } @@ -701,19 +713,23 @@ class UpdateEdgeNode : public UpdateNode { LOG(ERROR) << "Bad format row"; return folly::none; } - auto nik = indexKey(partId, nReader.get(), edgeKey, index); - if (!nik.empty()) { + auto niks = indexKeys(partId, nReader.get(), edgeKey, index); + if (!niks.empty()) { auto v = CommonUtils::ttlValue(schema_, nReader.get()); auto niv = v.ok() ? IndexKeyUtils::indexVal(std::move(v).value()) : ""; auto indexState = context_->env()->getIndexState(context_->spaceId(), partId); if (context_->env()->checkRebuilding(indexState)) { - auto modifyKey = OperationKeyUtils::modifyOperationKey(partId, std::move(nik)); - batchHolder->put(std::move(modifyKey), std::move(niv)); + for (auto& nik : niks) { + auto modifyKey = OperationKeyUtils::modifyOperationKey(partId, std::move(nik)); + batchHolder->put(std::move(modifyKey), std::string(niv)); + } } else if (context_->env()->checkIndexLocked(indexState)) { LOG(ERROR) << "The index has been locked: " << index->get_index_name(); return folly::none; } else { - batchHolder->put(std::move(nik), std::move(niv)); + for (auto& nik : niks) { + batchHolder->put(std::move(nik), std::string(niv)); + } } } } @@ -732,21 +748,21 @@ class UpdateEdgeNode : public UpdateNode { return encodeBatchValue(batchHolder->getBatch()); } - std::string indexKey(PartitionID partId, - RowReader* reader, - const cpp2::EdgeKey& edgeKey, - std::shared_ptr index) { + std::vector indexKeys(PartitionID partId, + RowReader* reader, + const cpp2::EdgeKey& edgeKey, + std::shared_ptr index) { auto values = IndexKeyUtils::collectIndexValues(reader, index->get_fields()); if (!values.ok()) { - return ""; - } - return IndexKeyUtils::edgeIndexKey(context_->vIdLen(), - partId, - index->get_index_id(), - edgeKey.get_src().getStr(), - edgeKey.get_ranking(), - edgeKey.get_dst().getStr(), - std::move(values).value()); + return {}; + } + return IndexKeyUtils::edgeIndexKeys(context_->vIdLen(), + partId, + index->get_index_id(), + edgeKey.get_src().getStr(), + edgeKey.get_ranking(), + edgeKey.get_dst().getStr(), + std::move(values).value()); } private: diff --git a/src/storage/index/LookupBaseProcessor-inl.h b/src/storage/index/LookupBaseProcessor-inl.h index 5fb46d38a83..6a280f078bb 100644 --- a/src/storage/index/LookupBaseProcessor-inl.h +++ b/src/storage/index/LookupBaseProcessor-inl.h @@ -210,7 +210,11 @@ StatusOr> LookupBaseProcessor::buildPlan( auto it = std::find_if(fields.begin(), fields.end(), [&yieldCol](const auto& columnDef) { return yieldCol == columnDef.get_name(); }); - if (it == fields.end()) { + if (it == fields.end() || + it->get_type().get_type() == + nebula::meta::cpp2::PropertyType::GEOGRAPHY) { // geography index just stores + // S2CellId, so must read the + // original geo data. needData = true; break; } diff --git a/src/storage/mutate/AddEdgesProcessor.cpp b/src/storage/mutate/AddEdgesProcessor.cpp index 2d0b29f5916..bc266bd379c 100644 --- a/src/storage/mutate/AddEdgesProcessor.cpp +++ b/src/storage/mutate/AddEdgesProcessor.cpp @@ -245,19 +245,23 @@ void AddEdgesProcessor::doProcessWithIndex(const cpp2::AddEdgesRequest& req) { * step 1 , Delete old version index if exists. */ if (oReader != nullptr) { - auto oi = indexKey(partId, oReader.get(), key, index); - if (!oi.empty()) { + auto ois = indexKeys(partId, oReader.get(), key, index); + if (!ois.empty()) { // Check the index is building for the specified partition or not. auto indexState = env_->getIndexState(spaceId_, partId); if (env_->checkRebuilding(indexState)) { auto delOpKey = OperationKeyUtils::deleteOperationKey(partId); - batchHolder->put(std::move(delOpKey), std::move(oi)); + for (auto& oi : ois) { + batchHolder->put(std::string(delOpKey), std::move(oi)); + } } else if (env_->checkIndexLocked(indexState)) { LOG(ERROR) << "The index has been locked: " << index->get_index_name(); code = nebula::cpp2::ErrorCode::E_DATA_CONFLICT_ERROR; break; } else { - batchHolder->remove(std::move(oi)); + for (auto& oi : ois) { + batchHolder->remove(std::move(oi)); + } } } } @@ -265,21 +269,25 @@ void AddEdgesProcessor::doProcessWithIndex(const cpp2::AddEdgesRequest& req) { * step 2 , Insert new edge index */ if (nReader != nullptr) { - auto nik = indexKey(partId, nReader.get(), key, index); - if (!nik.empty()) { + auto niks = indexKeys(partId, nReader.get(), key, index); + if (!niks.empty()) { auto v = CommonUtils::ttlValue(schema.get(), nReader.get()); auto niv = v.ok() ? IndexKeyUtils::indexVal(std::move(v).value()) : ""; // Check the index is building for the specified partition or not. auto indexState = env_->getIndexState(spaceId_, partId); if (env_->checkRebuilding(indexState)) { - auto opKey = OperationKeyUtils::modifyOperationKey(partId, std::move(nik)); - batchHolder->put(std::move(opKey), std::move(niv)); + for (auto& nik : niks) { + auto opKey = OperationKeyUtils::modifyOperationKey(partId, std::move(nik)); + batchHolder->put(std::move(opKey), std::string(niv)); + } } else if (env_->checkIndexLocked(indexState)) { LOG(ERROR) << "The index has been locked: " << index->get_index_name(); code = nebula::cpp2::ErrorCode::E_DATA_CONFLICT_ERROR; break; } else { - batchHolder->put(std::move(nik), std::move(niv)); + for (auto& nik : niks) { + batchHolder->put(std::move(nik), std::string(niv)); + } } } } @@ -368,18 +376,22 @@ ErrorOr AddEdgesProcessor::addEdges( } if (!val.empty()) { - auto oi = indexKey(partId, oReader.get(), e.first, index); - if (!oi.empty()) { + auto ois = indexKeys(partId, oReader.get(), e.first, index); + if (!ois.empty()) { // Check the index is building for the specified partition or not. auto indexState = env_->getIndexState(spaceId_, partId); if (env_->checkRebuilding(indexState)) { auto deleteOpKey = OperationKeyUtils::deleteOperationKey(partId); - batchHolder->put(std::move(deleteOpKey), std::move(oi)); + for (auto& oi : ois) { + batchHolder->put(std::string(deleteOpKey), std::move(oi)); + } } else if (env_->checkIndexLocked(indexState)) { LOG(ERROR) << "The index has been locked: " << index->get_index_name(); return nebula::cpp2::ErrorCode::E_DATA_CONFLICT_ERROR; } else { - batchHolder->remove(std::move(oi)); + for (auto& oi : ois) { + batchHolder->remove(std::move(oi)); + } } } } @@ -396,20 +408,24 @@ ErrorOr AddEdgesProcessor::addEdges( } } - auto nik = indexKey(partId, nReader.get(), e.first, index); - if (!nik.empty()) { + auto niks = indexKeys(partId, nReader.get(), e.first, index); + if (!niks.empty()) { auto v = CommonUtils::ttlValue(schema.get(), nReader.get()); auto niv = v.ok() ? IndexKeyUtils::indexVal(std::move(v).value()) : ""; // Check the index is building for the specified partition or not. auto indexState = env_->getIndexState(spaceId_, partId); if (env_->checkRebuilding(indexState)) { - auto modifyOpKey = OperationKeyUtils::modifyOperationKey(partId, std::move(nik)); - batchHolder->put(std::move(modifyOpKey), std::move(niv)); + for (auto& nik : niks) { + auto modifyOpKey = OperationKeyUtils::modifyOperationKey(partId, std::move(nik)); + batchHolder->put(std::move(modifyOpKey), std::string(niv)); + } } else if (env_->checkIndexLocked(indexState)) { LOG(ERROR) << "The index has been locked: " << index->get_index_name(); return nebula::cpp2::ErrorCode::E_DATA_CONFLICT_ERROR; } else { - batchHolder->put(std::move(nik), std::move(niv)); + for (auto& nik : niks) { + batchHolder->put(std::move(nik), std::string(niv)); + } } } } @@ -445,21 +461,22 @@ ErrorOr AddEdgesProcessor::findOldValue( } } -std::string AddEdgesProcessor::indexKey(PartitionID partId, - RowReader* reader, - const folly::StringPiece& rawKey, - std::shared_ptr index) { +std::vector AddEdgesProcessor::indexKeys( + PartitionID partId, + RowReader* reader, + const folly::StringPiece& rawKey, + std::shared_ptr index) { auto values = IndexKeyUtils::collectIndexValues(reader, index->get_fields()); if (!values.ok()) { - return ""; + return {}; } - return IndexKeyUtils::edgeIndexKey(spaceVidLen_, - partId, - index->get_index_id(), - NebulaKeyUtils::getSrcId(spaceVidLen_, rawKey).str(), - NebulaKeyUtils::getRank(spaceVidLen_, rawKey), - NebulaKeyUtils::getDstId(spaceVidLen_, rawKey).str(), - std::move(values).value()); + return IndexKeyUtils::edgeIndexKeys(spaceVidLen_, + partId, + index->get_index_id(), + NebulaKeyUtils::getSrcId(spaceVidLen_, rawKey).str(), + NebulaKeyUtils::getRank(spaceVidLen_, rawKey), + NebulaKeyUtils::getDstId(spaceVidLen_, rawKey).str(), + std::move(values).value()); } } // namespace storage diff --git a/src/storage/mutate/AddEdgesProcessor.h b/src/storage/mutate/AddEdgesProcessor.h index 90596995105..ec509cb2f89 100644 --- a/src/storage/mutate/AddEdgesProcessor.h +++ b/src/storage/mutate/AddEdgesProcessor.h @@ -43,10 +43,10 @@ class AddEdgesProcessor : public BaseProcessor { ErrorOr findOldValue(PartitionID partId, const folly::StringPiece& rawKey); - std::string indexKey(PartitionID partId, - RowReader* reader, - const folly::StringPiece& rawKey, - std::shared_ptr index); + std::vector indexKeys(PartitionID partId, + RowReader* reader, + const folly::StringPiece& rawKey, + std::shared_ptr index); private: GraphSpaceID spaceId_; diff --git a/src/storage/mutate/AddVerticesProcessor.cpp b/src/storage/mutate/AddVerticesProcessor.cpp index 1506d024b25..4f3186864b0 100644 --- a/src/storage/mutate/AddVerticesProcessor.cpp +++ b/src/storage/mutate/AddVerticesProcessor.cpp @@ -218,24 +218,29 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re } for (auto& index : indexes_) { if (tagId == index->get_schema_id().get_tag_id()) { + auto indexFields = index->get_fields(); /* * step 1 , Delete old version index if exists. */ if (oReader != nullptr) { - auto oi = indexKey(partId, vid, oReader.get(), index); - if (!oi.empty()) { + auto ois = indexKeys(partId, vid, oReader.get(), index); + if (!ois.empty()) { // Check the index is building for the specified partition or // not. auto indexState = env_->getIndexState(spaceId_, partId); if (env_->checkRebuilding(indexState)) { auto delOpKey = OperationKeyUtils::deleteOperationKey(partId); - batchHolder->put(std::move(delOpKey), std::move(oi)); + for (auto& oi : ois) { + batchHolder->put(std::string(delOpKey), std::move(oi)); + } } else if (env_->checkIndexLocked(indexState)) { LOG(ERROR) << "The index has been locked: " << index->get_index_name(); code = nebula::cpp2::ErrorCode::E_DATA_CONFLICT_ERROR; break; } else { - batchHolder->remove(std::move(oi)); + for (auto& oi : ois) { + batchHolder->remove(std::move(oi)); + } } } } @@ -244,22 +249,26 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re * step 2 , Insert new vertex index */ if (nReader != nullptr) { - auto nik = indexKey(partId, vid, nReader.get(), index); - if (!nik.empty()) { + auto niks = indexKeys(partId, vid, nReader.get(), index); + if (!niks.empty()) { auto v = CommonUtils::ttlValue(schema.get(), nReader.get()); auto niv = v.ok() ? IndexKeyUtils::indexVal(std::move(v).value()) : ""; // Check the index is building for the specified partition or // not. auto indexState = env_->getIndexState(spaceId_, partId); if (env_->checkRebuilding(indexState)) { - auto opKey = OperationKeyUtils::modifyOperationKey(partId, nik); - batchHolder->put(std::move(opKey), std::move(niv)); + for (auto& nik : niks) { + auto opKey = OperationKeyUtils::modifyOperationKey(partId, nik); + batchHolder->put(std::move(opKey), std::string(niv)); + } } else if (env_->checkIndexLocked(indexState)) { LOG(ERROR) << "The index has been locked: " << index->get_index_name(); code = nebula::cpp2::ErrorCode::E_DATA_CONFLICT_ERROR; break; } else { - batchHolder->put(std::move(nik), std::move(niv)); + for (auto& nik : niks) { + batchHolder->put(std::move(nik), std::string(niv)); + } } } } @@ -295,7 +304,7 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re handleAsync(spaceId_, partId, retCode); }); } -} +} // namespace storage ErrorOr AddVerticesProcessor::findOldValue( PartitionID partId, const VertexID& vId, TagID tagId) { @@ -313,16 +322,17 @@ ErrorOr AddVerticesProcessor::findOldValue } } -std::string AddVerticesProcessor::indexKey(PartitionID partId, - const VertexID& vId, - RowReader* reader, - std::shared_ptr index) { +std::vector AddVerticesProcessor::indexKeys( + PartitionID partId, + const VertexID& vId, + RowReader* reader, + std::shared_ptr index) { auto values = IndexKeyUtils::collectIndexValues(reader, index->get_fields()); if (!values.ok()) { - return ""; + return {}; } - return IndexKeyUtils::vertexIndexKey( + return IndexKeyUtils::vertexIndexKeys( spaceVidLen_, partId, index->get_index_id(), vId, std::move(values).value()); } diff --git a/src/storage/mutate/AddVerticesProcessor.h b/src/storage/mutate/AddVerticesProcessor.h index 3c8483beb76..3baf8e5ebcf 100644 --- a/src/storage/mutate/AddVerticesProcessor.h +++ b/src/storage/mutate/AddVerticesProcessor.h @@ -39,10 +39,10 @@ class AddVerticesProcessor : public BaseProcessor { const VertexID& vId, TagID tagId); - std::string indexKey(PartitionID partId, - const VertexID& vId, - RowReader* reader, - std::shared_ptr index); + std::vector indexKeys(PartitionID partId, + const VertexID& vId, + RowReader* reader, + std::shared_ptr index); private: GraphSpaceID spaceId_; diff --git a/src/storage/mutate/DeleteEdgesProcessor.cpp b/src/storage/mutate/DeleteEdgesProcessor.cpp index b8d5dff3c36..4bf1b54b87e 100644 --- a/src/storage/mutate/DeleteEdgesProcessor.cpp +++ b/src/storage/mutate/DeleteEdgesProcessor.cpp @@ -166,18 +166,22 @@ ErrorOr DeleteEdgesProcessor::deleteEdges( if (!valuesRet.ok()) { continue; } - auto indexKey = IndexKeyUtils::edgeIndexKey( + auto indexKeys = IndexKeyUtils::edgeIndexKeys( spaceVidLen_, partId, indexId, srcId, rank, dstId, std::move(valuesRet).value()); auto indexState = env_->getIndexState(spaceId_, partId); if (env_->checkRebuilding(indexState)) { auto deleteOpKey = OperationKeyUtils::deleteOperationKey(partId); - batchHolder->put(std::move(deleteOpKey), std::move(indexKey)); + for (auto& indexKey : indexKeys) { + batchHolder->put(std::string(deleteOpKey), std::move(indexKey)); + } } else if (env_->checkIndexLocked(indexState)) { LOG(ERROR) << "The index has been locked: " << index->get_index_name(); return nebula::cpp2::ErrorCode::E_DATA_CONFLICT_ERROR; } else { - batchHolder->remove(std::move(indexKey)); + for (auto& indexKey : indexKeys) { + batchHolder->remove(std::move(indexKey)); + } } } } diff --git a/src/storage/mutate/DeleteTagsProcessor.cpp b/src/storage/mutate/DeleteTagsProcessor.cpp index b856504e746..311587d7b61 100644 --- a/src/storage/mutate/DeleteTagsProcessor.cpp +++ b/src/storage/mutate/DeleteTagsProcessor.cpp @@ -128,18 +128,22 @@ ErrorOr DeleteTagsProcessor::deleteTags( if (!valuesRet.ok()) { return nebula::cpp2::ErrorCode::E_INVALID_DATA; } - auto indexKey = IndexKeyUtils::vertexIndexKey( + auto indexKeys = IndexKeyUtils::vertexIndexKeys( spaceVidLen_, partId, indexId, vId, std::move(valuesRet).value()); // Check the index is building for the specified partition or not auto indexState = env_->getIndexState(spaceId_, partId); if (env_->checkRebuilding(indexState)) { auto deleteOpKey = OperationKeyUtils::deleteOperationKey(partId); - batchHolder->put(std::move(deleteOpKey), std::move(indexKey)); + for (auto& indexKey : indexKeys) { + batchHolder->put(std::string(deleteOpKey), std::move(indexKey)); + } } else if (env_->checkIndexLocked(indexState)) { return nebula::cpp2::ErrorCode::E_DATA_CONFLICT_ERROR; } else { - batchHolder->remove(std::move(indexKey)); + for (auto& indexKey : indexKeys) { + batchHolder->remove(std::move(indexKey)); + } } } } diff --git a/src/storage/mutate/DeleteVerticesProcessor.cpp b/src/storage/mutate/DeleteVerticesProcessor.cpp index 1a42c70b519..617bafa0db9 100644 --- a/src/storage/mutate/DeleteVerticesProcessor.cpp +++ b/src/storage/mutate/DeleteVerticesProcessor.cpp @@ -150,19 +150,23 @@ ErrorOr DeleteVerticesProcessor::deleteVer if (!valuesRet.ok()) { continue; } - auto indexKey = IndexKeyUtils::vertexIndexKey( + auto indexKeys = IndexKeyUtils::vertexIndexKeys( spaceVidLen_, partId, indexId, vertex.getStr(), std::move(valuesRet).value()); // Check the index is building for the specified partition or not auto indexState = env_->getIndexState(spaceId_, partId); if (env_->checkRebuilding(indexState)) { auto deleteOpKey = OperationKeyUtils::deleteOperationKey(partId); - batchHolder->put(std::move(deleteOpKey), std::move(indexKey)); + for (auto& indexKey : indexKeys) { + batchHolder->put(std::string(deleteOpKey), std::move(indexKey)); + } } else if (env_->checkIndexLocked(indexState)) { LOG(ERROR) << "The index has been locked: " << index->get_index_name(); return nebula::cpp2::ErrorCode::E_DATA_CONFLICT_ERROR; } else { - batchHolder->remove(std::move(indexKey)); + for (auto& indexKey : indexKeys) { + batchHolder->remove(std::move(indexKey)); + } } } } diff --git a/src/storage/test/CMakeLists.txt b/src/storage/test/CMakeLists.txt index 33dee520e63..9ad3b3e7529 100644 --- a/src/storage/test/CMakeLists.txt +++ b/src/storage/test/CMakeLists.txt @@ -49,6 +49,7 @@ set(storage_test_deps $ $ $ + $ ) nebula_add_test( diff --git a/src/storage/test/IndexScanLimitTest.cpp b/src/storage/test/IndexScanLimitTest.cpp index 3dd2482c339..a73e0186a80 100644 --- a/src/storage/test/IndexScanLimitTest.cpp +++ b/src/storage/test/IndexScanLimitTest.cpp @@ -124,24 +124,28 @@ class IndexScanLimitTest : public ::testing::Test { data.emplace_back(std::move(vertexKey), std::move(val)); if (indexMan_ != nullptr) { if (indexMan_->getTagIndex(spaceId, tagIndex).ok()) { - auto vertexIndexKey = - IndexKeyUtils::vertexIndexKey(vertexLen, - pId, - tagIndex, - vertex, - IndexKeyUtils::encodeValues({col1Val}, genCols())); - data.emplace_back(std::move(vertexIndexKey), ""); + auto vertexIndexKeys = + IndexKeyUtils::vertexIndexKeys(vertexLen, + pId, + tagIndex, + vertex, + IndexKeyUtils::encodeValues({col1Val}, genCols())); + for (auto& vertexIndexKey : vertexIndexKeys) { + data.emplace_back(std::move(vertexIndexKey), ""); + } } if (indexMan_->getEdgeIndex(spaceId, edgeIndex).ok()) { - auto edgeIndexKey = - IndexKeyUtils::edgeIndexKey(vertexLen, - pId, - edgeIndex, - vertex, - 0, - vertex, - IndexKeyUtils::encodeValues({col1Val}, genCols())); - data.emplace_back(std::move(edgeIndexKey), ""); + auto edgeIndexKeys = + IndexKeyUtils::edgeIndexKeys(vertexLen, + pId, + edgeIndex, + vertex, + 0, + vertex, + IndexKeyUtils::encodeValues({col1Val}, genCols())); + for (auto& edgeIndexKey : edgeIndexKeys) { + data.emplace_back(std::move(edgeIndexKey), ""); + } } } } diff --git a/src/storage/test/IndexWriteTest.cpp b/src/storage/test/IndexWriteTest.cpp index 8b08c5384e4..7a8dc0bd85b 100644 --- a/src/storage/test/IndexWriteTest.cpp +++ b/src/storage/test/IndexWriteTest.cpp @@ -319,13 +319,13 @@ TEST(IndexTest, VerticesValueTest) { values.emplace_back(Value(date)); // col_date_null values.emplace_back(nullValue); - auto index = + auto indexes = IndexKeyUtils::encodeValues(std::move(values), mock::MockData::mockTypicaIndexColumns()); for (auto partId = 1; partId <= 6; partId++) { auto prefix = IndexKeyUtils::indexPrefix(partId, indexId); - auto indexKey = IndexKeyUtils::vertexIndexKey( - vIdLen, partId, indexId, convertVertexId(vIdLen, partId), std::move(index)); + auto indexKey = IndexKeyUtils::vertexIndexKeys( + vIdLen, partId, indexId, convertVertexId(vIdLen, partId), std::move(indexes))[0]; std::unique_ptr iter; auto ret = env->kvstore_->prefix(spaceId, partId, prefix, &iter); EXPECT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, ret); diff --git a/src/storage/test/LookupIndexTest.cpp b/src/storage/test/LookupIndexTest.cpp index c9d35dc892e..1f0fc4986d2 100644 --- a/src/storage/test/LookupIndexTest.cpp +++ b/src/storage/test/LookupIndexTest.cpp @@ -94,11 +94,15 @@ TEST_P(LookupIndexTest, LookupIndexTestV1) { indexVal1.append(IndexKeyUtils::encodeValue("row1")); std::string indexVal2 = indexVal1; - key = IndexKeyUtils::vertexIndexKey(vIdLen.value(), 1, 3, vId1, std::move(indexVal1)); - keyValues.emplace_back(std::move(key), ""); + auto keys = IndexKeyUtils::vertexIndexKeys(vIdLen.value(), 1, 3, vId1, {std::move(indexVal1)}); + for (auto& k : keys) { + keyValues.emplace_back(std::move(k), ""); + } - key = IndexKeyUtils::vertexIndexKey(vIdLen.value(), 1, 3, vId2, std::move(indexVal2)); - keyValues.emplace_back(std::move(key), ""); + keys = IndexKeyUtils::vertexIndexKeys(vIdLen.value(), 1, 3, vId2, {std::move(indexVal2)}); + for (auto& k : keys) { + keyValues.emplace_back(std::move(k), ""); + } folly::Baton baton; env->kvstore_->asyncMultiPut( diff --git a/src/storage/test/QueryTestUtils.h b/src/storage/test/QueryTestUtils.h index 28cd5872a54..b6cbca6037a 100644 --- a/src/storage/test/QueryTestUtils.h +++ b/src/storage/test/QueryTestUtils.h @@ -260,9 +260,12 @@ class QueryTestUtils { row.append(IndexKeyUtils::encodeValue(v)); } } - auto index = IndexKeyUtils::vertexIndexKey(spaceVidLen, partId, indexId, vId, std::move(row)); + auto indexes = + IndexKeyUtils::vertexIndexKeys(spaceVidLen, partId, indexId, vId, {std::move(row)}); auto val = FLAGS_mock_ttl_col ? IndexKeyUtils::indexVal(time::WallClock::fastNowInSec()) : ""; - data.emplace_back(std::move(index), std::move(val)); + for (auto& index : indexes) { + data.emplace_back(std::move(index), std::move(val)); + } } static void encodeEdgeIndex(size_t spaceVidLen, @@ -283,10 +286,12 @@ class QueryTestUtils { row.append(IndexKeyUtils::encodeValue(v)); } } - auto index = IndexKeyUtils::edgeIndexKey( - spaceVidLen, partId, indexId, srcId, rank, dstId, std::move(row)); + auto indexes = IndexKeyUtils::edgeIndexKeys( + spaceVidLen, partId, indexId, srcId, rank, dstId, {std::move(row)}); auto val = FLAGS_mock_ttl_col ? IndexKeyUtils::indexVal(time::WallClock::fastNowInSec()) : ""; - data.emplace_back(std::move(index), std::move(val)); + for (auto& index : indexes) { + data.emplace_back(std::move(index), val); + } } static cpp2::GetNeighborsRequest buildRequest( diff --git a/src/tools/db-dump/CMakeLists.txt b/src/tools/db-dump/CMakeLists.txt index d48e0f73575..893bd53e494 100644 --- a/src/tools/db-dump/CMakeLists.txt +++ b/src/tools/db-dump/CMakeLists.txt @@ -45,6 +45,7 @@ set(tools_test_deps $ $ $ + $ ) nebula_add_executable( diff --git a/src/tools/db-upgrade/CMakeLists.txt b/src/tools/db-upgrade/CMakeLists.txt index 7b3670d20ee..4aa3f68be00 100644 --- a/src/tools/db-upgrade/CMakeLists.txt +++ b/src/tools/db-upgrade/CMakeLists.txt @@ -53,6 +53,7 @@ nebula_add_executable( $ $ $ + $ LIBRARIES ${ROCKSDB_LIBRARIES} ${THRIFT_LIBRARIES} diff --git a/src/tools/db-upgrade/DbUpgrader.cpp b/src/tools/db-upgrade/DbUpgrader.cpp index abf39881a5e..e1d5664203a 100644 --- a/src/tools/db-upgrade/DbUpgrader.cpp +++ b/src/tools/db-upgrade/DbUpgrader.cpp @@ -656,8 +656,8 @@ void UpgraderSpace::encodeVertexValue(PartitionID partId, return; } for (auto& index : it->second) { - auto newIndexKey = indexVertexKey(partId, strVid, nReader.get(), index); - if (!newIndexKey.empty()) { + auto newIndexKeys = indexVertexKeys(partId, strVid, nReader.get(), index); + for (auto& newIndexKey : newIndexKeys) { data.emplace_back(std::move(newIndexKey), ""); } } @@ -881,15 +881,16 @@ std::string UpgraderSpace::encodeRowVal(const RowReader* reader, return std::move(rowWrite).moveEncodedStr(); } -std::string UpgraderSpace::indexVertexKey(PartitionID partId, - VertexID& vId, - RowReader* reader, - std::shared_ptr index) { +std::vector UpgraderSpace::indexVertexKeys( + PartitionID partId, + VertexID& vId, + RowReader* reader, + std::shared_ptr index) { auto values = IndexKeyUtils::collectIndexValues(reader, index->get_fields()); if (!values.ok()) { - return ""; + return {}; } - return IndexKeyUtils::vertexIndexKey( + return IndexKeyUtils::vertexIndexKeys( spaceVidLen_, partId, index->get_index_id(), vId, std::move(values).value()); } @@ -926,25 +927,26 @@ void UpgraderSpace::encodeEdgeValue(PartitionID partId, return; } for (auto& index : it->second) { - auto newIndexKey = indexEdgeKey(partId, nReader.get(), svId, rank, dstId, index); - if (!newIndexKey.empty()) { + auto newIndexKeys = indexEdgeKeys(partId, nReader.get(), svId, rank, dstId, index); + for (auto& newIndexKey : newIndexKeys) { data.emplace_back(std::move(newIndexKey), ""); } } } } -std::string UpgraderSpace::indexEdgeKey(PartitionID partId, - RowReader* reader, - VertexID& svId, - EdgeRanking rank, - VertexID& dstId, - std::shared_ptr index) { +std::vector UpgraderSpace::indexEdgeKeys( + PartitionID partId, + RowReader* reader, + VertexID& svId, + EdgeRanking rank, + VertexID& dstId, + std::shared_ptr index) { auto values = IndexKeyUtils::collectIndexValues(reader, index->get_fields()); if (!values.ok()) { - return ""; + return {}; } - return IndexKeyUtils::edgeIndexKey( + return IndexKeyUtils::edgeIndexKeys( spaceVidLen_, partId, index->get_index_id(), svId, rank, dstId, std::move(values).value()); } diff --git a/src/tools/db-upgrade/DbUpgrader.h b/src/tools/db-upgrade/DbUpgrader.h index 7b0ab61aaec..e0ed041eb66 100644 --- a/src/tools/db-upgrade/DbUpgrader.h +++ b/src/tools/db-upgrade/DbUpgrader.h @@ -82,10 +82,10 @@ class UpgraderSpace { const meta::NebulaSchemaProvider* schema, std::vector& fieldName); - std::string indexVertexKey(PartitionID partId, - VertexID& vId, - RowReader* reader, - std::shared_ptr index); + std::vector indexVertexKeys(PartitionID partId, + VertexID& vId, + RowReader* reader, + std::shared_ptr index); void encodeEdgeValue(PartitionID partId, RowReader* reader, @@ -97,12 +97,12 @@ class UpgraderSpace { VertexID& dstId, std::vector& data); - std::string indexEdgeKey(PartitionID partId, - RowReader* reader, - VertexID& svId, - EdgeRanking rank, - VertexID& dstId, - std::shared_ptr index); + std::vector indexEdgeKeys(PartitionID partId, + RowReader* reader, + VertexID& svId, + EdgeRanking rank, + VertexID& dstId, + std::shared_ptr index); WriteResult convertValue(const meta::NebulaSchemaProvider* newSchema, const meta::SchemaProviderIf* oldSchema, diff --git a/src/tools/meta-dump/CMakeLists.txt b/src/tools/meta-dump/CMakeLists.txt index 663e4686366..46c508f3b1f 100644 --- a/src/tools/meta-dump/CMakeLists.txt +++ b/src/tools/meta-dump/CMakeLists.txt @@ -50,6 +50,7 @@ nebula_add_executable( $ $ $ + $ LIBRARIES ${ROCKSDB_LIBRARIES} ${THRIFT_LIBRARIES} diff --git a/src/tools/simple-kv-verify/CMakeLists.txt b/src/tools/simple-kv-verify/CMakeLists.txt index 75c8b9cfb88..17a2592fe13 100644 --- a/src/tools/simple-kv-verify/CMakeLists.txt +++ b/src/tools/simple-kv-verify/CMakeLists.txt @@ -44,6 +44,7 @@ nebula_add_executable( $ $ $ + $ LIBRARIES ${ROCKSDB_LIBRARIES} ${THRIFT_LIBRARIES} diff --git a/src/tools/storage-perf/CMakeLists.txt b/src/tools/storage-perf/CMakeLists.txt index 5a4400210d2..e278053ecb2 100644 --- a/src/tools/storage-perf/CMakeLists.txt +++ b/src/tools/storage-perf/CMakeLists.txt @@ -45,6 +45,7 @@ set(perf_test_deps $ $ $ + $ ) nebula_add_executable( diff --git a/tests/query/stateless/test_schema.py b/tests/query/stateless/test_schema.py index b0b27ca3851..0e13d8d228a 100644 --- a/tests/query/stateless/test_schema.py +++ b/tests/query/stateless/test_schema.py @@ -148,20 +148,6 @@ def test_alter_tag_succeed(self): ['age', 'int64', 'YES', T_EMPTY, T_EMPTY]] self.check_result(resp, expect) - # alter all - resp = self.execute('ALTER TAG student drop (name),' - 'ADD (gender string),' - 'CHANGE (gender int)') - self.check_resp_succeeded(resp) - - resp = self.execute('DESC TAG student') - self.check_resp_succeeded(resp) - expect = [['email', 'string', 'YES', T_EMPTY, T_EMPTY], - ['birthday', 'timestamp', 'YES', T_EMPTY, T_EMPTY], - ['age', 'int64', 'YES', T_EMPTY, T_EMPTY], - ['gender', 'int64', 'YES', T_EMPTY, T_EMPTY]] - self.check_result(resp, expect) - def test_alter_tag_failed(self): # alter ttl_col on wrong type try: @@ -304,19 +290,6 @@ def test_alter_edge_succeed(self): ['start_year', 'int64', 'YES', T_EMPTY, T_EMPTY]] self.check_result(resp, expect) - # alter all - resp = self.execute('ALTER EDGE relationship drop (name),' - 'ADD (end_year string),' - 'CHANGE (end_year int)') - self.check_resp_succeeded(resp) - - resp = self.execute('DESC EDGE relationship') - self.check_resp_succeeded(resp) - expect = [['email', 'string', 'YES', T_EMPTY, T_EMPTY], - ['start_year', 'int64', 'YES', T_EMPTY, T_EMPTY], - ['end_year', 'int64', 'YES', T_EMPTY, T_EMPTY]] - self.check_result(resp, expect) - def test_alter_edge_failed(self): # alter ttl_col on wrong type try: diff --git a/tests/tck/features/fetch/FetchVertices.intVid.feature b/tests/tck/features/fetch/FetchVertices.intVid.feature index aac6a8eac58..cdbfe636d7a 100644 --- a/tests/tck/features/fetch/FetchVertices.intVid.feature +++ b/tests/tck/features/fetch/FetchVertices.intVid.feature @@ -402,3 +402,17 @@ Feature: Fetch Int Vid Vertices Then the result should be, in any order, and the columns 0, 2 should be hashed: | VertexID | player.name | id(VERTEX) | | "Boris Diaw" | "Boris Diaw" | "Boris Diaw" | + When executing query: + """ + FETCH PROP ON player hash('Tim Duncan') YIELD id(vertex), properties(vertex).name as name, properties(vertex) + """ + Then the result should be, in any order, and the columns 0, 1 should be hashed: + | VertexID | id(VERTEX) | name | properties(VERTEX) | + | "Tim Duncan" | "Tim Duncan" | "Tim Duncan" | {age: 42, name: "Tim Duncan"} | + When executing query: + """ + FETCH PROP ON * hash('Tim Duncan') YIELD id(vertex), keys(vertex) as keys, tags(vertex) as tagss, properties(vertex) as props + """ + Then the result should be, in any order, and the columns 0, 1 should be hashed: + | VertexID | id(VERTEX) | keys | tagss | props | + | "Tim Duncan" | "Tim Duncan" | ["age", "name", "speciality"] | ["bachelor", "player"] | {age: 42, name: "Tim Duncan", speciality: "psychology"} | diff --git a/tests/tck/features/fetch/FetchVertices.strVid.feature b/tests/tck/features/fetch/FetchVertices.strVid.feature index 8d89694e0a0..ad1dc516c01 100644 --- a/tests/tck/features/fetch/FetchVertices.strVid.feature +++ b/tests/tck/features/fetch/FetchVertices.strVid.feature @@ -513,3 +513,17 @@ Feature: Fetch String Vertices Then the result should be, in any order: | VertexID | player.name | id(VERTEX) | | "Boris Diaw" | "Boris Diaw" | "Boris Diaw" | + When executing query: + """ + FETCH PROP ON player 'Tim Duncan' YIELD id(vertex), properties(vertex).name as name, properties(vertex) + """ + Then the result should be, in any order: + | VertexID | id(VERTEX) | name | properties(VERTEX) | + | "Tim Duncan" | "Tim Duncan" | "Tim Duncan" | {age: 42, name: "Tim Duncan"} | + When executing query: + """ + FETCH PROP ON * 'Tim Duncan' YIELD id(vertex), keys(vertex) as keys, tags(vertex) as tagss, properties(vertex) as props + """ + Then the result should be, in any order: + | VertexID | id(VERTEX) | keys | tagss | props | + | "Tim Duncan" | "Tim Duncan" | ["age", "name", "speciality"] | ["bachelor", "player"] | {age: 42, name: "Tim Duncan", speciality: "psychology"} | diff --git a/tests/tck/features/geo/GeoBase.feature b/tests/tck/features/geo/GeoBase.feature new file mode 100644 index 00000000000..c390f20a6ed --- /dev/null +++ b/tests/tck/features/geo/GeoBase.feature @@ -0,0 +1,705 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. +Feature: Geo base + + Background: + Given an empty graph + And create a space with following options: + | partition_num | 9 | + | replica_factor | 1 | + | vid_type | FIXED_STRING(30) | + | charset | utf8 | + | collate | utf8_bin | + And having executed: + """ + CREATE TAG any_shape(geo geography); + CREATE TAG only_point(geo geography(point)); + CREATE TAG only_linestring(geo geography(linestring)); + CREATE TAG only_polygon(geo geography(polygon)); + CREATE EDGE any_shape_edge(geo geography); + """ + And wait 3 seconds + + Scenario: test geo schema + # Desc geo schema + When executing query: + """ + DESC TAG any_shape; + """ + Then the result should be, in any order: + | Field | Type | Null | Default | Comment | + | "geo" | "geography" | "YES" | EMPTY | EMPTY | + When executing query: + """ + DESC TAG only_point; + """ + Then the result should be, in any order: + | Field | Type | Null | Default | Comment | + | "geo" | "geography(point)" | "YES" | EMPTY | EMPTY | + When executing query: + """ + DESC TAG only_linestring; + """ + Then the result should be, in any order: + | Field | Type | Null | Default | Comment | + | "geo" | "geography(linestring)" | "YES" | EMPTY | EMPTY | + When executing query: + """ + DESC TAG only_polygon; + """ + Then the result should be, in any order: + | Field | Type | Null | Default | Comment | + | "geo" | "geography(polygon)" | "YES" | EMPTY | EMPTY | + When executing query: + """ + DESC EDGE any_shape_edge; + """ + Then the result should be, in any order: + | Field | Type | Null | Default | Comment | + | "geo" | "geography" | "YES" | EMPTY | EMPTY | + # Show create geo schema + When executing query: + """ + SHOW CREATE TAG only_point; + """ + Then the result should be, in any order: + | Tag | Create Tag | + | "only_point" | 'CREATE TAG `only_point` (\n `geo` geography(point) NULL\n) ttl_duration = 0, ttl_col = ""' | + + Scenario: test geo CURD + # Any geo shape(point/linestring/polygon) is allowed to insert to the column geography + When try to execute query: + """ + INSERT VERTEX any_shape(geo) VALUES "101":(ST_GeogFromText("POINT(3 8)")); + """ + Then the execution should be successful + When executing query: + """ + INSERT VERTEX any_shape(geo) VALUES "102":(ST_GeogFromText("LINESTRING(3 8, 4.7 73.23)")); + """ + Then the execution should be successful + When executing query: + """ + INSERT VERTEX any_shape(geo) VALUES "103":(ST_GeogFromText("POLYGON((0 1, 1 2, 2 3, 0 1))")); + """ + Then the execution should be successful + # Only point is allowed to insert to the column geograph(point) + When executing query: + """ + INSERT VERTEX only_point(geo) VALUES "201":(ST_GeogFromText("POINT(3 8)")); + """ + Then the execution should be successful + When executing query: + """ + INSERT VERTEX only_point(geo) VALUES "202":(ST_GeogFromText("LINESTRING(3 8, 4.7 73.23)")); + """ + Then a ExecutionError should be raised at runtime: Storage Error: The data type does not meet the requirements. Use the correct type of data. + When executing query: + """ + INSERT VERTEX only_point(geo) VALUES "203":(ST_GeogFromText("POLYGON((0 1, 1 2, 2 3, 0 1))")); + """ + Then a ExecutionError should be raised at runtime: Storage Error: The data type does not meet the requirements. Use the correct type of data. + # Only linestring is allowed to insert to the column geograph(linestring) + When executing query: + """ + INSERT VERTEX only_linestring(geo) VALUES "301":(ST_GeogFromText("POINT(3 8)")); + """ + Then a ExecutionError should be raised at runtime: Storage Error: The data type does not meet the requirements. Use the correct type of data. + When executing query: + """ + INSERT VERTEX only_linestring(geo) VALUES "302":(ST_GeogFromText("LINESTRING(3 8, 4.7 73.23)")); + """ + Then the execution should be successful + When executing query: + """ + INSERT VERTEX only_linestring(geo) VALUES "303":(ST_GeogFromText("POLYGON((0 1, 1 2, 2 3, 0 1))")); + """ + Then a ExecutionError should be raised at runtime: Storage Error: The data type does not meet the requirements. Use the correct type of data. + # Only polygon is allowed to insert to the column geograph(polygon) + When executing query: + """ + INSERT VERTEX only_polygon(geo) VALUES "401":(ST_GeogFromText("POINT(3 8)")); + """ + Then a ExecutionError should be raised at runtime: Storage Error: The data type does not meet the requirements. Use the correct type of data. + When executing query: + """ + INSERT VERTEX only_polygon(geo) VALUES "402":(ST_GeogFromText("LINESTRING(3 8, 4.7 73.23)")); + """ + Then a ExecutionError should be raised at runtime: Storage Error: The data type does not meet the requirements. Use the correct type of data. + When executing query: + """ + INSERT VERTEX only_polygon(geo) VALUES "403":(ST_GeogFromText("POLYGON((0 1, 1 2, 2 3, 0 1))")); + """ + Then the execution should be successful + When executing query: + """ + INSERT EDGE any_shape_edge(geo) VALUES "201"->"302":(ST_GeogFromText("POLYGON((0 1, 1 2, 2 3, 0 1))")); + """ + Then the execution should be successful + # Fetch the geo column + When executing query: + """ + FETCH PROP ON any_shape "101","102","103" YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + | "101" | "POINT(3 8)" | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + When executing query: + """ + FETCH PROP ON only_point "201","202","203" YIELD ST_ASText(only_point.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(only_point.geo) | + | "201" | "POINT(3 8)" | + When executing query: + """ + FETCH PROP ON only_linestring "301","302","303" YIELD ST_ASText(only_linestring.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(only_linestring.geo) | + | "302" | "LINESTRING(3 8, 4.7 73.23)" | + When executing query: + """ + FETCH PROP ON only_polygon "401","402","403" YIELD ST_ASText(only_polygon.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(only_polygon.geo) | + | "403" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + When executing query: + """ + FETCH PROP ON any_shape_edge "201"->"302" YIELD ST_ASText(any_shape_edge.geo); + """ + Then the result should be, in any order: + | any_shape_edge._src | any_shape_edge._dst | any_shape_edge._rank | ST_ASText(any_shape_edge.geo) | + | "201" | "302" | 0 | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + # Create index on geo column + When executing query: + """ + CREATE TAG INDEX any_shape_geo_index ON any_shape(geo); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX only_point_geo_index ON only_point(geo); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX only_linestring_geo_index ON only_linestring(geo); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX only_polygon_geo_index ON only_polygon(geo); + """ + Then the execution should be successful + When executing query: + """ + CREATE EDGE INDEX any_shape_edge_geo_index ON any_shape_edge(geo); + """ + Then the execution should be successful + And wait 3 seconds + # Rebuild the geo index + When submit a job: + """ + REBUILD TAG INDEX any_shape_geo_index; + """ + Then wait the job to finish + When submit a job: + """ + REBUILD TAG INDEX only_point_geo_index; + """ + Then wait the job to finish + When submit a job: + """ + REBUILD TAG INDEX only_linestring_geo_index; + """ + Then wait the job to finish + When submit a job: + """ + REBUILD TAG INDEX only_polygon_geo_index; + """ + Then wait the job to finish + When submit a job: + """ + REBUILD EDGE INDEX any_shape_edge_geo_index; + """ + Then wait the job to finish + # Lookup on geo index + When profiling query: + """ + LOOKUP ON any_shape YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + And the execution plan should be: + | id | name | dependencies | operator info | + | 2 | Project | 3 | | + | 3 | TagIndexFullScan | 0 | | + | 0 | Start | | | + When executing query: + """ + LOOKUP ON only_point YIELD ST_ASText(only_point.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(only_point.geo) | + | "201" | "POINT(3 8)" | + When executing query: + """ + LOOKUP ON only_linestring YIELD ST_ASText(only_linestring.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(only_linestring.geo) | + | "302" | "LINESTRING(3 8, 4.7 73.23)" | + When executing query: + """ + LOOKUP ON only_polygon YIELD ST_ASText(only_polygon.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(only_polygon.geo) | + | "403" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + When executing query: + """ + LOOKUP ON any_shape_edge YIELD ST_ASText(any_shape_edge.geo); + """ + Then the result should be, in any order: + | SrcVID | DstVID | Ranking | ST_ASText(any_shape_edge.geo) | + | "201" | "302" | 0 | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + # Match with geo index + When executing query: + """ + MATCH (v:any_shape) RETURN ST_ASText(v.geo); + """ + Then the result should be, in any order: + | ST_ASText(v.geo) | + | "POINT(3 8)" | + | "LINESTRING(3 8, 4.7 73.23)" | + | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + # Insert geo data with index + When try to execute query: + """ + INSERT VERTEX any_shape(geo) VALUES "108":(ST_GeogFromText("POINT(72.3 84.6)")); + """ + Then the execution should be successful + When executing query: + """ + INSERT VERTEX only_point(geo) VALUES "208":(ST_GeogFromText("POINT(0.01 0.01)")); + """ + Then the execution should be successful + When executing query: + """ + INSERT VERTEX only_linestring(geo) VALUES "308":(ST_GeogFromText("LINESTRING(9 9, 8 8, 7 7, 9 9)")); + """ + Then the execution should be successful + When executing query: + """ + INSERT VERTEX only_polygon(geo) VALUES "408":(ST_GeogFromText("POLYGON((0 1, 1 2, 2 3, 0 1))")); + """ + Then the execution should be successful + When executing query: + """ + INSERT EDGE any_shape_edge(geo) VALUES "108"->"408":(ST_GeogFromText("POLYGON((-20 -20, -20 20, 20 20, 20 -20, -20 -20), (1.0 1.0, 2.0 2.0, 0.0 2.0, 1.0 1.0))")); + """ + Then the execution should be successful + # Lookup on geo index agagin + When executing query: + """ + LOOKUP ON any_shape YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + | "108" | "POINT(72.3 84.6)" | + When executing query: + """ + LOOKUP ON only_point YIELD ST_ASText(only_point.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(only_point.geo) | + | "201" | "POINT(3 8)" | + | "208" | "POINT(0.01 0.01)" | + When executing query: + """ + LOOKUP ON only_linestring YIELD ST_ASText(only_linestring.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(only_linestring.geo) | + | "302" | "LINESTRING(3 8, 4.7 73.23)" | + | "308" | "LINESTRING(9 9, 8 8, 7 7, 9 9)" | + When executing query: + """ + LOOKUP ON only_polygon YIELD ST_ASText(only_polygon.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(only_polygon.geo) | + | "403" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + | "408" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + When executing query: + """ + LOOKUP ON any_shape_edge YIELD ST_ASText(any_shape_edge.geo); + """ + Then the result should be, in any order: + | SrcVID | DstVID | Ranking | ST_ASText(any_shape_edge.geo) | + | "108" | "408" | 0 | "POLYGON((-20 -20, -20 20, 20 20, 20 -20, -20 -20), (1 1, 2 2, 0 2, 1 1))" | + | "201" | "302" | 0 | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + # Lookup and Yield geo functions + When executing query: + """ + LOOKUP ON any_shape YIELD S2_CellIdFromPoint(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | S2_CellIdFromPoint(any_shape.geo) | + | "101" | 1166542697063163289 | + | "102" | BAD_DATA | + | "103" | BAD_DATA | + | "108" | 4987215245349669805 | + When executing query: + """ + LOOKUP ON any_shape YIELD S2_CoveringCellIds(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | S2_CoveringCellIds(any_shape.geo) | + | "101" | [1166542697063163289] | + | "102" | [1167558203395801088, 1279022294173220864, 1315051091192184832, 1351079888211148800, 5039527983027585024, 5062045981164437504, 5174635971848699904, 5183643171103440896] | + | "103" | [1152391494368201343, 1153466862374223872, 1153554823304445952, 1153836298281156608, 1153959443583467520, 1154240918560178176, 1160503736791990272, 1160591697722212352] | + | "108" | [4987215245349669805] | + # Lookup with geo predicates which could be index accelerated + # ST_Intersects + When profiling query: + """ + LOOKUP ON any_shape WHERE ST_Intersects(any_shape.geo, ST_GeogFromText('POINT(3 8)')) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + And the execution plan should be: + | id | name | dependencies | operator info | + | 3 | Project | 4 | | + | 4 | IndexScan | 0 | | + | 0 | Start | | | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Intersects(any_shape.geo, ST_GeogFromText('POINT(0 1)')) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Intersects(any_shape.geo, ST_GeogFromText('POINT(4.7 73.23)')) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Intersects(any_shape.geo, ST_Point(72.3, 84.6)) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "108" | "POINT(72.3 84.6)" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Intersects(ST_Point(72.3, 84.6), any_shape.geo) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "108" | "POINT(72.3 84.6)" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Intersects(any_shape.geo, any_shape.geo) YIELD ST_ASText(any_shape.geo); + """ + Then a SemanticError should be raised at runtime: Expression ST_Intersects(any_shape.geo,any_shape.geo) not supported yet + # Match with geo predicate + When executing query: + """ + MATCH (v:any_shape) WHERE ST_Intersects(v.geo, ST_GeogFromText('POINT(3 8)')) RETURN ST_ASText(v.geo); + """ + Then the result should be, in any order: + | ST_ASText(v.geo) | + | "POINT(3 8)" | + | "LINESTRING(3 8, 4.7 73.23)" | + # ST_Distance + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Distance(any_shape.geo, ST_Point(3, 8)) < 1.0 YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Distance(any_shape.geo, ST_Point(3, 8)) <= 1.0 YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Distance(any_shape.geo, ST_Point(3, 8)) <= 8909524.383934561 YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + | "108" | "POINT(72.3 84.6)" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Distance(any_shape.geo, ST_Point(3, 8)) < 8909524.383934561 YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Distance(any_shape.geo, ST_Point(3, 8)) < 8909524.383934563 YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + | "108" | "POINT(72.3 84.6)" | + When executing query: + """ + LOOKUP ON any_shape WHERE 8909524.383934560 > ST_Distance(any_shape.geo, ST_Point(3, 8)) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + When executing query: + """ + LOOKUP ON any_shape WHERE 8909524.3839345630 >= ST_Distance(any_shape.geo, ST_Point(3, 8)) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + | "108" | "POINT(72.3 84.6)" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Distance(any_shape.geo, ST_Point(3, 8)) > 1.0 YIELD ST_ASText(any_shape.geo); + """ + Then a SemanticError should be raised at runtime: Expression (ST_Distance(any_shape.geo,ST_Point(3,8))>1) not supported yet + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Distance(any_shape.geo, ST_Point(3, 8)) != 1.0 YIELD ST_ASText(any_shape.geo); + """ + Then a SemanticError should be raised at runtime: Expression (ST_Distance(any_shape.geo,ST_Point(3,8))!=1) not supported yet + # ST_DWithin + When executing query: + """ + LOOKUP ON any_shape WHERE ST_DWithin(any_shape.geo, ST_Point(3, 8), 8909524.383934561) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + | "108" | "POINT(72.3 84.6)" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_DWithin(any_shape.geo, ST_Point(3, 8), 100.0) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + # ST_Covers + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Covers(any_shape.geo, ST_Point(3, 8)) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Covers(any_shape.geo, ST_Point(3, 8)) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "POINT(3 8)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Covers(ST_GeogFromText('POLYGON((-0.7 3.8,3.6 3.2,1.8 -0.8,-3.4 2.4,-0.7 3.8))'), any_shape.geo) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_CoveredBy(any_shape.geo, ST_GeogFromText('POLYGON((-0.7 3.8,3.6 3.2,1.8 -0.8,-3.4 2.4,-0.7 3.8))')) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + # Update vertex with index + When executing query: + """ + UPDATE VERTEX ON any_shape "101" SET any_shape.geo = ST_GeogFromText('LINESTRING(3 8, 6 16)'); + """ + Then the execution should be successful + When executing query: + """ + FETCH PROP ON any_shape "101" YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "LINESTRING(3 8, 6 16)" | + When executing query: + """ + LOOKUP ON any_shape YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "LINESTRING(3 8, 6 16)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + | "108" | "POINT(72.3 84.6)" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_DWithin(any_shape.geo, ST_Point(3, 8), 100.0) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "101" | "LINESTRING(3 8, 6 16)" | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + # Update edge with index + When executing query: + """ + UPDATE EDGE ON any_shape_edge "201"->"302" SET any_shape_edge.geo = ST_GeogFromText('POINT(-1 -1)'); + """ + Then the execution should be successful + When executing query: + """ + FETCH PROP ON any_shape_edge "201"->"302" YIELD ST_ASText(any_shape_edge.geo); + """ + Then the result should be, in any order: + | any_shape_edge._src | any_shape_edge._dst | any_shape_edge._rank | ST_ASText(any_shape_edge.geo) | + | "201" | "302" | 0 | "POINT(-1 -1)" | + When executing query: + """ + LOOKUP ON any_shape_edge YIELD ST_ASText(any_shape_edge.geo); + """ + Then the result should be, in any order: + | SrcVID | DstVID | Ranking | ST_ASText(any_shape_edge.geo) | + | "108" | "408" | 0 | "POLYGON((-20 -20, -20 20, 20 20, 20 -20, -20 -20), (1 1, 2 2, 0 2, 1 1))" | + | "201" | "302" | 0 | "POINT(-1 -1)" | + When executing query: + """ + LOOKUP ON any_shape_edge WHERE ST_Intersects(any_shape_edge.geo, ST_Point(-1, -1)) YIELD ST_ASText(any_shape_edge.geo); + """ + Then the result should be, in any order: + | SrcVID | DstVID | Ranking | ST_ASText(any_shape_edge.geo) | + | "108" | "408" | 0 | "POLYGON((-20 -20, -20 20, 20 20, 20 -20, -20 -20), (1 1, 2 2, 0 2, 1 1))" | + | "201" | "302" | 0 | "POINT(-1 -1)" | + # Delete vertex with index + When executing query: + """ + DELETE VERTEX "101"; + """ + Then the execution should be successful + When executing query: + """ + FETCH PROP ON any_shape "101" YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + When executing query: + """ + LOOKUP ON any_shape YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + | "103" | "POLYGON((0 1, 1 2, 2 3, 0 1))" | + | "108" | "POINT(72.3 84.6)" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Covers(any_shape.geo, ST_Point(3, 8)) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + | "102" | "LINESTRING(3 8, 4.7 73.23)" | + # Delete edge with index + When executing query: + """ + DELETE EDGE any_shape_edge "201"->"302"; + """ + Then the execution should be successful + When executing query: + """ + FETCH PROP ON any_shape_edge "201"->"302" YIELD ST_ASText(any_shape_edge.geo); + """ + Then the result should be, in any order: + | any_shape_edge._src | any_shape_edge._dst | any_shape_edge._rank | ST_ASText(any_shape_edge.geo) | + When executing query: + """ + LOOKUP ON any_shape_edge YIELD ST_ASText(any_shape_edge.geo); + """ + Then the result should be, in any order: + | SrcVID | DstVID | Ranking | ST_ASText(any_shape_edge.geo) | + | "108" | "408" | 0 | "POLYGON((-20 -20, -20 20, 20 20, 20 -20, -20 -20), (1 1, 2 2, 0 2, 1 1))" | + When executing query: + """ + LOOKUP ON any_shape WHERE ST_Intersects(ST_Point(-1, -1), any_shape.geo) YIELD ST_ASText(any_shape.geo); + """ + Then the result should be, in any order: + | VertexID | ST_ASText(any_shape.geo) | + # Drop tag index + When executing query: + """ + DROP TAG INDEX any_shape_geo_index; + """ + Then the execution should be successful + And wait 3 seconds + When executing query: + """ + LOOKUP ON any_shape; + """ + Then a ExecutionError should be raised at runtime: There is no index to use at runtime + # Drop edge index + When executing query: + """ + DROP EDGE INDEX any_shape_edge_geo_index; + """ + Then the execution should be successful + And wait 3 seconds + When executing query: + """ + LOOKUP ON any_shape_edge; + """ + Then a ExecutionError should be raised at runtime: There is no index to use at runtime + # Drop tag + When executing query: + """ + DROP TAG any_shape; + """ + Then the execution should be successful + # Drop edge + When executing query: + """ + DROP EDGE any_shape_edge; + """ + Then the execution should be successful diff --git a/tests/tck/features/go/GO.IntVid.feature b/tests/tck/features/go/GO.IntVid.feature index 1bb89bfca70..10636afa173 100644 --- a/tests/tck/features/go/GO.IntVid.feature +++ b/tests/tck/features/go/GO.IntVid.feature @@ -1434,6 +1434,45 @@ Feature: IntegerVid Go Sentence | EMPTY | "Russell Westbrook" | | EMPTY | "Luka Doncic" | | EMPTY | "Russell Westbrook" | + When executing query: + """ + go 1 to 4 steps from hash("Tim Duncan") over like where like.likeness > 90 yield like.likeness, edge as e + """ + Then the result should be, in any order, with relax comparison: + | like.likeness | e | + | 95 | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | 95 | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | 95 | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + When executing query: + """ + go 1 to 4 steps from hash("Tim Duncan") over like yield like.likeness, edge as e + """ + Then the result should be, in any order, with relax comparison: + | like.likeness | e | + | 95 | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | 90 | [:like "Manu Ginobili"->"Tim Duncan" @0 {likeness: 90}] | + | 90 | [:like "Tony Parker"->"LaMarcus Aldridge" @0 {likeness: 90}] | + | 95 | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | + | 75 | [:like "LaMarcus Aldridge"->"Tim Duncan" @0 {likeness: 75}] | + | 75 | [:like "LaMarcus Aldridge"->"Tony Parker" @0 {likeness: 75}] | + | 95 | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | 90 | [:like "Manu Ginobili"->"Tim Duncan" @0 {likeness: 90}] | + | 90 | [:like "Tony Parker"->"LaMarcus Aldridge" @0 {likeness: 90}] | + | 95 | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | 90 | [:like "Manu Ginobili"->"Tim Duncan" @0 {likeness: 90}] | Scenario: Integer Vid error message When executing query: diff --git a/tests/tck/features/go/GO.feature b/tests/tck/features/go/GO.feature index 6f46375ec5e..7eef941ae94 100644 --- a/tests/tck/features/go/GO.feature +++ b/tests/tck/features/go/GO.feature @@ -1490,6 +1490,45 @@ Feature: Go Sentence | EMPTY | "Russell Westbrook" | | EMPTY | "Luka Doncic" | | EMPTY | "Russell Westbrook" | + When executing query: + """ + go 1 to 4 steps from "Tim Duncan" over like where like.likeness > 90 yield like.likeness, edge as e + """ + Then the result should be, in any order, with relax comparison: + | like.likeness | e | + | 95 | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | 95 | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | 95 | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + When executing query: + """ + go 1 to 4 steps from "Tim Duncan" over like yield like.likeness, edge as e + """ + Then the result should be, in any order, with relax comparison: + | like.likeness | e | + | 95 | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | 90 | [:like "Manu Ginobili"->"Tim Duncan" @0 {likeness: 90}] | + | 90 | [:like "Tony Parker"->"LaMarcus Aldridge" @0 {likeness: 90}] | + | 95 | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | + | 75 | [:like "LaMarcus Aldridge"->"Tim Duncan" @0 {likeness: 75}] | + | 75 | [:like "LaMarcus Aldridge"->"Tony Parker" @0 {likeness: 75}] | + | 95 | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | 90 | [:like "Manu Ginobili"->"Tim Duncan" @0 {likeness: 90}] | + | 90 | [:like "Tony Parker"->"LaMarcus Aldridge" @0 {likeness: 90}] | + | 95 | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | 95 | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | 90 | [:like "Manu Ginobili"->"Tim Duncan" @0 {likeness: 90}] | Scenario: error message When executing query: diff --git a/tests/tck/features/go/GoYieldVertexEdge.feature b/tests/tck/features/go/GoYieldVertexEdge.feature index 4075300fbbf..92d920a6650 100644 --- a/tests/tck/features/go/GoYieldVertexEdge.feature +++ b/tests/tck/features/go/GoYieldVertexEdge.feature @@ -8,6 +8,14 @@ Feature: Go Yield Vertex And Edge Sentence Given a graph with space named "nba" Scenario: one step + When executing query: + """ + GO FROM "Tim Duncan" OVER like YIELD edge as e, properties(edge) as props, concat(src(edge), " like ", dst(edge), " @ ", properties($$).name, " # ", properties($^).age) as result + """ + Then the result should be, in any order, with relax comparison: + | e | props | result | + | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | {likeness: 95} | "Tim Duncan like Manu Ginobili @ Manu Ginobili # 42" | + | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | {likeness: 95} | "Tim Duncan like Tony Parker @ Tony Parker # 42" | When executing query: """ GO FROM "Tim Duncan" OVER serve Yield src(edge) as src, dst(edge) as dst, type(edge) as type, edge as e @@ -1409,6 +1417,23 @@ Feature: Go Yield Vertex And Edge Sentence | "James Harden" | ("James Harden" :player{age: 29, name: "James Harden"}) | "like" | | "James Harden" | ("James Harden" :player{age: 29, name: "James Harden"}) | "like" | | "Paul George" | ("Paul George" :player{age: 28, name: "Paul George"}) | "like" | + When executing query: + """ + GO 1 to 3 steps FROM "Tim Duncan" OVER like YIELD edge as e, properties(edge) as props, concat(src(edge), " like ", dst(edge), " @ ", properties($$).name, " # ", properties($^).age) as result + """ + Then the result should be, in any order, with relax comparison: + | e | props | result | + | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | {likeness: 95} | "Tim Duncan like Manu Ginobili @ Manu Ginobili # 42" | + | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | {likeness: 95} | "Tim Duncan like Tony Parker @ Tony Parker # 42" | + | [:like "Manu Ginobili"->"Tim Duncan" @0 {likeness: 90}] | {likeness: 90} | "Manu Ginobili like Tim Duncan @ Tim Duncan # 41" | + | [:like "Tony Parker"->"LaMarcus Aldridge" @0 {likeness: 90}] | {likeness: 90} | "Tony Parker like LaMarcus Aldridge @ LaMarcus Aldridge # 36" | + | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | {likeness: 95} | "Tony Parker like Manu Ginobili @ Manu Ginobili # 36" | + | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | {likeness: 95} | "Tony Parker like Tim Duncan @ Tim Duncan # 36" | + | [:like "LaMarcus Aldridge"->"Tim Duncan" @0 {likeness: 75}] | {likeness: 75} | "LaMarcus Aldridge like Tim Duncan @ Tim Duncan # 33" | + | [:like "LaMarcus Aldridge"->"Tony Parker" @0 {likeness: 75}] | {likeness: 75} | "LaMarcus Aldridge like Tony Parker @ Tony Parker # 33" | + | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | {likeness: 95} | "Tim Duncan like Manu Ginobili @ Manu Ginobili # 42" | + | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | {likeness: 95} | "Tim Duncan like Tony Parker @ Tony Parker # 42" | + | [:like "Manu Ginobili"->"Tim Duncan" @0 {likeness: 90}] | {likeness: 90} | "Manu Ginobili like Tim Duncan @ Tim Duncan # 41" | Scenario: error message When executing query: diff --git a/tests/tck/features/lookup/EdgeIndexFullScan.feature b/tests/tck/features/lookup/EdgeIndexFullScan.feature index d07348979c3..d5aae560e51 100644 --- a/tests/tck/features/lookup/EdgeIndexFullScan.feature +++ b/tests/tck/features/lookup/EdgeIndexFullScan.feature @@ -26,11 +26,11 @@ Feature: Lookup edge index full scan And having executed: """ INSERT EDGE - edge_1(col1_str, col2_int) + edge_1(col1_str, col2_int) VALUES - '101'->'102':('Red1', 11), - '102'->'103':('Yellow', 22), - '103'->'101':('Blue', 33); + '101'->'102':('Red1', 11), + '102'->'103':('Yellow', 22), + '103'->'101':('Blue', 33); """ Scenario: Edge with relational RegExp filter @@ -147,10 +147,11 @@ Feature: Lookup edge index full scan | SrcVID | DstVID | Ranking | edge_1.col2_int | | "101" | "102" | 0 | 11 | And the execution plan should be: - | id | name | dependencies | operator info | - | 3 | Project | 4 | | - | 4 | EdgeIndexPrefixScan | 0 | | - | 0 | Start | | | + | id | name | dependencies | operator info | + | 3 | Project | 2 | | + | 2 | Filter | 4 | | + | 4 | EdgeIndexFullScan | 0 | | + | 0 | Start | | | Scenario: Edge with complex relational IN filter # (a IN b) AND (c IN d) diff --git a/tests/tck/features/ttl/TTL.feature b/tests/tck/features/ttl/TTL.feature index ffd176b79f4..aebc2ab1b8e 100644 --- a/tests/tck/features/ttl/TTL.feature +++ b/tests/tck/features/ttl/TTL.feature @@ -326,6 +326,50 @@ Feature: TTLTest Then the result should be, in any order: | Edge | Create Edge | | "work2" | 'CREATE EDGE `work2` (\n `email` string NULL,\n `age` string NULL,\n `gender` string NULL\n) ttl_duration = 0, ttl_col = ""' | + When executing query: + """ + CREATE EDGE player(id int, name string, age int, address string, score float); + """ + Then the execution should be successful + When executing query: + """ + SHOW CREATE EDGE player; + """ + Then the result should be, in any order: + | Edge | Create Edge | + | "player" | 'CREATE EDGE `player` (\n `id` int64 NULL,\n `name` string NULL,\n `age` int64 NULL,\n `address` string NULL,\n `score` float NULL\n) ttl_duration = 0, ttl_col = ""' | + When executing query: + """ + ALTER EDGE player change(name int), drop(name); + """ + Then a SemanticError should be raised at runtime: Duplicate column name `name' + When executing query: + """ + ALTER EDGE player drop(name), change(name int); + """ + Then a SemanticError should be raised at runtime: Duplicate column name `name' + When executing query: + """ + ALTER EDGE player drop(name, name), change(address int); + """ + Then a SemanticError should be raised at runtime: Duplicate column name `name' + When executing query: + """ + ALTER EDGE player change(address int, address string); + """ + Then a SemanticError should be raised at runtime: Duplicate column name `address' + When executing query: + """ + ALTER EDGE player change(address int), drop(name); + """ + Then the execution should be successful + When executing query: + """ + SHOW CREATE EDGE player; + """ + Then the result should be, in any order: + | Edge | Create Edge | + | "player" | 'CREATE EDGE `player` (\n `id` int64 NULL,\n `age` int64 NULL,\n `address` int64 NULL,\n `score` float NULL\n) ttl_duration = 0, ttl_col = ""' | And drop the used space Scenario: TTLTest Datatest