Skip to content

Commit

Permalink
support s2 index params
Browse files Browse the repository at this point in the history
  • Loading branch information
@jievince
jievince committed Dec 8, 2021
1 parent eb9ddf3 commit 1944639
Show file tree
Hide file tree
Showing 34 changed files with 599 additions and 226 deletions.
28 changes: 4 additions & 24 deletions src/clients/meta/MetaClient.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1712,24 +1712,14 @@ folly::Future<StatusOr<IndexID>> MetaClient::createTagIndex(GraphSpaceID spaceID
std::string tagName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists,
const std::string* comment,
const int* s2_max_level,
const int* s2_max_cells) {
cpp2::IndexParams indexParams) {
cpp2::CreateTagIndexReq req;
req.set_space_id(spaceID);
req.set_index_name(std::move(indexName));
req.set_tag_name(std::move(tagName));
req.set_fields(std::move(fields));
req.set_if_not_exists(ifNotExists);
if (comment != nullptr) {
req.set_comment(*comment);
}
if (s2_max_level != nullptr) {
req.set_s2_max_level(*s2_max_level);
}
if (s2_max_cells != nullptr) {
req.set_s2_max_cells(*s2_max_cells);
}
req.set_index_params(std::move(indexParams));

folly::Promise<StatusOr<IndexID>> promise;
auto future = promise.getFuture();
Expand Down Expand Up @@ -1832,24 +1822,14 @@ folly::Future<StatusOr<IndexID>> MetaClient::createEdgeIndex(
std::string edgeName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists,
const std::string* comment,
const int* s2_max_level,
const int* s2_max_cells) {
cpp2::IndexParams indexParams) {
cpp2::CreateEdgeIndexReq req;
req.set_space_id(spaceID);
req.set_index_name(std::move(indexName));
req.set_edge_name(std::move(edgeName));
req.set_fields(std::move(fields));
req.set_if_not_exists(ifNotExists);
if (comment != nullptr) {
req.set_comment(*comment);
}
if (s2_max_level != nullptr) {
req.set_s2_max_level(*s2_max_level);
}
if (s2_max_cells != nullptr) {
req.set_s2_max_cells(*s2_max_cells);
}
req.set_index_params(std::move(indexParams));

folly::Promise<StatusOr<IndexID>> promise;
auto future = promise.getFuture();
Expand Down
30 changes: 14 additions & 16 deletions src/clients/meta/MetaClient.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,14 +311,13 @@ class MetaClient {
bool ifExists = false);

// Operations for index
folly::Future<StatusOr<IndexID>> createTagIndex(GraphSpaceID spaceID,
std::string indexName,
std::string tagName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists = false,
const std::string* comment = nullptr,
const int* s2_max_level = nullptr,
const int* s2_max_cells = nullptr);
folly::Future<StatusOr<IndexID>> createTagIndex(
GraphSpaceID spaceID,
std::string indexName,
std::string tagName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists = false,
meta::cpp2::IndexParams indexParams = cpp2::IndexParams{});

// Remove the define of tag index
folly::Future<StatusOr<bool>> dropTagIndex(GraphSpaceID spaceId,
Expand All @@ -333,14 +332,13 @@ class MetaClient {

folly::Future<StatusOr<std::vector<cpp2::IndexStatus>>> listTagIndexStatus(GraphSpaceID spaceId);

folly::Future<StatusOr<IndexID>> createEdgeIndex(GraphSpaceID spaceID,
std::string indexName,
std::string edgeName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists = false,
const std::string* comment = nullptr,
const int* s2_max_level = nullptr,
const int* s2_max_cells = nullptr);
folly::Future<StatusOr<IndexID>> createEdgeIndex(
GraphSpaceID spaceID,
std::string indexName,
std::string edgeName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists = false,
cpp2::IndexParams indexParams = cpp2::IndexParams{});

// Remove the definition of edge index
folly::Future<StatusOr<bool>> dropEdgeIndex(GraphSpaceID spaceId,
Expand Down
2 changes: 1 addition & 1 deletion src/common/datatypes/Geography.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ struct Geography {
bool needNormalize = false,
bool verifyValidity = false);

Geography() {}
Geography() = default;
Geography(const Point& v) : geo_(v) {} // NOLINT
Geography(Point&& v) : geo_(std::move(v)) {} // NOLINT
Geography(const LineString& v) : geo_(v) {} // NOLINT
Expand Down
12 changes: 11 additions & 1 deletion src/common/geo/GeoIndex.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,17 @@
namespace nebula {
namespace geo {

nebula::storage::cpp2::IndexColumnHint ScanRange::toIndexColumnHint() {
bool ScanRange::operator==(const ScanRange& rhs) const {
if (isRangeScan != rhs.isRangeScan) {
return false;
}
if (isRangeScan) {
return rangeMin == rhs.rangeMin && rangeMax == rhs.rangeMax;
}
return rangeMin == rhs.rangeMin;
}

nebula::storage::cpp2::IndexColumnHint ScanRange::toIndexColumnHint() const {
nebula::storage::cpp2::IndexColumnHint hint;
// column_name should be set by the caller
if (isRangeScan) {
Expand Down
5 changes: 4 additions & 1 deletion src/common/geo/GeoIndex.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@ struct ScanRange {

explicit ScanRange(uint64_t v) : rangeMin(v), isRangeScan(false) {}

nebula::storage::cpp2::IndexColumnHint toIndexColumnHint();
bool operator==(const ScanRange& rhs) const;

nebula::storage::cpp2::IndexColumnHint toIndexColumnHint() const;
};

class GeoIndex {
Expand Down Expand Up @@ -82,6 +84,7 @@ class GeoIndex {

private:
RegionCoverParams rcParams_;
// pointsOnly_ indicates whether the indexed column only has points.
// For the column Geography(Point), we don't need to build ancestor cells
bool pointsOnly_{false};
};
Expand Down
161 changes: 154 additions & 7 deletions src/common/geo/test/GeoIndexTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

#include <gtest/gtest.h>

#include <cstdint>
#include <unordered_set>

#include "common/base/Base.h"
Expand All @@ -13,22 +14,23 @@
namespace nebula {
namespace geo {

std::vector<uint64_t> toUint64Vector(std::vector<int64_t> expect) {
auto reinterpretInt64AsUint64 = [](int64_t i) -> uint64_t {
const char* c = reinterpret_cast<const char*>(&i);
uint64_t u = *reinterpret_cast<const uint64_t*>(c);
return u;
};
uint64_t asUint64(int64_t i) {
const char* c = reinterpret_cast<const char*>(&i);
uint64_t u = *reinterpret_cast<const uint64_t*>(c);
return u;
}

std::vector<uint64_t> toUint64Vector(std::vector<int64_t> expect) {
std::vector<uint64_t> transformedExpect;
transformedExpect.reserve(expect.size());
for (int64_t i : expect) {
transformedExpect.push_back(reinterpretInt64AsUint64(i));
transformedExpect.push_back(asUint64(i));
}
return transformedExpect;
}

// The tested wkt data is generated by https://clydedacruz.github.io/openstreetmap-wkt-playground/
// And the expected result comes from BigQuery
TEST(indexCells, point) {
geo::RegionCoverParams rc;
geo::GeoIndex geoIndex(rc);
Expand Down Expand Up @@ -105,6 +107,7 @@ TEST(indexCells, lineString) {
EXPECT_EQ(toUint64Vector(expect), cells);
}
}

TEST(indexCells, polygon) {
geo::RegionCoverParams rc;
geo::GeoIndex geoIndex(rc);
Expand Down Expand Up @@ -172,6 +175,150 @@ TEST(indexCells, polygon) {
}
}

TEST(indexCells, tunningRegionCoverParamsForPoint) {
// TODO(jie)
}
TEST(indexCells, tunningRegionCoverParamsForLineString) {
// TODO(jie)
}
TEST(indexCells, tunningRegionCoverParamsForPolygon) {
// TODO(jie)
}

TEST(intersects, point) {
geo::RegionCoverParams rc(0, 30, 8);
{
bool pointsOnly = false;
geo::GeoIndex geoIndex(rc, pointsOnly);
auto point = Geography::fromWKT("POINT(1.0 1.0)").value();
auto got = geoIndex.intersects(point);

std::vector<ScanRange> expect;
auto cells = toUint64Vector({1153277837650709461});
expect.reserve(cells.size());
for (uint64_t cellId : cells) {
expect.emplace_back(cellId);
}

EXPECT_EQ(expect, got);
}
{
bool pointsOnly = false; // The indexed geo column only has points;
geo::GeoIndex geoIndex(rc, pointsOnly);
auto point = Geography::fromWKT("POINT(1.0 1.0)").value();
auto got = geoIndex.intersects(point);

std::vector<ScanRange> expect;
auto cells = toUint64Vector({1153277837650709461});
expect.reserve(cells.size());
for (uint64_t cellId : cells) {
expect.emplace_back(cellId);
}

EXPECT_EQ(expect, got);
}
}

TEST(intersects, lineString) {
geo::RegionCoverParams rc(0, 30, 8);
{
bool pointsOnly = false;
geo::GeoIndex geoIndex(rc, pointsOnly);
auto line = Geography::fromWKT("LINESTRING(68.9 48.9,76.1 35.5,125.7 28.2)").value();
auto got = geoIndex.intersects(line);

std::vector<ScanRange> expect;
auto cells = toUint64Vector({3765009288481734656,
3818771009033469952,
3909124476557590528,
3928264774973915136,
4017210867614482432,
4053239664633446400,
4089268461652410368,
4773815605012725760});
expect.reserve(cells.size());
for (uint64_t cellId : cells) {
expect.emplace_back(cellId);
}

EXPECT_EQ(expect, got);
}
{
bool pointsOnly = true; // The indexed geo column only has points;
geo::GeoIndex geoIndex(rc, pointsOnly);
auto line = Geography::fromWKT("POINT(1.0 1.0)").value();
auto got = geoIndex.intersects(line);

std::vector<ScanRange> expect;
auto cells = toUint64Vector({3765009288481734656,
3818771009033469952,
3909124476557590528,
3928264774973915136,
4017210867614482432,
4053239664633446400,
4089268461652410368,
4773815605012725760});
expect.reserve(cells.size());
for (uint64_t cellId : cells) {
expect.emplace_back(cellId);
}

EXPECT_EQ(expect, got);
}
}

TEST(intersects, polygon) {
geo::RegionCoverParams rc(0, 30, 8);
{
bool pointsOnly = false;
geo::GeoIndex geoIndex(rc, pointsOnly);
auto point =
Geography::fromWKT("POLYGON((102.5 33.5, 110.6 36.9,113.6 30.4,102.7 27.3,102.5 33.5))")
.value();
auto got = geoIndex.intersects(point);

std::vector<ScanRange> expect;
auto cells = toUint64Vector({3759379788947521536,
3879094614979772416,
3915809507254468608,
3917005775905488896,
3922635275439702016,
3931642474694443008,
3949656873203924992,
3958664072458665984});
expect.reserve(cells.size());
for (uint64_t cellId : cells) {
expect.emplace_back(cellId);
}

EXPECT_EQ(expect, got);
}
{
bool pointsOnly = false; // The indexed geo column only has points;
geo::GeoIndex geoIndex(rc, pointsOnly);
auto point =
Geography::fromWKT("POLYGON((102.5 33.5, 110.6 36.9,113.6 30.4,102.7 27.3,102.5 33.5))")
.value();
auto got = geoIndex.intersects(point);

std::vector<ScanRange> expect;
auto cells = toUint64Vector({3759379788947521536,
3879094614979772416,
3915809507254468608,
3917005775905488896,
3922635275439702016,
3931642474694443008,
3949656873203924992,
3958664072458665984});
expect.reserve(cells.size());
for (uint64_t cellId : cells) {
expect.emplace_back(cellId);
}

EXPECT_EQ(expect, got);
}
}

} // namespace geo
} // namespace nebula

Expand Down
12 changes: 10 additions & 2 deletions src/common/utils/IndexKeyUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

#include <thrift/lib/cpp2/protocol/Serializer.h>

#include "common/geo/GeoIndex.h"

namespace nebula {

// static
Expand Down Expand Up @@ -53,8 +55,14 @@ std::vector<std::string> IndexKeyUtils::encodeValues(std::vector<Value>&& values
const auto& value = values.back();
if (!value.isNull()) {
DCHECK(value.type() == Value::Type::GEOGRAPHY);
indexes = encodeGeography(
value.getGeography(), indexItem->get_s2_max_level(), indexItem->get_s2_max_cells());
geo::RegionCoverParams rc;
if (indexItem->get_s2_max_level()) {
rc.maxCellLevel_ = *indexItem->get_s2_max_level();
}
if (indexItem->get_s2_max_cells()) {
rc.maxCellNum_ = *indexItem->get_s2_max_level();
}
indexes = encodeGeography(value.getGeography(), rc);
} else {
nullableBitSet |= 0x8000;
auto type = IndexKeyUtils::toValueType(cols.back().type.get_type());
Expand Down
Loading

0 comments on commit 1944639

Please sign in to comment.