Skip to content

Commit

Permalink
support s2 index params
Browse files Browse the repository at this point in the history
  • Loading branch information
@jievince
jievince committed Dec 6, 2021
1 parent dd06a53 commit eb9ddf3
Show file tree
Hide file tree
Showing 29 changed files with 340 additions and 88 deletions.
20 changes: 18 additions & 2 deletions src/clients/meta/MetaClient.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1712,7 +1712,9 @@ folly::Future<StatusOr<IndexID>> MetaClient::createTagIndex(GraphSpaceID spaceID
std::string tagName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists,
const std::string* comment) {
const std::string* comment,
const int* s2_max_level,
const int* s2_max_cells) {
cpp2::CreateTagIndexReq req;
req.set_space_id(spaceID);
req.set_index_name(std::move(indexName));
Expand All @@ -1722,6 +1724,12 @@ folly::Future<StatusOr<IndexID>> MetaClient::createTagIndex(GraphSpaceID spaceID
if (comment != nullptr) {
req.set_comment(*comment);
}
if (s2_max_level != nullptr) {
req.set_s2_max_level(*s2_max_level);
}
if (s2_max_cells != nullptr) {
req.set_s2_max_cells(*s2_max_cells);
}

folly::Promise<StatusOr<IndexID>> promise;
auto future = promise.getFuture();
Expand Down Expand Up @@ -1824,7 +1832,9 @@ folly::Future<StatusOr<IndexID>> MetaClient::createEdgeIndex(
std::string edgeName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists,
const std::string* comment) {
const std::string* comment,
const int* s2_max_level,
const int* s2_max_cells) {
cpp2::CreateEdgeIndexReq req;
req.set_space_id(spaceID);
req.set_index_name(std::move(indexName));
Expand All @@ -1834,6 +1844,12 @@ folly::Future<StatusOr<IndexID>> MetaClient::createEdgeIndex(
if (comment != nullptr) {
req.set_comment(*comment);
}
if (s2_max_level != nullptr) {
req.set_s2_max_level(*s2_max_level);
}
if (s2_max_cells != nullptr) {
req.set_s2_max_cells(*s2_max_cells);
}

folly::Promise<StatusOr<IndexID>> promise;
auto future = promise.getFuture();
Expand Down
8 changes: 6 additions & 2 deletions src/clients/meta/MetaClient.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,9 @@ class MetaClient {
std::string tagName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists = false,
const std::string* comment = nullptr);
const std::string* comment = nullptr,
const int* s2_max_level = nullptr,
const int* s2_max_cells = nullptr);

// Remove the define of tag index
folly::Future<StatusOr<bool>> dropTagIndex(GraphSpaceID spaceId,
Expand All @@ -336,7 +338,9 @@ class MetaClient {
std::string edgeName,
std::vector<cpp2::IndexFieldDef> fields,
bool ifNotExists = false,
const std::string* comment = nullptr);
const std::string* comment = nullptr,
const int* s2_max_level = nullptr,
const int* s2_max_cells = nullptr);

// Remove the definition of edge index
folly::Future<StatusOr<bool>> dropEdgeIndex(GraphSpaceID spaceId,
Expand Down
13 changes: 8 additions & 5 deletions src/common/utils/IndexKeyUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@
namespace nebula {

// static
std::vector<std::string> IndexKeyUtils::encodeValues(
std::vector<Value>&& values, const std::vector<nebula::meta::cpp2::ColumnDef>& cols) {
std::vector<std::string> IndexKeyUtils::encodeValues(std::vector<Value>&& values,
const meta::cpp2::IndexItem* indexItem) {
auto& cols = indexItem->get_fields();
bool hasNullCol = false;
// An index has a maximum of 16 columns. 2 byte (16 bit) is enough.
u_short nullableBitSet = 0;
Expand Down Expand Up @@ -52,7 +53,8 @@ std::vector<std::string> IndexKeyUtils::encodeValues(
const auto& value = values.back();
if (!value.isNull()) {
DCHECK(value.type() == Value::Type::GEOGRAPHY);
indexes = encodeGeography(value.getGeography());
indexes = encodeGeography(
value.getGeography(), indexItem->get_s2_max_level(), indexItem->get_s2_max_cells());
} else {
nullableBitSet |= 0x8000;
auto type = IndexKeyUtils::toValueType(cols.back().type.get_type());
Expand Down Expand Up @@ -157,10 +159,11 @@ Value IndexKeyUtils::parseIndexTTL(const folly::StringPiece& raw) {

// static
StatusOr<std::vector<std::string>> IndexKeyUtils::collectIndexValues(
RowReader* reader, const std::vector<nebula::meta::cpp2::ColumnDef>& cols) {
RowReader* reader, const meta::cpp2::IndexItem* indexItem) {
if (reader == nullptr) {
return Status::Error("Invalid row reader");
}
auto& cols = indexItem->get_fields();
std::vector<Value> values;
for (const auto& col : cols) {
auto v = reader->getValueByName(col.get_name());
Expand All @@ -172,7 +175,7 @@ StatusOr<std::vector<std::string>> IndexKeyUtils::collectIndexValues(
}
values.emplace_back(std::move(v));
}
return encodeValues(std::move(values), cols);
return encodeValues(std::move(values), indexItem);
}

// static
Expand Down
21 changes: 14 additions & 7 deletions src/common/utils/IndexKeyUtils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,13 +300,20 @@ class IndexKeyUtils final {
return buf;
}

static std::vector<std::string> encodeGeography(const nebula::Geography& gg) {
// TODO(jie): Get index params from meta to construct RegionCoverParams
static std::vector<std::string> encodeGeography(const nebula::Geography& gg,
const int* s2_max_level,
const int* s2_max_cells) {
geo::RegionCoverParams rc;
// TODO(jie): Get schema meta to know if it's point only
geo::GeoIndex geoIndex(rc, false);
if (!s2_max_level) {
rc.maxCellLevel_ = *s2_max_level;
}
if (!s2_max_cells) {
rc.maxCellNum_ = *s2_max_cells;
}
geo::GeoIndex geoIndex(rc);
auto cellIds = geoIndex.indexCells(gg);
std::vector<std::string> bufs;
bufs.reserve(cellIds.size());
for (auto cellId : cellIds) {
bufs.emplace_back(encodeUint64(cellId));
}
Expand Down Expand Up @@ -494,8 +501,8 @@ class IndexKeyUtils final {
/**
* Generate vertex|edge index key for kv store
**/
static std::vector<std::string> encodeValues(
std::vector<Value>&& values, const std::vector<nebula::meta::cpp2::ColumnDef>& cols);
static std::vector<std::string> encodeValues(std::vector<Value>&& values,
const meta::cpp2::IndexItem* indexItem);

/**
* param valueTypes ： column type of each index column. If there are no
Expand Down Expand Up @@ -528,7 +535,7 @@ class IndexKeyUtils final {
static Value parseIndexTTL(const folly::StringPiece& raw);

static StatusOr<std::vector<std::string>> collectIndexValues(
RowReader* reader, const std::vector<nebula::meta::cpp2::ColumnDef>& cols);
RowReader* reader, const meta::cpp2::IndexItem* indexItem);

private:
IndexKeyUtils() = delete;
Expand Down
4 changes: 3 additions & 1 deletion src/graph/executor/maintain/EdgeIndexExecutor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ folly::Future<Status> CreateEdgeIndexExecutor::execute() {
ceiNode->getSchemaName(),
ceiNode->getFields(),
ceiNode->getIfNotExists(),
ceiNode->getComment())
ceiNode->getComment(),
ceiNode->getS2MaxLevel(),
ceiNode->getS2MaxCells())
.via(runner())
.thenValue([ceiNode, spaceId](StatusOr<IndexID> resp) {
if (!resp.ok()) {
Expand Down
4 changes: 3 additions & 1 deletion src/graph/executor/maintain/TagIndexExecutor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ folly::Future<Status> CreateTagIndexExecutor::execute() {
ctiNode->getSchemaName(),
ctiNode->getFields(),
ctiNode->getIfNotExists(),
ctiNode->getComment())
ctiNode->getComment(),
ctiNode->getS2MaxLevel(),
ctiNode->getS2MaxCells())
.via(runner())
.thenValue([ctiNode, spaceId](StatusOr<IndexID> resp) {
if (!resp.ok()) {
Expand Down
22 changes: 16 additions & 6 deletions src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,23 @@ StatusOr<TransformResult> GeoPredicateIndexScanBaseRule::transform(
DCHECK(secondVal.isGeography());
const auto& geog = secondVal.getGeography();

// TODO(jie): Get index params from meta to construct RegionCoverParams
auto indexItem = indexItems.back();
const auto& fields = indexItem->get_fields();
DCHECK_EQ(fields.size(), 1); // geo field
auto& geoField = fields.back();
auto& geoColumnTypeDef = geoField.get_type();
bool indexedGeoColumnOnlyHasPoints =
geoColumnTypeDef.geo_shape_ref().has_value() &&
geoColumnTypeDef.geo_shape_ref().value() == meta::cpp2::GeoShape::POINT;

geo::RegionCoverParams rc;
// TODO(jie): Get schema meta to know if it's point only
geo::GeoIndex geoIndex(rc, false);
if (indexItem.s2_max_level_ref().has_value()) {
rc.maxCellLevel_ = indexItem.s2_max_level_ref().value();
}
if (indexItem.s2_max_cells_ref().has_value()) {
rc.maxCellNum_ = indexItem.s2_max_cells_ref().value();
}
geo::GeoIndex geoIndex(rc, indexedGeoColumnOnlyHasPoints);
std::vector<geo::ScanRange> scanRanges;
if (geoPredicateName == "st_intersects") {
scanRanges = geoIndex.intersects(geog);
Expand All @@ -102,9 +115,6 @@ StatusOr<TransformResult> GeoPredicateIndexScanBaseRule::transform(
}
std::vector<IndexQueryContext> idxCtxs;
idxCtxs.reserve(scanRanges.size());
auto indexItem = indexItems.back();
const auto& fields = indexItem->get_fields();
DCHECK_EQ(fields.size(), 1); // geo field
auto fieldName = fields.back().get_name();
for (auto& scanRange : scanRanges) {
IndexQueryContext ictx;
Expand Down
3 changes: 3 additions & 0 deletions src/graph/planner/plan/Maintain.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ std::unique_ptr<PlanNodeDescription> CreateIndexNode::explain() const {
}
addDescription("fields", folly::toJson(util::toJson(fields)), desc.get());
addDescription("ifNotExists", folly::to<std::string>(ifNotExists_), desc.get());
addDescription("comment", folly::toJson(util::toJson(*comment_)), desc.get());
addDescription("s2_max_level", folly::toJson(util::toJson(*s2MaxLevel_)), desc.get());
addDescription("s2_max_cells", folly::toJson(util::toJson(*s2MaxCells_)), desc.get());
return desc;
}

Expand Down
44 changes: 34 additions & 10 deletions src/graph/planner/plan/Maintain.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,13 +300,17 @@ class CreateIndexNode : public SingleDependencyNode {
std::string indexName,
std::vector<meta::cpp2::IndexFieldDef> fields,
bool ifNotExists,
const std::string* comment)
const std::string* comment,
const int* s2MaxLevel,
const int* s2MaxCells)
: SingleDependencyNode(qctx, kind, input),
schemaName_(std::move(schemaName)),
indexName_(std::move(indexName)),
fields_(std::move(fields)),
ifNotExists_(ifNotExists),
comment_(comment) {}
comment_(comment),
s2MaxLevel_(s2MaxLevel),
s2MaxCells_(s2MaxCells) {}

public:
const std::string& getSchemaName() const { return schemaName_; }
Expand All @@ -318,6 +322,8 @@ class CreateIndexNode : public SingleDependencyNode {
bool getIfNotExists() const { return ifNotExists_; }

const std::string* getComment() const { return comment_; }
const int* getS2MaxLevel() const { return s2MaxLevel_; }
const int* getS2MaxCells() const { return s2MaxCells_; }

std::unique_ptr<PlanNodeDescription> explain() const override;

Expand All @@ -327,6 +333,8 @@ class CreateIndexNode : public SingleDependencyNode {
std::vector<meta::cpp2::IndexFieldDef> fields_;
bool ifNotExists_;
const std::string* comment_;
const int* s2MaxLevel_;
const int* s2MaxCells_;
};

class CreateTagIndex final : public CreateIndexNode {
Expand All @@ -337,14 +345,18 @@ class CreateTagIndex final : public CreateIndexNode {
std::string indexName,
std::vector<meta::cpp2::IndexFieldDef> fields,
bool ifNotExists,
const std::string* comment) {
const std::string* comment,
const int* s2MaxLevel,
const int* s2MaxCells) {
return qctx->objPool()->add(new CreateTagIndex(qctx,
input,
std::move(tagName),
std::move(indexName),
std::move(fields),
ifNotExists,
comment));
comment,
s2MaxLevel,
s2MaxCells));
}

private:
Expand All @@ -354,15 +366,19 @@ class CreateTagIndex final : public CreateIndexNode {
std::string indexName,
std::vector<meta::cpp2::IndexFieldDef> fields,
bool ifNotExists,
const std::string* comment)
const std::string* comment,
const int* s2MaxLevel,
const int* s2MaxCells)
: CreateIndexNode(qctx,
input,
Kind::kCreateTagIndex,
std::move(tagName),
std::move(indexName),
std::move(fields),
ifNotExists,
comment) {}
comment,
s2MaxLevel,
s2MaxCells) {}
};

class CreateEdgeIndex final : public CreateIndexNode {
Expand All @@ -373,14 +389,18 @@ class CreateEdgeIndex final : public CreateIndexNode {
std::string indexName,
std::vector<meta::cpp2::IndexFieldDef> fields,
bool ifNotExists,
const std::string* comment) {
const std::string* comment,
const int* s2MaxLevel,
const int* s2MaxCells) {
return qctx->objPool()->add(new CreateEdgeIndex(qctx,
input,
std::move(edgeName),
std::move(indexName),
std::move(fields),
ifNotExists,
comment));
comment,
s2MaxLevel,
s2MaxCells));
}

private:
Expand All @@ -390,15 +410,19 @@ class CreateEdgeIndex final : public CreateIndexNode {
std::string indexName,
std::vector<meta::cpp2::IndexFieldDef> fields,
bool ifNotExists,
const std::string* comment)
const std::string* comment,
const int* s2MaxLevel,
const int* s2MaxCells)
: CreateIndexNode(qctx,
input,
Kind::kCreateEdgeIndex,
std::move(edgeName),
std::move(indexName),
std::move(fields),
ifNotExists,
comment) {}
comment,
s2MaxLevel,
s2MaxCells) {}
};

class DescIndexNode : public SingleDependencyNode {
Expand Down
16 changes: 14 additions & 2 deletions src/graph/util/IndexUtil.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

#include "graph/util/IndexUtil.h"

#include <string>

namespace nebula {
namespace graph {

Expand Down Expand Up @@ -63,8 +65,18 @@ StatusOr<DataSet> IndexUtil::toShowCreateIndex(bool isTagIndex,
}
createStr += ")";
if (indexItem.comment_ref().has_value()) {
createStr += " comment = \"";
createStr += *indexItem.get_comment();
createStr += ", comment = \"";
createStr += *indexItem.comment_ref();
createStr += "\"";
}
if (indexItem.s2_max_level_ref().has_value()) {
createStr += ", s2_max_level = \"";
createStr += std::to_string(*indexItem.s2_max_level_ref());
createStr += "\"";
}
if (indexItem.s2_max_cells_ref().has_value()) {
createStr += ", s2_max_cells = \"";
createStr += std::to_string(*indexItem.s2_max_cells_ref());
createStr += "\"";
}
row.emplace_back(std::move(createStr));
Expand Down
Loading

0 comments on commit eb9ddf3

Please sign in to comment.