Skip to content

Commit

Permalink
address yee's comment
Browse files Browse the repository at this point in the history
  • Loading branch information
@jievince
jievince committed Oct 19, 2021
1 parent 8263816 commit 7b1b40a
Show file tree
Hide file tree
Showing 27 changed files with 481 additions and 813 deletions.
4 changes: 2 additions & 2 deletions src/common/geo/GeoIndex.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ nebula::storage::cpp2::IndexColumnHint ScanRange::toIndexColumnHint() {
// column_name should be set by the caller
if (isRangeScan) {
hint.set_scan_type(nebula::storage::cpp2::ScanType::RANGE);
hint.set_begin_value(
IndexKeyUtils::encodeUint64(rangeMin)); // Encode uint64_t as string in advance
// Encode uint64_t as string in advance
hint.set_begin_value(IndexKeyUtils::encodeUint64(rangeMin));
hint.set_end_value(IndexKeyUtils::encodeUint64(rangeMax));
} else {
hint.set_scan_type(nebula::storage::cpp2::ScanType::PREFIX);
Expand Down
4 changes: 2 additions & 2 deletions src/common/geo/GeoIndex.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@ class GeoIndex {

private:
RegionCoverParams rcParams_;
bool pointsOnly_{
false}; // For the column Geography(Point), we don't need to build ancestor cells
// For the column Geography(Point), we don't need to build ancestor cells
bool pointsOnly_{false};
};

} // namespace geo
Expand Down
11 changes: 6 additions & 5 deletions src/common/geo/GeoUtils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,11 @@ class GeoUtils final {
std::vector<std::unique_ptr<S2Loop>> s2Loops;
s2Loops.reserve(numCoordList);
for (const auto& coordList : polygon.coordListList) {
auto s2Points = s2PointsFromCoordinateList(
coordList, true); // S2 doesn't need the redundant last point
// S2 doesn't need the redundant last point
auto s2Points = s2PointsFromCoordinateList(coordList, true);
auto s2Loop = std::make_unique<S2Loop>(std::move(s2Points), S2Debug::DISABLE);
s2Loop->Normalize(); // All loops must be oriented CCW(counterclockwise) for S2
// All loops must be oriented CCW(counterclockwise) for S2
s2Loop->Normalize();
s2Loops.emplace_back(std::move(s2Loop));
}
return std::make_unique<S2Polygon>(std::move(s2Loops), S2Debug::DISABLE);
Expand All @@ -53,8 +54,8 @@ class GeoUtils final {
}

static S2Point s2PointFromCoordinate(const Coordinate& coord) {
auto latlng = S2LatLng::FromDegrees(
coord.y, coord.x); // Note: S2Point requires latitude to be first, and longitude to be last
// Note: S2Point requires latitude to be first, and longitude to be last
auto latlng = S2LatLng::FromDegrees(coord.y, coord.x);
return latlng.ToPoint();
}

Expand Down
163 changes: 51 additions & 112 deletions src/common/utils/IndexKeyUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,88 +11,69 @@
namespace nebula {

// static
std::string IndexKeyUtils::encodeValues(std::vector<Value>&& values,
const std::vector<nebula::meta::cpp2::ColumnDef>& cols) {
std::vector<std::string> IndexKeyUtils::encodeValues(
std::vector<Value>&& values, const std::vector<nebula::meta::cpp2::ColumnDef>& cols) {
bool hasNullCol = false;
// An index has a maximum of 16 columns. 2 byte (16 bit) is enough.
u_short nullableBitSet = 0;
std::string index;
auto findGeo = [](const meta::cpp2::ColumnDef& col) {
return col.get_type().get_type() == meta::cpp2::PropertyType::GEOGRAPHY;
};
bool hasGeo = std::find_if(cols.begin(), cols.end(), findGeo) != cols.end();
DCHECK(!hasGeo || cols.size() == 1);
std::vector<std::string> indexes;

for (size_t i = 0; i < values.size(); i++) {
auto isNullable = cols[i].nullable_ref().value_or(false);
if (isNullable) {
hasNullCol = true;
}
if (!hasGeo) {
std::string index;
for (size_t i = 0; i < values.size(); i++) {
auto isNullable = cols[i].nullable_ref().value_or(false);
if (isNullable) {
hasNullCol = true;
}

if (!values[i].isNull()) {
// string index need to fill with '\0' if length is less than schema
if (cols[i].type.type == meta::cpp2::PropertyType::FIXED_STRING) {
auto len = static_cast<size_t>(*cols[i].type.get_type_length());
index.append(encodeValue(values[i], len));
if (!values[i].isNull()) {
// string index need to fill with '\0' if length is less than schema
if (cols[i].type.type == meta::cpp2::PropertyType::FIXED_STRING) {
auto len = static_cast<size_t>(*cols[i].type.get_type_length());
index.append(encodeValue(values[i], len));
} else {
index.append(encodeValue(values[i]));
}
} else {
index.append(encodeValue(values[i]));
nullableBitSet |= 0x8000 >> i;
auto type = IndexKeyUtils::toValueType(cols[i].type.get_type());
index.append(encodeNullValue(type, cols[i].type.get_type_length()));
}
}
indexes.emplace_back(std::move(index));
} else {
hasNullCol = cols.back().nullable_ref().value_or(false);
DCHECK_EQ(values.size(), 1);
const auto& value = values.back();
if (!value.isNull()) {
DCHECK(value.type() == Value::Type::GEOGRAPHY);
indexes = encodeGeography(value.getGeography());
} else {
nullableBitSet |= 0x8000 >> i;
auto type = IndexKeyUtils::toValueType(cols[i].type.get_type());
index.append(encodeNullValue(type, cols[i].type.get_type_length()));
nullableBitSet |= 0x8000;
auto type = IndexKeyUtils::toValueType(cols.back().type.get_type());
indexes.emplace_back(encodeNullValue(type, nullptr));
}
}
// if has nullable field, append nullableBitSet to the end
if (hasNullCol) {
index.append(reinterpret_cast<const char*>(&nullableBitSet), sizeof(u_short));
}
return index;
}

std::vector<std::string> IndexKeyUtils::encodeValueForGeography(
Value&& value, const nebula::meta::cpp2::ColumnDef& col) {
DCHECK(col.get_type().get_type() == meta::cpp2::PropertyType::GEOGRAPHY);
u_short nullableBitSet = 0;
std::vector<std::string> indexes;

auto hasNullCol = col.nullable_ref().value_or(false);

if (!value.isNull()) {
DCHECK(value.type() == Value::Type::GEOGRAPHY);
indexes = encodeGeography(value.getGeography());
} else {
nullableBitSet |= 0x8000;
auto type = IndexKeyUtils::toValueType(col.type.get_type());
indexes.emplace_back(encodeNullValue(type, nullptr));
}

if (hasNullCol) {
// if has nullable field, append nullableBitSet to the end
for (auto& index : indexes) {
index.append(reinterpret_cast<const char*>(&nullableBitSet), sizeof(u_short));
}
}

return indexes;
}

// static
std::string IndexKeyUtils::vertexIndexKey(
size_t vIdLen, PartitionID partId, IndexID indexId, const VertexID& vId, std::string&& values) {
int32_t item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kIndex);
std::string key;
key.reserve(256);
key.append(reinterpret_cast<const char*>(&item), sizeof(int32_t))
.append(reinterpret_cast<const char*>(&indexId), sizeof(IndexID))
.append(values)
.append(vId.data(), vId.size())
.append(vIdLen - vId.size(), '\0');
return key;
}

// static
std::vector<std::string> IndexKeyUtils::vertexIndexKeysForGeography(
size_t vIdLen,
PartitionID partId,
IndexID indexId,
const VertexID& vId,
std::vector<std::string>&& values) {
std::vector<std::string> IndexKeyUtils::vertexIndexKeys(size_t vIdLen,
PartitionID partId,
IndexID indexId,
const VertexID& vId,
std::vector<std::string>&& values) {
int32_t item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kIndex);
std::vector<std::string> keys;
for (const auto& value : values) {
Expand All @@ -109,36 +90,13 @@ std::vector<std::string> IndexKeyUtils::vertexIndexKeysForGeography(
}

// static
std::string IndexKeyUtils::edgeIndexKey(size_t vIdLen,
PartitionID partId,
IndexID indexId,
const VertexID& srcId,
EdgeRanking rank,
const VertexID& dstId,
std::string&& values) {
int32_t item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kIndex);
std::string key;
key.reserve(256);
key.append(reinterpret_cast<const char*>(&item), sizeof(int32_t))
.append(reinterpret_cast<const char*>(&indexId), sizeof(IndexID))
.append(values)
.append(srcId.data(), srcId.size())
.append(vIdLen - srcId.size(), '\0')
.append(IndexKeyUtils::encodeRank(rank))
.append(dstId.data(), dstId.size())
.append(vIdLen - dstId.size(), '\0');
return key;
}

// static
std::vector<std::string> IndexKeyUtils::edgeIndexKeysForGeography(
size_t vIdLen,
PartitionID partId,
IndexID indexId,
const VertexID& srcId,
EdgeRanking rank,
const VertexID& dstId,
std::vector<std::string>&& values) {
std::vector<std::string> IndexKeyUtils::edgeIndexKeys(size_t vIdLen,
PartitionID partId,
IndexID indexId,
const VertexID& srcId,
EdgeRanking rank,
const VertexID& dstId,
std::vector<std::string>&& values) {
int32_t item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kIndex);
std::vector<std::string> keys;
for (const auto& value : values) {
Expand Down Expand Up @@ -196,7 +154,7 @@ Value IndexKeyUtils::parseIndexTTL(const folly::StringPiece& raw) {
}

// static
StatusOr<std::string> IndexKeyUtils::collectIndexValues(
StatusOr<std::vector<std::string>> IndexKeyUtils::collectIndexValues(
RowReader* reader, const std::vector<nebula::meta::cpp2::ColumnDef>& cols) {
if (reader == nullptr) {
return Status::Error("Invalid row reader");
Expand All @@ -215,25 +173,6 @@ StatusOr<std::string> IndexKeyUtils::collectIndexValues(
return encodeValues(std::move(values), cols);
}

// TODO(jie) Should be refactored
// static
StatusOr<std::vector<std::string>> IndexKeyUtils::collectIndexValueForGeography(
RowReader* reader, const nebula::meta::cpp2::ColumnDef& col) {
if (reader == nullptr) {
return Status::Error("Invalid row reader");
}
DCHECK(col.get_type().get_type() == meta::cpp2::PropertyType::GEOGRAPHY);

Value value = reader->getValueByName(col.get_name());
auto isNullable = col.nullable_ref().value_or(false);
auto ret = checkValue(value, isNullable);
if (!ret.ok()) {
LOG(ERROR) << "prop error by : " << col.get_name() << ". status : " << ret;
return ret;
}
return encodeValueForGeography(std::move(value), col);
}

// static
Status IndexKeyUtils::checkValue(const Value& v, bool isNullable) {
if (!v.isNull()) {
Expand Down
57 changes: 21 additions & 36 deletions src/common/utils/IndexKeyUtils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ class IndexKeyUtils final {
return encodeDateTime(v.getDateTime());
}
case Value::Type::GEOGRAPHY: {
LOG(FATAL) << "Should call encodeValueForGeography separately";
LOG(FATAL) << "Should call encodeGeography separately";
return "";
}
default:
Expand Down Expand Up @@ -301,15 +301,16 @@ class IndexKeyUtils final {
}

static std::vector<std::string> encodeGeography(const nebula::Geography& gg) {
geo::RegionCoverParams
rc; // TODO(jie): Get index params from meta to construct RegionCoverParams
geo::GeoIndex geoIndex(rc, false); // TODO(jie): Get schema meta to know if it's point only
// TODO(jie): Get index params from meta to construct RegionCoverParams
geo::RegionCoverParams rc;
// TODO(jie): Get schema meta to know if it's point only
geo::GeoIndex geoIndex(rc, false);
auto cellIds = geoIndex.indexCells(gg);
std::vector<std::string> bufs;
for (auto cellId : cellIds) {
bufs.emplace_back(encodeUint64(cellId));
}
return bufs; // just support index point here.
return bufs;
}

static nebula::DateTime decodeDateTime(const folly::StringPiece& raw) {
Expand Down Expand Up @@ -493,44 +494,30 @@ class IndexKeyUtils final {
/**
* Generate vertex|edge index key for kv store
**/
static std::string encodeValues(std::vector<Value>&& values,
const std::vector<nebula::meta::cpp2::ColumnDef>& cols);
static std::vector<std::string> encodeValueForGeography(Value&& value,
const nebula::meta::cpp2::ColumnDef& col);
static std::vector<std::string> encodeValues(
std::vector<Value>&& values, const std::vector<nebula::meta::cpp2::ColumnDef>& cols);

/**
* param valueTypes ： column type of each index column. If there are no
*nullable columns in the index, the parameter can be empty.
**/
static std::string vertexIndexKey(size_t vIdLen,
PartitionID partId,
IndexID indexId,
const VertexID& vId,
std::string&& values);
static std::vector<std::string> vertexIndexKeysForGeography(size_t vIdLen,
PartitionID partId,
IndexID indexId,
const VertexID& vId,
std::vector<std::string>&& values);
static std::vector<std::string> vertexIndexKeys(size_t vIdLen,
PartitionID partId,
IndexID indexId,
const VertexID& vId,
std::vector<std::string>&& values);

/**
* param valueTypes ： column type of each index column. If there are no
*nullable columns in the index, the parameter can be empty.
**/
static std::string edgeIndexKey(size_t vIdLen,
PartitionID partId,
IndexID indexId,
const VertexID& srcId,
EdgeRanking rank,
const VertexID& dstId,
std::string&& values);
static std::vector<std::string> edgeIndexKeysForGeography(size_t vIdLen,
PartitionID partId,
IndexID indexId,
const VertexID& srcId,
EdgeRanking rank,
const VertexID& dstId,
std::vector<std::string>&& values);
static std::vector<std::string> edgeIndexKeys(size_t vIdLen,
PartitionID partId,
IndexID indexId,
const VertexID& srcId,
EdgeRanking rank,
const VertexID& dstId,
std::vector<std::string>&& values);

static std::string indexPrefix(PartitionID partId, IndexID indexId);

Expand All @@ -540,10 +527,8 @@ class IndexKeyUtils final {

static Value parseIndexTTL(const folly::StringPiece& raw);

static StatusOr<std::string> collectIndexValues(
static StatusOr<std::vector<std::string>> collectIndexValues(
RowReader* reader, const std::vector<nebula::meta::cpp2::ColumnDef>& cols);
static StatusOr<std::vector<std::string>> collectIndexValueForGeography(
RowReader* reader, const nebula::meta::cpp2::ColumnDef& col);

private:
IndexKeyUtils() = delete;
Expand Down
Loading

0 comments on commit 7b1b40a

Please sign in to comment.