Skip to content

Commit

Permalink
Support paging for scan api
Browse files Browse the repository at this point in the history 
implemented: #360

Change-Id: Idea152b3d53ef519c7ed9847ee9e65092b264e08
  • Loading branch information
@zhoney
zhoney committed Mar 14, 2019
1 parent 1e8e7bd commit e873530
Show file tree
Hide file tree
Showing 9 changed files with 72 additions and 36 deletions.
2 changes: 1 addition & 1 deletion hugegraph-api/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@
</addDefaultSpecificationEntries>
</manifest>
<manifestEntries>
<Implementation-Version>0.34.0.0</Implementation-Version>
<Implementation-Version>0.35.0.0</Implementation-Version>
</manifestEntries>
</archive>
</configuration>
Expand Down
12 changes: 8 additions & 4 deletions hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import com.baidu.hugegraph.api.filter.CompressInterceptor.Compress;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.query.ConditionQuery;
import com.baidu.hugegraph.backend.query.Query;
import com.baidu.hugegraph.backend.store.Shard;
import com.baidu.hugegraph.core.GraphManager;
import com.baidu.hugegraph.server.RestServer;
Expand Down Expand Up @@ -101,16 +102,19 @@ public String shards(@Context GraphManager manager,
public String scan(@Context GraphManager manager,
@PathParam("graph") String graph,
@QueryParam("start") String start,
@QueryParam("end") String end) {
LOG.debug("Graph [{}] query edges by shard(start: {}, end: {}) ",
graph, start, end);
@QueryParam("end") String end,
@QueryParam("page") String page) {
LOG.debug("Graph [{}] query edges by shard(start: {}, end: {}, " +
"page: {}) ", graph, start, end, page);

HugeGraph g = graph(manager, graph);

ConditionQuery query = new ConditionQuery(HugeType.EDGE_OUT);
query.scan(start, end);
query.limit(Query.DEFAULT_CAPACITY);
query.page(page);
Iterator<Edge> edges = g.edges(query);

return manager.serializer(g).writeEdges(edges, false);
return manager.serializer(g).writeEdges(edges, true);
}
}
12 changes: 8 additions & 4 deletions hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import com.baidu.hugegraph.api.graph.VertexAPI;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.query.ConditionQuery;
import com.baidu.hugegraph.backend.query.Query;
import com.baidu.hugegraph.backend.store.Shard;
import com.baidu.hugegraph.core.GraphManager;
import com.baidu.hugegraph.server.RestServer;
Expand Down Expand Up @@ -101,16 +102,19 @@ public String shards(@Context GraphManager manager,
public String scan(@Context GraphManager manager,
@PathParam("graph") String graph,
@QueryParam("start") String start,
@QueryParam("end") String end) {
LOG.debug("Graph [{}] query vertices by shard(start: {}, end: {}) ",
graph, start, end);
@QueryParam("end") String end,
@QueryParam("page") String page) {
LOG.debug("Graph [{}] query vertices by shard(start: {}, end: {}, " +
"page: {}) ", graph, start, end, page);

HugeGraph g = graph(manager, graph);

ConditionQuery query = new ConditionQuery(HugeType.VERTEX);
query.scan(start, end);
query.limit(Query.DEFAULT_CAPACITY);
query.page(page);
Iterator<Vertex> vertices = g.vertices(query);

return manager.serializer(g).writeVertices(vertices, false);
return manager.serializer(g).writeVertices(vertices, true);
}
}
12 changes: 11 additions & 1 deletion hugegraph-api/src/main/java/com/baidu/hugegraph/serializer/JsonSerializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import com.baidu.hugegraph.api.API;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.store.Shard;
import com.baidu.hugegraph.iterator.Metadatable;
import com.baidu.hugegraph.schema.EdgeLabel;
import com.baidu.hugegraph.schema.IndexLabel;
import com.baidu.hugegraph.schema.PropertyKey;
Expand Down Expand Up @@ -100,7 +101,16 @@ private String writeIterator(String label, Iterator<?> itor,

// Write page
if (paging) {
String page = TraversalUtil.page((GraphTraversal<?, ?>) itor);
String page;
if (itor instanceof GraphTraversal<?, ?>) {
page = TraversalUtil.page((GraphTraversal<?, ?>) itor);
} else if (itor instanceof Metadatable) {
page = (String) ((Metadatable) itor).metadata("page");
} else {
throw new HugeException("Error type '%s' of paging " +
"iterator '%s'",
itor.getClass(), itor);
}
if (page != null) {
page = String.format(",\"page\": \"%s\"", page);
} else {
Expand Down
3 changes: 2 additions & 1 deletion hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,11 @@ public final class ApiVersion {
* [0.32] Issue-250: Keep depth and degree consistent for traverser api
* [0.33] Issue-305: Implement customized paths and crosspoints RESTful API
* [0.34] Issue-307: Let VertexAPI use simplified property serializer
* [0.35] Issue-360: Support paging for scan api
*/

// The second parameter of Version.of() is for IDE running without JAR
public static final Version VERSION = Version.of(ApiVersion.class, "0.34");
public static final Version VERSION = Version.of(ApiVersion.class, "0.35");

public static final void check() {
// Check version of hugegraph-core. Firstly do check from version 0.3
Expand Down
45 changes: 26 additions & 19 deletions ...h-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,25 +127,6 @@ protected List<Select> query2Select(String table, Query query) {
"currently, it will be replaced by [0, offset + limit)");
}

// Set limit
if (query.limit() != Query.NO_LIMIT) {
long total = query.total();
String page = query.page();
if (page == null) {
select.limit((int) total);
} else {
select.setFetchSize((int) total);
// It's the first time if page is empty
if (!page.isEmpty()) {
try {
select.setPagingState(PagingState.fromString(page));
} catch (PagingStateException e) {
throw new BackendException(e.getMessage());
}
}
}
}

// Set order-by
for (Map.Entry<HugeKeys, Order> order : query.orders().entrySet()) {
String name = formatKey(order.getKey());
Expand All @@ -162,6 +143,7 @@ protected List<Select> query2Select(String table, Query query) {

if (query.conditions().isEmpty()) {
// Query only by id
this.setPageState(query, ids);
LOG.debug("Query only by id(s): {}", ids);
return ids;
} else {
Expand All @@ -170,11 +152,36 @@ protected List<Select> query2Select(String table, Query query) {
// Query by condition
conds.addAll(this.queryCondition2Select(query, selection));
}
this.setPageState(query, conds);
LOG.debug("Query by conditions: {}", conds);
return conds;
}
}

protected void setPageState(Query query, List<Select> selects) {
// Set limit
if (query.limit() == Query.NO_LIMIT) {
return;
}
for (Select select : selects) {
long total = query.total();
String page = query.page();
if (page == null) {
select.limit((int) total);
} else {
select.setFetchSize((int) total);
// It's the first time if page is empty
if (!page.isEmpty()) {
try {
select.setPagingState(PagingState.fromString(page));
} catch (PagingStateException e) {
throw new BackendException(e);
}
}
}
}
}

protected List<Select> queryId2Select(Query query, Select select) {
// Query by id(s)
if (query.ids().isEmpty()) {
Expand Down
4 changes: 2 additions & 2 deletions hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,8 @@ private Iterator<HugeVertex> queryVerticesByIds(IdQuery query) {

@Override
protected Iterator<HugeEdge> queryEdgesFromBackend(Query query) {
if (query.empty()) {
// Query all edges, don't cache it
if (query.empty() || query.paging()) {
// Query all edges or query edges with paging, don't cache it
return super.queryEdgesFromBackend(query);
}

Expand Down
9 changes: 7 additions & 2 deletions hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,14 +201,19 @@ protected RowIterator queryByCond(Session session, ConditionQuery query) {
"Invalid scan with multi conditions: %s", query);
Relation scan = query.relations().iterator().next();
Shard shard = (Shard) scan.value();
return this.queryByRange(session, shard);
return this.queryByRange(session, shard, query.page());
}
throw new NotSupportException("query: %s", query);
}

protected RowIterator queryByRange(Session session, Shard shard) {
protected RowIterator queryByRange(Session session, Shard shard,
String page) {
byte[] start = this.shardSpliter.position(shard.start());
byte[] end = this.shardSpliter.position(shard.end());
if (page != null && !page.isEmpty()) {
byte[] position = PageState.fromString(page).position();
return session.scan(this.table(), position, end);
}
return session.scan(this.table(), start, end);
}

Expand Down
9 changes: 7 additions & 2 deletions hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,14 +193,19 @@ protected BackendColumnIterator queryByCond(Session session,
"Invalid scan with multi conditions: %s", query);
Relation scan = query.relations().iterator().next();
Shard shard = (Shard) scan.value();
return this.queryByRange(session, shard);
return this.queryByRange(session, shard, query.page());
}
throw new NotSupportException("query: %s", query);
}

protected BackendColumnIterator queryByRange(Session session, Shard shard) {
protected BackendColumnIterator queryByRange(Session session, Shard shard,
String page) {
byte[] start = this.shardSpliter.position(shard.start());
byte[] end = this.shardSpliter.position(shard.end());
if (page != null && !page.isEmpty()) {
byte[] position = PageState.fromString(page).position();
return session.scan(this.table(), position, end);
}
return session.scan(this.table(), start, end);
}

Expand Down

0 comments on commit e873530

Please sign in to comment.