From 201530c23e406d0b41a420b5bd9c638a73e5ceac Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 28 Mar 2019 21:24:21 +0800 Subject: [PATCH 1/6] mprove schema deletion by paging for large data implemented: #416 Change-Id: I61f11dfb6367acab9369b53af57c9114a11782c7 --- .../backend/tx/GraphTransaction.java | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java index c3b178379b..f0df43abae 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java @@ -63,6 +63,7 @@ import com.baidu.hugegraph.iterator.FilterIterator; import com.baidu.hugegraph.iterator.FlatMapperIterator; import com.baidu.hugegraph.iterator.MapperIterator; +import com.baidu.hugegraph.iterator.Metadatable; import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.schema.EdgeLabel; import com.baidu.hugegraph.schema.IndexLabel; @@ -88,6 +89,7 @@ public class GraphTransaction extends IndexableTransaction { public static final int COMMIT_BATCH = 500; + private static final long TRAVERSE_BATCH = 100_000L; private final GraphIndexTransaction indexTx; @@ -1410,6 +1412,20 @@ public void traverseEdgesByLabel(EdgeLabel label, Consumer consumer, private void traverseByLabel(SchemaLabel label, Function> fetcher, Consumer consumer, boolean remove) { + if (!this.store().features().supportsQueryByPage()) { + this.traverseByLabelWithoutPage(label, fetcher, consumer, remove); + return; + } + if (!label.enableLabelIndex()) { + this.traverseByLabelWithoutIndexInPage(label, fetcher, consumer); + } else { + this.traverseByLabelWithIndexInPage(label, fetcher, consumer); + } + } + + private void traverseByLabelWithoutPage( + SchemaLabel label, Function> fetcher, + Consumer consumer, boolean remove) { HugeType type = label.type() == HugeType.VERTEX_LABEL ? HugeType.VERTEX : HugeType.EDGE; ConditionQuery query = new ConditionQuery(type); @@ -1453,4 +1469,58 @@ private void traverseByLabel(SchemaLabel label, assert counter <= Query.DEFAULT_CAPACITY; } while (counter == Query.DEFAULT_CAPACITY); // If not, means finish } + + private void traverseByLabelWithIndexInPage( + SchemaLabel label, Function> fetcher, + Consumer consumer) { + HugeType type = label.type() == HugeType.VERTEX_LABEL ? + HugeType.VERTEX : HugeType.EDGE; + ConditionQuery query = new ConditionQuery(type); + query.eq(HugeKeys.LABEL, label.id()); + query.limit(TRAVERSE_BATCH); + // Whether query system vertices + if (label.hidden()) { + query.showHidden(true); + } + + Iterator itor; + String page = ""; + while (page != null) { + query.page(page); + itor = fetcher.apply(query); + while (itor.hasNext()) { + consumer.accept(itor.next()); + } + page = (String) ((Metadatable) itor).metadata("page"); + } + } + + private void traverseByLabelWithoutIndexInPage( + SchemaLabel label, Function> fetcher, + Consumer consumer) { + HugeType type = label.type() == HugeType.VERTEX_LABEL ? + HugeType.VERTEX : HugeType.EDGE; + Query query = new Query(type); + query.limit(TRAVERSE_BATCH); + // Whether query system vertices + if (label.hidden()) { + query.showHidden(true); + } + + Iterator itor; + // Not support label index, query all and filter by label + String page = ""; + while (page != null) { + query.page(page); + itor = fetcher.apply(query); + while (itor.hasNext()) { + T e = itor.next(); + SchemaLabel elemLabel = ((HugeElement) e).schemaLabel(); + if (label.equals(elemLabel)) { + consumer.accept(e); + } + } + page = (String) ((Metadatable) itor).metadata("page"); + } + } } From 81a95508736fc087a290672d880cb2e981afd634 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Fri, 29 Mar 2019 13:11:45 +0800 Subject: [PATCH 2/6] add EmptyIterator for backend store query Change-Id: Ifca1ab84a66778cea0669c971a0440e8b1c9d674 --- .../backend/store/BackendEntryIterator.java | 24 +++++++++++++++++++ .../store/scylladb/ScyllaDBTables.java | 20 ++++++++-------- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendEntryIterator.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendEntryIterator.java index d179240050..38f46883fd 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendEntryIterator.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendEntryIterator.java @@ -27,6 +27,7 @@ import com.baidu.hugegraph.exception.LimitExceedException; import com.baidu.hugegraph.exception.NotSupportException; import com.baidu.hugegraph.iterator.Metadatable; +import com.baidu.hugegraph.util.E; public abstract class BackendEntryIterator implements Iterator, AutoCloseable, Metadatable { @@ -38,6 +39,7 @@ public abstract class BackendEntryIterator private long count; public BackendEntryIterator(Query query) { + E.checkNotNull(query, "query"); this.query = query; this.count = 0L; this.current = null; @@ -150,4 +152,26 @@ protected long skip(BackendEntry entry, long skip) { protected abstract boolean fetch(); protected abstract String pageState(); + + public static final class EmptyIterator extends BackendEntryIterator { + + public EmptyIterator(Query query) { + super(query); + } + + @Override + protected boolean fetch() { + return false; + } + + @Override + protected String pageState() { + return null; + } + + @Override + public void close() throws Exception { + return; + } + } } diff --git a/hugegraph-scylladb/src/main/java/com/baidu/hugegraph/backend/store/scylladb/ScyllaDBTables.java b/hugegraph-scylladb/src/main/java/com/baidu/hugegraph/backend/store/scylladb/ScyllaDBTables.java index 4e1305cb9d..a4094369c5 100644 --- a/hugegraph-scylladb/src/main/java/com/baidu/hugegraph/backend/store/scylladb/ScyllaDBTables.java +++ b/hugegraph-scylladb/src/main/java/com/baidu/hugegraph/backend/store/scylladb/ScyllaDBTables.java @@ -33,6 +33,7 @@ import com.baidu.hugegraph.backend.query.ConditionQuery; import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.store.BackendEntry; +import com.baidu.hugegraph.backend.store.BackendEntryIterator; import com.baidu.hugegraph.backend.store.cassandra.CassandraBackendEntry; import com.baidu.hugegraph.backend.store.cassandra.CassandraSessionPool; import com.baidu.hugegraph.backend.store.cassandra.CassandraTable; @@ -123,7 +124,7 @@ private static Query queryByLabelIndex( } ConditionQuery q = (ConditionQuery) query; - Id label = (Id) q.condition(HugeKeys.LABEL); + Id label = q.condition(HugeKeys.LABEL); if (label != null && q.allSysprop() && conditions.size() == 1 && q.containsCondition(HugeKeys.LABEL, Condition.RelationType.EQ)) { @@ -211,11 +212,11 @@ public void delete(CassandraSessionPool.Session session, public Iterator query( CassandraSessionPool.Session session, Query query) { - query = queryByLabelIndex(session, indexTable(), query); - if (query == null) { - return ImmutableList.of().iterator(); + Query idQuery = queryByLabelIndex(session, indexTable(), query); + if (idQuery == null) { + return new BackendEntryIterator.EmptyIterator(query); } - return super.query(session, query); + return super.query(session, idQuery); } } @@ -317,12 +318,11 @@ protected void deleteEdgesByLabel(CassandraSessionPool.Session session, @Override public Iterator query( CassandraSessionPool.Session session, Query query) { - query = queryByLabelIndex(session, indexTable(), query); - - if (query == null) { - return ImmutableList.of().iterator(); + Query idQuery = queryByLabelIndex(session, indexTable(), query); + if (idQuery == null) { + return new BackendEntryIterator.EmptyIterator(query); } - return super.query(session, query); + return super.query(session, idQuery); } public static Edge out(String store) { From 1340de8f080d9180c804bf7a465c8037a771f428 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Sun, 31 Mar 2019 12:16:44 +0800 Subject: [PATCH 3/6] filter vertices/edges of deleting vertex/edge label Change-Id: I0687cab77284895d8e803a74cd3f30b0df0a004b --- .../baidu/hugegraph/backend/query/Query.java | 10 ++++++++ .../backend/tx/GraphTransaction.java | 25 +++++++++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java index 406ccb5a31..0137c38049 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java @@ -48,6 +48,7 @@ public class Query implements Cloneable { private String page; private long capacity; private boolean showHidden; + private boolean includeDeleting; private Query originQuery; @@ -70,6 +71,7 @@ public Query(HugeType resultType, Query originQuery) { this.capacity = defaultCapacity(); this.showHidden = false; + this.includeDeleting = false; } public HugeType resultType() { @@ -212,6 +214,14 @@ public void showHidden(boolean showHidden) { this.showHidden = showHidden; } + public boolean includeDeleting() { + return this.includeDeleting; + } + + public void includeDeleting(boolean includeDeleting) { + this.includeDeleting = includeDeleting; + } + public Set ids() { return ImmutableSet.of(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java index f0df43abae..cdc7d7da27 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java @@ -81,6 +81,7 @@ import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.type.define.IdStrategy; +import com.baidu.hugegraph.type.define.SchemaStatus; import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.InsertionOrderUtil; import com.baidu.hugegraph.util.LockUtil; @@ -482,6 +483,11 @@ public Iterator queryVertices(Query query) { if (!query.showHidden() && Graph.Hidden.isHidden(vertex.label())) { return false; } + // Filter vertices of deleting vertex label + if (vertex.schemaLabel().status() == SchemaStatus.DELETING && + !query.includeDeleting()) { + return false; + } // Process results that query from left index or primary-key if (query.resultType().isVertex() && !filterResultFromIndexQuery(query, vertex)) { @@ -608,6 +614,11 @@ public Iterator queryEdges(Query query) { if (!query.showHidden() && Graph.Hidden.isHidden(edge.label())) { return false; } + // Filter edges of deleting edge label + if (edge.schemaLabel().status() == SchemaStatus.DELETING && + !query.includeDeleting()) { + return false; + } // Process results that query from left index if (!this.filterResultFromIndexQuery(query, edge)) { return false; @@ -1417,9 +1428,11 @@ private void traverseByLabel(SchemaLabel label, return; } if (!label.enableLabelIndex()) { - this.traverseByLabelWithoutIndexInPage(label, fetcher, consumer); + this.traverseByLabelWithoutIndexInPage(label, fetcher, + consumer, remove); } else { - this.traverseByLabelWithIndexInPage(label, fetcher, consumer); + this.traverseByLabelWithIndexInPage(label, fetcher, + consumer, remove); } } @@ -1433,7 +1446,7 @@ private void traverseByLabelWithoutPage( if (label.hidden()) { query.showHidden(true); } - + query.includeDeleting(remove); // Not support label index, query all and filter by label if (!label.enableLabelIndex()) { query.capacity(Query.NO_CAPACITY); @@ -1472,7 +1485,7 @@ private void traverseByLabelWithoutPage( private void traverseByLabelWithIndexInPage( SchemaLabel label, Function> fetcher, - Consumer consumer) { + Consumer consumer, boolean remove) { HugeType type = label.type() == HugeType.VERTEX_LABEL ? HugeType.VERTEX : HugeType.EDGE; ConditionQuery query = new ConditionQuery(type); @@ -1482,6 +1495,7 @@ private void traverseByLabelWithIndexInPage( if (label.hidden()) { query.showHidden(true); } + query.includeDeleting(remove); Iterator itor; String page = ""; @@ -1497,7 +1511,7 @@ private void traverseByLabelWithIndexInPage( private void traverseByLabelWithoutIndexInPage( SchemaLabel label, Function> fetcher, - Consumer consumer) { + Consumer consumer, boolean remove) { HugeType type = label.type() == HugeType.VERTEX_LABEL ? HugeType.VERTEX : HugeType.EDGE; Query query = new Query(type); @@ -1506,6 +1520,7 @@ private void traverseByLabelWithoutIndexInPage( if (label.hidden()) { query.showHidden(true); } + query.includeDeleting(remove); Iterator itor; // Not support label index, query all and filter by label From 4546c6bfd26193ab375f96c8baf0d8e96599b25a Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Mon, 1 Apr 2019 18:27:30 +0800 Subject: [PATCH 4/6] merge traverser logic no matter if supporting paging Change-Id: I265f6f226dfe19f6ab66d5eaae6d5f969f591949 --- .../hugegraph/backend/page/IdHolderList.java | 2 + .../backend/page/PageEntryIterator.java | 16 +- .../hugegraph/backend/page/QueryList.java | 31 ++-- .../baidu/hugegraph/backend/query/Query.java | 16 +- .../backend/tx/GraphIndexTransaction.java | 2 +- .../backend/tx/GraphTransaction.java | 147 +++++------------- 6 files changed, 83 insertions(+), 131 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/IdHolderList.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/IdHolderList.java index 98c8428233..8e6db384e8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/IdHolderList.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/IdHolderList.java @@ -26,6 +26,8 @@ public final class IdHolderList extends ArrayList { + private static final long serialVersionUID = -738694176552424990L; + private final boolean paging; public IdHolderList(boolean paging) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/PageEntryIterator.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/PageEntryIterator.java index 70b5532b28..e03a98b315 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/PageEntryIterator.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/PageEntryIterator.java @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.NoSuchElementException; +import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.store.BackendEntry; import com.baidu.hugegraph.exception.NotSupportException; import com.baidu.hugegraph.iterator.Metadatable; @@ -44,7 +45,7 @@ public PageEntryIterator(QueryList queries, long pageSize) { } private PageState parsePageState() { - String page = this.queries.parent().page(); + String page = this.queries.parent().pageWithoutCheck(); PageState pageState = PageState.fromString(page); E.checkState(pageState.offset() < this.queries.total(), "Invalid page '%s' with an offset '%s' exceeds " + @@ -61,13 +62,15 @@ public boolean hasNext() { } private boolean fetch() { - if (this.remaining <= 0 || + if ((this.remaining != Query.NO_LIMIT && this.remaining <= 0L) || this.pageState.offset() >= this.queries.total()) { return false; } - long pageSize = this.remaining < this.pageSize ? - this.remaining : this.pageSize; + long pageSize = this.pageSize; + if (this.remaining != Query.NO_LIMIT && this.remaining < pageSize) { + pageSize = this.remaining; + } this.results = this.queries.fetchNext(this.pageState, pageSize); assert this.results != null; @@ -90,7 +93,10 @@ public BackendEntry next() { throw new NoSuchElementException(); } BackendEntry entry = this.results.iterator().next(); - this.remaining--; + if (this.remaining != Query.NO_LIMIT) { + // Assume one result in each entry (just for index query) + this.remaining--; + } return entry; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/QueryList.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/QueryList.java index 2aed96b807..ded4b09518 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/QueryList.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/QueryList.java @@ -90,7 +90,7 @@ public boolean empty() { } public Iterator fetch() { - assert !queries.isEmpty(); + assert !this.queries.isEmpty(); if (this.parent.paging()) { int pageSize = this.graph.configuration() .get(CoreOptions.QUERY_PAGE_SIZE); @@ -122,6 +122,7 @@ protected PageIterator fetchNext(PageState pageState, long pageSize) { return query.iterator(offset - current, pageState.page(), pageSize); } + @SuppressWarnings("unused") private static Set limit(Set ids, Query query) { long fromIndex = query.offset(); E.checkArgument(fromIndex <= Integer.MAX_VALUE, @@ -167,7 +168,7 @@ private interface QueryHolder { } /** - * Generate queries from index.optimizeQuery() + * Generate queries from tx.optimizeQuery() */ private class OptimizedQuery implements QueryHolder { @@ -183,9 +184,14 @@ public Iterator iterator() { public PageIterator iterator(int index, String page, long pageSize) { assert index == 0; - this.query.page(page); - Iterator iterator = fetcher.apply(this.query); - // Must iterate all entries before getting the next page info + Query query = this.query.copy(); + query.page(page); + // Not set limit to pageSize due to PageEntryIterator.remaining + if (this.query.limit() == Query.NO_LIMIT) { + query.limit(pageSize); + } + Iterator iterator = fetcher.apply(query); + // Must iterate all entries before get the next page List results = IteratorUtils.list(iterator); return new PageIterator(results.iterator(), PageState.page(iterator)); @@ -197,7 +203,7 @@ public int total() { } /** - * Generate queries from index.indexQuery() + * Generate queries from tx.indexQuery() */ private class IndexQuery implements QueryHolder { @@ -214,10 +220,15 @@ public Iterator iterator() { return null; } Set ids = holder.ids(); - if (ids.size() > parent.limit()) { - ids = limit(ids, parent); + if (parent.limit() != Query.NO_LIMIT && + ids.size() > parent.limit()) { + /* + * Avoid too many ids in one time query, + * Assume it will get one result by each id + */ + ids = CollectionUtil.subSet(ids, 0, (int) parent.limit()); } - IdQuery query = new IdQuery(parent.resultType(), ids); + IdQuery query = new IdQuery(parent, ids); return fetcher.apply(query); }); } @@ -228,7 +239,7 @@ public PageIterator iterator(int index, String page, long pageSize) { if (pageIds.empty()) { return PageIterator.EMPTY; } - IdQuery query = new IdQuery(parent.resultType(), pageIds.ids()); + IdQuery query = new IdQuery(parent, pageIds.ids()); return new PageIterator(fetcher.apply(query), pageIds.page()); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java index 0137c38049..12eaceadba 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java @@ -48,7 +48,7 @@ public class Query implements Cloneable { private String page; private long capacity; private boolean showHidden; - private boolean includeDeleting; + private boolean showDeleting; private Query originQuery; @@ -71,7 +71,7 @@ public Query(HugeType resultType, Query originQuery) { this.capacity = defaultCapacity(); this.showHidden = false; - this.includeDeleting = false; + this.showDeleting = false; } public HugeType resultType() { @@ -176,6 +176,10 @@ public String page() { return this.page; } + public String pageWithoutCheck() { + return this.page; + } + public void page(String page) { this.page = page; } @@ -214,12 +218,12 @@ public void showHidden(boolean showHidden) { this.showHidden = showHidden; } - public boolean includeDeleting() { - return this.includeDeleting; + public boolean showDeleting() { + return this.showDeleting; } - public void includeDeleting(boolean includeDeleting) { - this.includeDeleting = includeDeleting; + public void showDeleting(boolean showDeleting) { + this.showDeleting = showDeleting; } public Set ids() { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java index b7deb5837d..3ee581a73c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java @@ -285,7 +285,7 @@ private List queryByLabel(ConditionQuery query) { indexQuery.eq(HugeKeys.INDEX_LABEL_ID, il.id()); indexQuery.eq(HugeKeys.FIELD_VALUES, label); // Set offset and limit to avoid redundant element ids - indexQuery.page(query.page()); + indexQuery.page(query.pageWithoutCheck()); indexQuery.limit(query.limit()); indexQuery.offset(query.offset()); indexQuery.capacity(query.capacity()); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java index cdc7d7da27..81974b28b0 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java @@ -46,6 +46,7 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.id.SplicingIdGenerator; import com.baidu.hugegraph.backend.page.IdHolder; +import com.baidu.hugegraph.backend.page.PageState; import com.baidu.hugegraph.backend.page.QueryList; import com.baidu.hugegraph.backend.query.Condition; import com.baidu.hugegraph.backend.query.ConditionQuery; @@ -485,7 +486,7 @@ public Iterator queryVertices(Query query) { } // Filter vertices of deleting vertex label if (vertex.schemaLabel().status() == SchemaStatus.DELETING && - !query.includeDeleting()) { + !query.showDeleting()) { return false; } // Process results that query from left index or primary-key @@ -616,7 +617,7 @@ public Iterator queryEdges(Query query) { } // Filter edges of deleting edge label if (edge.schemaLabel().status() == SchemaStatus.DELETING && - !query.includeDeleting()) { + !query.showDeleting()) { return false; } // Process results that query from left index @@ -1411,131 +1412,59 @@ public void removeEdges(EdgeLabel edgeLabel) { public void traverseVerticesByLabel(VertexLabel label, Consumer consumer, - boolean remove) { - this.traverseByLabel(label, this::queryVertices, consumer, remove); + boolean deleting) { + this.traverseByLabel(label, this::queryVertices, consumer, deleting); } public void traverseEdgesByLabel(EdgeLabel label, Consumer consumer, - boolean remove) { - this.traverseByLabel(label, this::queryEdges, consumer, remove); + boolean deleting) { + this.traverseByLabel(label, this::queryEdges, consumer, deleting); } private void traverseByLabel(SchemaLabel label, Function> fetcher, - Consumer consumer, boolean remove) { - if (!this.store().features().supportsQueryByPage()) { - this.traverseByLabelWithoutPage(label, fetcher, consumer, remove); - return; - } - if (!label.enableLabelIndex()) { - this.traverseByLabelWithoutIndexInPage(label, fetcher, - consumer, remove); - } else { - this.traverseByLabelWithIndexInPage(label, fetcher, - consumer, remove); - } - } - - private void traverseByLabelWithoutPage( - SchemaLabel label, Function> fetcher, - Consumer consumer, boolean remove) { + Consumer consumer, boolean deleting) { HugeType type = label.type() == HugeType.VERTEX_LABEL ? HugeType.VERTEX : HugeType.EDGE; - ConditionQuery query = new ConditionQuery(type); - // Whether query system vertices - if (label.hidden()) { - query.showHidden(true); - } - query.includeDeleting(remove); - // Not support label index, query all and filter by label - if (!label.enableLabelIndex()) { - query.capacity(Query.NO_CAPACITY); - Iterator itor = fetcher.apply(query); - while (itor.hasNext()) { - T e = itor.next(); - SchemaLabel elemLabel = ((HugeElement) e).schemaLabel(); - if (label.equals(elemLabel)) { - consumer.accept(e); - } - } - return; - } - - /* - * Support label index, query by label. Set limit&capacity to - * Query.DEFAULT_CAPACITY to limit elements number per pass - */ - query.limit(Query.DEFAULT_CAPACITY); + Query query = label.enableLabelIndex() ? + new ConditionQuery(type) : + new Query(type); query.capacity(Query.NO_CAPACITY); - query.eq(HugeKeys.LABEL, label.id()); - int pass = 0; - int counter; - do { - if (!remove) { - query.offset(pass++ * Query.DEFAULT_CAPACITY); - } - // Process every element in current batch - Iterator itor = fetcher.apply(query); - for (counter = 0; itor.hasNext(); ++counter) { - consumer.accept(itor.next()); - } - assert counter <= Query.DEFAULT_CAPACITY; - } while (counter == Query.DEFAULT_CAPACITY); // If not, means finish - } - - private void traverseByLabelWithIndexInPage( - SchemaLabel label, Function> fetcher, - Consumer consumer, boolean remove) { - HugeType type = label.type() == HugeType.VERTEX_LABEL ? - HugeType.VERTEX : HugeType.EDGE; - ConditionQuery query = new ConditionQuery(type); - query.eq(HugeKeys.LABEL, label.id()); - query.limit(TRAVERSE_BATCH); - // Whether query system vertices + query.limit(Query.NO_LIMIT); + if (this.store().features().supportsQueryByPage()) { + query.page(PageState.PAGE_NONE); + } if (label.hidden()) { query.showHidden(true); } - query.includeDeleting(remove); + query.showDeleting(deleting); - Iterator itor; - String page = ""; - while (page != null) { - query.page(page); - itor = fetcher.apply(query); + if (label.enableLabelIndex()) { + // Support label index, query by label index + ((ConditionQuery) query).eq(HugeKeys.LABEL, label.id()); + Iterator itor = fetcher.apply(query); while (itor.hasNext()) { consumer.accept(itor.next()); } - page = (String) ((Metadatable) itor).metadata("page"); - } - } - - private void traverseByLabelWithoutIndexInPage( - SchemaLabel label, Function> fetcher, - Consumer consumer, boolean remove) { - HugeType type = label.type() == HugeType.VERTEX_LABEL ? - HugeType.VERTEX : HugeType.EDGE; - Query query = new Query(type); - query.limit(TRAVERSE_BATCH); - // Whether query system vertices - if (label.hidden()) { - query.showHidden(true); - } - query.includeDeleting(remove); - - Iterator itor; - // Not support label index, query all and filter by label - String page = ""; - while (page != null) { - query.page(page); - itor = fetcher.apply(query); - while (itor.hasNext()) { - T e = itor.next(); - SchemaLabel elemLabel = ((HugeElement) e).schemaLabel(); - if (label.equals(elemLabel)) { - consumer.accept(e); - } + } else { + // Not support label index, query all and filter by label + if (query.paging()) { + query.limit(TRAVERSE_BATCH); } - page = (String) ((Metadatable) itor).metadata("page"); + String page = null; + do { + Iterator itor = fetcher.apply(query); + while (itor.hasNext()) { + T e = itor.next(); + SchemaLabel elemLabel = ((HugeElement) e).schemaLabel(); + if (label.equals(elemLabel)) { + consumer.accept(e); + } + } + if (query.paging()) { + page = (String) ((Metadatable) itor).metadata("page"); + } + } while (page != null); } } } From 48e24cfe28b1862cd874a19305254e382e77ff94 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 2 Apr 2019 16:21:20 +0800 Subject: [PATCH 5/6] change query.page_size range to (1, 80w) Change-Id: I0284b6ea7edd33898c54b5bf223ab069a229245d --- .../main/java/com/baidu/hugegraph/backend/query/Query.java | 5 ++--- .../main/java/com/baidu/hugegraph/config/CoreOptions.java | 4 +++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java index 12eaceadba..bdd35bc6c8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Objects; import java.util.Set; import com.baidu.hugegraph.backend.BackendException; @@ -31,7 +32,6 @@ import com.baidu.hugegraph.type.HugeType; import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.util.E; -import com.google.common.base.Objects; import com.google.common.collect.ImmutableSet; public class Query implements Cloneable { @@ -260,8 +260,7 @@ public boolean equals(Object object) { this.orders.equals(other.orders) && this.offset == other.offset && this.limit == other.limit && - ((this.page == null && other.page == null) || - this.page.equals(other.page)) && + Objects.equals(this.page, other.page) && this.ids().equals(other.ids()) && this.conditions().equals(other.conditions()); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java index b2cece77e8..8623df8bb6 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java @@ -19,6 +19,8 @@ package com.baidu.hugegraph.config; +import com.baidu.hugegraph.backend.query.Query; + import static com.baidu.hugegraph.backend.tx.GraphTransaction.COMMIT_BATCH; import static com.baidu.hugegraph.config.OptionChecker.disallowEmpty; import static com.baidu.hugegraph.config.OptionChecker.rangeInt; @@ -211,7 +213,7 @@ public static synchronized CoreOptions instance() { new ConfigOption<>( "query.page_size", "The size of each page when query using paging.", - rangeInt(0, 10000), + rangeInt(1, (int) Query.DEFAULT_CAPACITY), 500 ); From 2fa55f5ec05a4d72bdd8e25712a8af5090b4d710 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Wed, 3 Apr 2019 10:57:11 +0800 Subject: [PATCH 6/6] rebase PageState.page(Iterator) Change-Id: I4a4a13c42ef71f9f73544793c1e11503d37b21ff --- .../java/com/baidu/hugegraph/backend/tx/GraphTransaction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java index 81974b28b0..be78485b76 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java @@ -1462,7 +1462,7 @@ private void traverseByLabel(SchemaLabel label, } } if (query.paging()) { - page = (String) ((Metadatable) itor).metadata("page"); + page = PageState.page(itor); } } while (page != null); }