fix bug traverseByLabel() not set page every loop

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendMutation.java

@@ -279,7 +279,10 @@ public Iterator<BackendAction> values() {
         public int size() {
             int size = 0;
             for (Map<Id, List<BackendAction>> m : this.mutations.values()) {
-                size += m.size();
+                // NOTE: Index entry has same id with different subIds
+                for (List<BackendAction> actions : m.values()) {
+                    size += actions.size();
+                }
             }
             return size;
         }

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java

@@ -91,7 +91,7 @@
 public class GraphTransaction extends IndexableTransaction {
 
     public static final int COMMIT_BATCH = 500;
-    private static final long TRAVERSE_BATCH = 100_000L;
+    private final int pageSize;
 
     private final GraphIndexTransaction indexTx;
 
@@ -127,6 +127,7 @@ public GraphTransaction(HugeGraph graph, BackendStore store) {
                                 CoreOptions.VERTEX_CHECK_CUSTOMIZED_ID_EXIST);
         this.verticesCapacity = conf.get(CoreOptions.VERTEX_TX_CAPACITY);
         this.edgesCapacity = conf.get(CoreOptions.EDGE_TX_CAPACITY);
+        this.pageSize = conf.get(CoreOptions.QUERY_PAGE_SIZE);
         this.locksTable = new LockUtil.LocksTable(graph.name());
     }
 
@@ -1524,11 +1525,18 @@ private <T> void traverseByLabel(SchemaLabel label,
             Iterator<T> iter = fetcher.apply(query);
             while (iter.hasNext()) {
                 consumer.accept(iter.next());
+                /*
+                 * Commit per batch to avoid too much data in single commit,
+                 * especially for Cassandra backend
+                 */
+                this.commitIfGtSize(GraphTransaction.COMMIT_BATCH);
             }
+            // Commit changes if exists
+            this.commit();
         } else {
             // Not support label index, query all and filter by label
             if (query.paging()) {
-                query.limit(TRAVERSE_BATCH);
+                query.limit(this.pageSize);
             }
             String page = null;
             do {
@@ -1538,10 +1546,18 @@ private <T> void traverseByLabel(SchemaLabel label,
                     SchemaLabel elemLabel = ((HugeElement) e).schemaLabel();
                     if (label.equals(elemLabel)) {
                         consumer.accept(e);
+                        /*
+                         * Commit per batch to avoid too much data in single
+                         * commit, especially for Cassandra backend
+                         */
+                        this.commitIfGtSize(GraphTransaction.COMMIT_BATCH);
                     }
                 }
+                // Commit changes of every page before next page query
+                this.commit();
                 if (query.paging()) {
                     page = PageInfo.pageState(iter).toString();
+                    query.page(page);
                 }
             } while (page != null);
         }

hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/RebuildIndexCallable.java

@@ -86,11 +86,6 @@ private void rebuildIndex(SchemaLabel label, Collection<Id> indexLabelIds) {
         Consumer<?> indexUpdater = (elem) -> {
             for (Id id : indexLabelIds) {
                 graphTx.updateIndex(id, (HugeElement) elem);
-                /*
-                 * Commit per batch to avoid too much data in single commit,
-                 * especially for Cassandra backend
-                 */
-                graphTx.commitIfGtSize(GraphTransaction.COMMIT_BATCH);
             }
         };
 

hugegraph-dist/src/assembly/static/conf/gremlin-server.yaml

@@ -20,7 +20,8 @@ scriptEngines: {
         classImports: [
           java.lang.Math,
           com.baidu.hugegraph.util.DateUtil,
-          com.baidu.hugegraph.traversal.optimize.Text
+          com.baidu.hugegraph.traversal.optimize.Text,
+          com.baidu.hugegraph.traversal.optimize.TraversalUtil
         ],
         methodImports: [java.lang.Math#*]
       },

hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeLabelCoreTest.java

@@ -22,9 +22,11 @@
 import java.util.Date;
 import java.util.List;
 
+import org.apache.tinkerpop.gremlin.process.traversal.P;
 import org.apache.tinkerpop.gremlin.structure.Edge;
 import org.apache.tinkerpop.gremlin.structure.T;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.junit.Assume;
 import org.junit.Test;
 
 import com.baidu.hugegraph.HugeException;
@@ -433,7 +435,7 @@ public void testAddEdgeLabelWithEnableLabelIndex() {
     }
 
     @Test
-    public void testAddEdgeLabelWithDisableeLabelIndex() {
+    public void testAddEdgeLabelWithDisableLabelIndex() {
         super.initPropertyKeys();
         HugeGraph graph =  graph();
         SchemaManager schema = graph.schema();
@@ -873,6 +875,65 @@ public void testRemoveEdgeLabelWithEdgeAndSecondaryIndex() {
         });
     }
 
+    @Test
+    public void testRebuildIndexOfEdgeLabelWithoutLabelIndex() {
+        Assume.assumeFalse("Support query by label",
+                           storeFeatures().supportsQueryByLabel());
+
+        initDataWithoutLabelIndex();
+
+        // Not support query by label
+        Assert.assertThrows(NoIndexException.class, () -> {
+            graph().traversal().E().hasLabel("read").toList();
+        }, e -> {
+            Assert.assertTrue(
+                   e.getMessage().startsWith("Don't accept query by label") &&
+                   e.getMessage().endsWith("it disables label index"));
+        });
+
+        // Query by property index is ok
+        List<Edge> edges = graph().traversal().E()
+                                  .has("date", P.lt("2019-12-30 13:00:00"))
+                                  .toList();
+        Assert.assertEquals(20, edges.size());
+
+        graph().schema().edgeLabel("read").rebuildIndex();
+
+        edges = graph().traversal().E()
+                       .has("date", P.lt("2019-12-30 13:00:00")).toList();
+        Assert.assertEquals(20, edges.size());
+    }
+
+    @Test
+    public void testRemoveEdgeLabelWithoutLabelIndex() {
+        Assume.assumeFalse("Support query by label",
+                           storeFeatures().supportsQueryByLabel());
+
+        initDataWithoutLabelIndex();
+
+        // Not support query by label
+        Assert.assertThrows(NoIndexException.class, () -> {
+            graph().traversal().E().hasLabel("read").toList();
+        }, e -> {
+            Assert.assertTrue(
+                   e.getMessage().startsWith("Don't accept query by label") &&
+                   e.getMessage().endsWith("it disables label index"));
+        });
+
+        // Query by property index is ok
+        List<Edge> edges = graph().traversal().E()
+                                  .has("date", P.lt("2019-12-30 13:00:00"))
+                                  .toList();
+        Assert.assertEquals(20, edges.size());
+
+        graph().schema().edgeLabel("read").remove();
+
+        Assert.assertThrows(NoIndexException.class, () ->
+                graph().traversal().E()
+                       .has("date", P.lt("2019-12-30 13:00:00")).toList()
+        );
+    }
+
     @Test
     public void testAddEdgeLabelWithUserdata() {
         super.initPropertyKeys();

hugegraph-test/src/main/java/com/baidu/hugegraph/core/IndexLabelCoreTest.java

@@ -1401,6 +1401,121 @@ public void testRebuildIndexLabelOfEdge() {
         Assert.assertNotNull(edge);
     }
 
+    @Test
+    public void testRebuildIndexOfVertexWithoutLabelIndex() {
+        Assume.assumeFalse("Support query by label",
+                           storeFeatures().supportsQueryByLabel());
+
+        initDataWithoutLabelIndex();
+
+        // Not support query by label
+        Assert.assertThrows(NoIndexException.class, () -> {
+            graph().traversal().V().hasLabel("reader").toList();
+        }, e -> {
+            Assert.assertTrue(
+                   e.getMessage().startsWith("Don't accept query by label") &&
+                   e.getMessage().endsWith("it disables label index"));
+        });
+
+        // Query by property index is ok
+        List<Vertex> vertices = graph().traversal().V()
+                                       .has("city", "Shanghai").toList();
+        Assert.assertEquals(10, vertices.size());
+
+        graph().schema().indexLabel("readerByCity").rebuild();
+
+        vertices = graph().traversal().V()
+                          .has("city", "Shanghai").toList();
+        Assert.assertEquals(10, vertices.size());
+    }
+
+    @Test
+    public void testRebuildIndexOfEdgeWithoutLabelIndex() {
+        Assume.assumeFalse("Support query by label",
+                           storeFeatures().supportsQueryByLabel());
+
+        initDataWithoutLabelIndex();
+
+        // Not support query by label
+        Assert.assertThrows(NoIndexException.class, () -> {
+            graph().traversal().E().hasLabel("read").toList();
+        }, e -> {
+            Assert.assertTrue(
+                   e.getMessage().startsWith("Don't accept query by label") &&
+                   e.getMessage().endsWith("it disables label index"));
+        });
+
+        // Query by property index is ok
+        List<Edge> edges = graph().traversal().E()
+                                  .has("date", P.lt("2019-12-30 13:00:00"))
+                                  .toList();
+        Assert.assertEquals(20, edges.size());
+
+        graph().schema().indexLabel("readByDate").rebuild();
+
+        edges = graph().traversal().E()
+                       .has("date", P.lt("2019-12-30 13:00:00")).toList();
+        Assert.assertEquals(20, edges.size());
+    }
+
+    @Test
+    public void testRemoveIndexLabelOfVertexWithoutLabelIndex() {
+        Assume.assumeFalse("Support query by label",
+                           storeFeatures().supportsQueryByLabel());
+
+        initDataWithoutLabelIndex();
+
+        // Not support query by label
+        Assert.assertThrows(NoIndexException.class, () -> {
+            graph().traversal().V().hasLabel("reader").toList();
+        }, e -> {
+            Assert.assertTrue(
+                   e.getMessage().startsWith("Don't accept query by label") &&
+                   e.getMessage().endsWith("it disables label index"));
+        });
+
+        // Query by property index is ok
+        List<Vertex> vertices = graph().traversal().V()
+                                       .has("city", "Shanghai").toList();
+        Assert.assertEquals(10, vertices.size());
+
+        graph().schema().indexLabel("readerByCity").remove();
+
+        Assert.assertThrows(NoIndexException.class, () ->
+                graph().traversal().V().has("city", "Shanghai").toList()
+        );
+    }
+
+    @Test
+    public void testRemoveIndexLabelOfEdgeWithoutLabelIndex() {
+        Assume.assumeFalse("Support query by label",
+                           storeFeatures().supportsQueryByLabel());
+
+        initDataWithoutLabelIndex();
+
+        // Not support query by label
+        Assert.assertThrows(NoIndexException.class, () -> {
+            graph().traversal().E().hasLabel("read").toList();
+        }, e -> {
+            Assert.assertTrue(
+                   e.getMessage().startsWith("Don't accept query by label") &&
+                   e.getMessage().endsWith("it disables label index"));
+        });
+
+        // Query by property index is ok
+        List<Edge> edges = graph().traversal().E()
+                                  .has("date", P.lt("2019-12-30 13:00:00"))
+                                  .toList();
+        Assert.assertEquals(20, edges.size());
+
+        graph().schema().indexLabel("readByDate").remove();
+
+        Assert.assertThrows(NoIndexException.class, () ->
+                graph().traversal().E()
+                       .has("date", P.lt("2019-12-30 13:00:00")).toList()
+        );
+    }
+
     @Test
     public void testAddIndexLabelWithUserdata() {
         super.initPropertyKeys();

hugegraph-test/src/main/java/com/baidu/hugegraph/core/SchemaCoreTest.java

@@ -19,8 +19,14 @@
 
 package com.baidu.hugegraph.core;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.HugeGraph;
 import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.schema.EdgeLabel;
 import com.baidu.hugegraph.schema.IndexLabel;
@@ -81,4 +87,75 @@ protected void assertNotContainsIl(Collection<Id> ids, String... labels) {
             Assert.assertNull(graph().schemaTransaction().getIndexLabel(label));
         }
     }
+
+    protected void initDataWithoutLabelIndex() {
+        HugeGraph graph = graph();
+        SchemaManager schema = graph.schema();
+        initPropertyKeys();
+
+        schema.propertyKey("date")
+              .asDate()
+              .ifNotExist()
+              .create();
+
+        schema.vertexLabel("reader").properties("name", "city", "age")
+              .primaryKeys("name")
+              .enableLabelIndex(false)
+              .ifNotExist()
+              .create();
+
+        schema.indexLabel("readerByCity")
+              .onV("reader")
+              .by("city")
+              .secondary()
+              .ifNotExist()
+              .create();
+
+        schema.vertexLabel("book").properties("name")
+              .primaryKeys("name")
+              .enableLabelIndex(false)
+              .ifNotExist()
+              .create();
+
+        schema.edgeLabel("read")
+              .sourceLabel("reader")
+              .targetLabel("book")
+              .properties("date")
+              .enableLabelIndex(false)
+              .ifNotExist()
+              .create();
+
+        schema.indexLabel("readByDate")
+              .onE("read")
+              .by("date")
+              .range()
+              .ifNotExist()
+              .create();
+
+        String[] cities = {"Beijing Haidian", "Beijing Chaoyang",
+                           "Shanghai", "Nanjing", "Hangzhou"};
+        List<Vertex> sources = new ArrayList<>(50);
+        for (int i = 0; i < 50; i++) {
+            String city = cities[i / 10];
+            sources.add(graph.addVertex(T.label, "reader",
+                                        "name", "source" + i,
+                                        "city", city, "age", 21));
+        }
+
+        List<Vertex> targets = new ArrayList<>(50);
+        for (int i = 0; i < 50; i++) {
+            targets.add(graph.addVertex(T.label, "book",
+                                        "name", "java-book" + i));
+        }
+
+        String[] dates = {"2019-12-30 11:00:00", "2019-12-30 12:00:00",
+                          "2019-12-30 13:00:00", "2019-12-30 14:00:00",
+                          "2019-12-30 15:00:00"};
+        for (int i = 0; i < 50; i++) {
+            String date = dates[i / 10];
+            sources.get(i).addEdge("read", targets.get(i), "date", date);
+        }
+
+        graph.tx().commit();
+    }
 }