feat: improve index hits

hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java

@@ -76,7 +76,7 @@ public class ConditionQuery extends IdQuery {
 
     private OptimizedType optimizedType = OptimizedType.NONE;
     private ResultsFilter resultsFilter = null;
-    private Element2IndexValueMap element2IndexValueMap = null;
+    private transient Element2IndexValueMap element2IndexValueMap = null;
 
     public ConditionQuery(HugeType resultType) {
         super(resultType);

hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/AbstractTransaction.java

@@ -133,7 +133,7 @@ public Number queryNumber(Query query) {
     }
 
     @Watched(prefix = "tx")
-    public QueryResults<BackendEntry> query(Query query) {
+    protected QueryResults<BackendEntry> query(Query query) {
         LOG.debug("Transaction query: {}", query);
         /*
          * NOTE: it's dangerous if an IdQuery/ConditionQuery is empty

hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java

@@ -729,7 +729,7 @@ private PageIds doIndexQueryOnce(IndexLabel indexLabel,
     }
 
     @Watched(prefix = "index")
-    private Set<MatchedIndex> collectMatchedIndexes(ConditionQuery query) {
+    protected Set<MatchedIndex> collectMatchedIndexes(ConditionQuery query) {
         SchemaTransaction schema = this.params().schemaTransaction();
         Id label = query.condition(HugeKeys.LABEL);
 
@@ -764,7 +764,13 @@ private Set<MatchedIndex> collectMatchedIndexes(ConditionQuery query) {
         Set<MatchedIndex> matchedIndexes = InsertionOrderUtil.newSet();
         for (SchemaLabel schemaLabel : schemaLabels) {
             MatchedIndex index = this.collectMatchedIndex(schemaLabel, query);
-            if (index != null) {
+            if (index == null) {
+                continue;
+            }
+            if (matchedIndexes.contains(index)) {
+                matchedIndexes.stream().filter(m -> m.equals(index))
+                    .findFirst().ifPresent(i-> i.addSchemaLabel(schemaLabel));
+            } else {
                 matchedIndexes.add(index);
             }
         }
@@ -1554,20 +1560,31 @@ protected void removeIndex(IndexLabel indexLabel) {
         this.doRemove(this.serializer.writeIndex(index));
     }
 
-    private static class MatchedIndex {
+    protected static class MatchedIndex {
+
+        private final Set<SchemaLabel> schemaLabels;
+        private final Set<IndexLabel> indexLabels;
 
-        private SchemaLabel schemaLabel;
-        private Set<IndexLabel> indexLabels;
 
         public MatchedIndex(SchemaLabel schemaLabel,
                             Set<IndexLabel> indexLabels) {
-            this.schemaLabel = schemaLabel;
+            //this.schemaLabels = new HashSet<>();
+            this.schemaLabels = new HashSet<>();
+            this.schemaLabels.add(schemaLabel);
             this.indexLabels = indexLabels;
         }
 
-        @SuppressWarnings("unused")
-        public SchemaLabel schemaLabel() {
-            return this.schemaLabel;
+        public MatchedIndex(Set<SchemaLabel> schemaLabels,
+                            Set<IndexLabel> indexLabels) {
+            this.schemaLabels = schemaLabels;
+            this.indexLabels = indexLabels;
+        }
+
+        public Set<SchemaLabel> schemaLabels() {
+            return this.schemaLabels;
+        }
+        public void addSchemaLabel(SchemaLabel schemaLabel) {
+            this.schemaLabels.add(schemaLabel);
         }
 
         public Set<IndexLabel> indexLabels() {

hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java

@@ -29,6 +29,7 @@
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.hugegraph.HugeException;
@@ -56,6 +57,7 @@
 import org.apache.hugegraph.config.CoreOptions;
 import org.apache.hugegraph.config.HugeConfig;
 import org.apache.hugegraph.exception.LimitExceedException;
+import org.apache.hugegraph.exception.NoIndexException;
 import org.apache.hugegraph.exception.NotFoundException;
 import org.apache.hugegraph.iterator.BatchMapperIterator;
 import org.apache.hugegraph.iterator.ExtendableIterator;
@@ -1395,8 +1397,8 @@ private <R> QueryList<R> optimizeQueries(Query query,
         }
 
         boolean supportIn = this.storeFeatures().supportsQueryWithInCondition();
-        for (ConditionQuery cq : ConditionQueryFlatten.flatten(
-                                 (ConditionQuery) query, supportIn)) {
+        for (ConditionQuery cq: ConditionQueryFlatten.flatten(
+            (ConditionQuery) query, supportIn)) {
             // Optimize by sysprop
             Query q = this.optimizeQuery(cq);
             /*
@@ -1405,13 +1407,127 @@ private <R> QueryList<R> optimizeQueries(Query query,
              * 2.index-query result(ids after optimization), which may be empty.
              */
             if (q == null) {
-                queries.add(this.indexQuery(cq), this.batchSize);
+                boolean sys = cq.syspropConditions().size() != 0;
+                if (sys){
+                    queries.add(this.indexQuery(cq), this.batchSize);
+                } else {
+                    excludeOnlyLabelQuery(cq, queries);
+                }
+
             } else if (!q.empty()) {
                 queries.add(q);
             }
         }
         return queries;
     }
+    private boolean hasOlapCondition(ConditionQuery query) {
+        for (Id pk : query.userpropKeys()) {
+            if(this.graph().propertyKey(pk).olap()) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+    // Method to reorder conditions
+    private void excludeOnlyLabelQuery(ConditionQuery cq, QueryList queries) {
+        // 判断是否命中索引
+        Set<GraphIndexTransaction.MatchedIndex> indexes = this.indexTx
+            .collectMatchedIndexes(cq);
+        if (CollectionUtils.isEmpty(indexes)){
+            if (this.hasOlapCondition(cq)) {
+                // 未命中索引且包含olap属性查询，需要throw NoIndexException
+                throw new NoIndexException("Don't accept query " +
+                    "based on properties [olap] that are not indexed in any label");
+            } else {
+                // 走算子下沉
+//                E.checkState(this.indexTx.store().features()
+//                        .supportsFilterInStore(),
+//                    "Must support filter in store");
+                queries.add(cq);
+            }
+        } else {
+            // 命中索引
+            queries.add(this.indexQuery(cq), this.batchSize);
+
+            // 判断是否存在部分数据没有创建索引
+            fillQueryLabel(cq, indexes).stream().forEach(queries::add);
+        }
+    }
+    private List<ConditionQuery> fillQueryLabel(ConditionQuery rawQuery,
+                                                Set<GraphIndexTransaction.MatchedIndex> matchedIndices) {
+        List<ConditionQuery> queries = new ArrayList<>();
+
+        // 如果条件查询中包含olap相关属性查询，暂不做补充查询
+        if(this.hasOlapCondition(rawQuery)) {
+            return queries;
+        }
+
+        Collection<VertexLabel> vertexLabels = this.graph().vertexLabels();
+        Collection<EdgeLabel> edgeLabels = this.graph().edgeLabels();
+
+        Set<SchemaLabel> matchedLabels =
+            matchedIndices.stream()
+                .flatMap(matchedIndex -> matchedIndex.schemaLabels().stream())
+                .collect(Collectors.toSet());
+
+        Id label = rawQuery.condition(HugeKeys.LABEL);
+        if (label == null) {
+            // g.V().has('p', 'condition')...
+            // not g.('xx')
+            if (rawQuery.resultType().isVertex()) {
+                // !rawQuery.containsCondition(HugeKeys.ID))
+                for (VertexLabel vl : vertexLabels) {
+                    if (!vl.hidden() && !matchedLabels.contains(vl)
+                        && vl.properties().containsAll(rawQuery.userpropKeys())) {
+
+                        ConditionQuery newQuery = new ConditionQuery(HugeType.VERTEX);
+                        newQuery = newQuery.copy().eq(HugeKeys.LABEL, vl.id());
+
+                        //当binary——rocksdb查询
+                        //ConditionQuery newQuery = new ConditionQuery(HugeType.PROPERTY_KEY);
+                        //newQuery = newQuery.eq(HugeKeys.LABEL, vl.id());
+                        //newQuery = newQuery.copy().eq(HugeKeys.LABEL, vl.id());
+                        //rawQuery.copy().eq(HugeKeys.LABEL, vl.id());
+                        //rawQuery.eq(HugeKeys.LABEL, vl.id()).copy();
+                        //rawQuery.copy().eq(HugeKeys.LABEL, vl.id());
+                        // 如果指定Label信息， limit offset信息需要移除
+                        //原来代码
+//                        ConditionQuery newQuery =
+//                            rawQuery.copy().eq(HugeKeys.LABEL, vl.id());
+                        newQuery.offset(0L);
+//                        for (ConditionQuery cq1: ConditionQueryFlatten.flatten(
+//                            (ConditionQuery) newQuery)){
+//                            super.query(cq1);
+//                        }
+                        //super.query(newQuery);
+                        queries.add(newQuery);
+                        //queries.add(this.indexQuery(newQuery), this.batchSize);
+                        LOG.debug("Fill vertexlabel {} for {}",
+                            vl.name(), rawQuery);
+                    }
+                }
+            }
+
+            // g.E().has('c', 'condition1')
+            if (rawQuery.resultType().isEdge()) {
+                // !rawQuery.containsCondition(HugeKeys.OWNER_VERTEX))
+                for (EdgeLabel el : edgeLabels) {
+                    if (!el.hidden() && !matchedLabels.contains(el)
+                        && el.properties().containsAll(rawQuery.userpropKeys())) {
+                        ConditionQuery newQuery =
+                            rawQuery.copy().eq(HugeKeys.LABEL, el.id());
+                        newQuery.offset(0L);
+                        queries.add(newQuery);
+                        LOG.debug("Fill edgelabel {} for {}",
+                            el.name(), rawQuery);
+                    }
+                }
+            }
+        }
+
+        return queries;
+    }
 
     private Query optimizeQuery(ConditionQuery query) {
         if (query.idsSize() > 0) {

hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java

@@ -2763,14 +2763,14 @@ public void testQueryEdgesByInvalidSysprop() {
             graph.traversal().E().hasLabel("know").has("ID", id).toList();
         }, e -> {
             Assert.assertContains("Undefined property key: 'ID'",
-                                  e.getMessage());
+                e.getMessage());
         });
 
         Assert.assertThrows(IllegalArgumentException.class, () -> {
             graph.traversal().E().hasLabel("know").has("NAME", "n1").toList();
         }, e -> {
             Assert.assertContains("Undefined property key: 'NAME'",
-                                  e.getMessage());
+                e.getMessage());
         });
 
         Assert.assertThrows(HugeException.class, () -> {
@@ -2780,29 +2780,32 @@ public void testQueryEdgesByInvalidSysprop() {
             graph.edges(query).hasNext();
         }, e -> {
             Assert.assertContains("Not supported querying by id and conditions",
-                                  e.getMessage());
+                e.getMessage());
         });
 
+
+        Id look = graph.schema().getEdgeLabel("look").id();
+        Id score = graph.schema().getPropertyKey("score").id();
         Assert.assertThrows(HugeException.class, () -> {
             ConditionQuery query = new ConditionQuery(HugeType.EDGE);
             query.eq(HugeKeys.LABEL, know);
             query.eq(HugeKeys.NAME, "n1");
             graph.edges(query).hasNext();
         }, e -> {
             Assert.assertContains("Not supported querying edges by",
-                                  e.getMessage());
+                e.getMessage());
             Assert.assertContains("NAME == n1", e.getMessage());
         });
 
         Assert.assertThrows(HugeException.class, () -> {
             ConditionQuery query = new ConditionQuery(HugeType.EDGE);
-            query.eq(HugeKeys.LABEL, know);
+            query.eq(HugeKeys.LABEL, look);
             query.eq(HugeKeys.NAME, "n2");
-            query.query(Condition.eq(IdGenerator.of("fake"), "n3"));
+            query.query(Condition.eq(score, "n3"));
             graph.edges(query).hasNext();
         }, e -> {
             Assert.assertContains("Can't do index query with [",
-                                  e.getMessage());
+                e.getMessage());
             Assert.assertContains("LABEL == ", e.getMessage());
             Assert.assertContains("NAME == n2", e.getMessage());
         });

hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java

@@ -2738,7 +2738,29 @@ public void testQueryAll() {
 
         assertContains(vertices, T.label, "book", "name", "java-1");
     }
+    @Test
+    public void testQueryWithPartIndex() {
+        HugeGraph graph = graph();
+
+        graph.schema().indexLabel("allByAge").range()
+            .onV("person").by("age").create();
+
+        graph.addVertex(T.label, "person", "name", "James",
+            "city", "Beijing", "age", 19,
+            "birth", Utils.date("2013-01-01 00:00:00.000"));
 
+        graph.addVertex(T.label, "author", "id", 1,
+            "name", "James Gosling",  "age", 19,
+            "lived", "San Francisco Bay Area");
+        graph.tx().commit();
+        List<Vertex> vertices =
+            graph.traversal().V().has("age", 19).toList();
+        for (Vertex vertex : vertices) {
+            System.out.println(vertex.id());
+        }
+        System.out.printf("vertices.size() = %d%n", vertices.size());
+        //Assert.assertEquals(2, vertices.size());
+    }
     @Test
     public void testQueryAllWithGraphAPI() {
         HugeGraph graph = graph();