Custom kout/kneighbor, multi-node-shortest-path, jaccard-similar and …

…template paths (#1174)

* Add customized kout/kneighbor, template path, multi-node-shortest-path and jaccard similar
* fix OLTP algorithm not check if source/target vertex exist (#1156)
* support customized kout and kneighbor
* support multi node shortest path API
* support template paths api
* support jaccard similars find oltp api
* use multi-thread to accelerate
* big depth and both direction use multi threads, otherwise single threead
* fix shortest path api NLP
* Szzq paths improve (#51)
* template path use concurrent HashSet to save paths in concurrent mode
* support property filter for paths api
* fix configuration not supported with auth (#55)
* template path supports repeat times args (#57)
* paths supports nearest args
* oltp multiple threads reuse
* add multiple thread depth config
* fix group perperty can't be empty
* upgrade api version to 58
* move customized kout/kneighbor to kout/kneighbor
* move jaccard similar into jaccard similarity
* fix kout/kneighbor withVertex not work
* rebase ramtable
* extract common traverser from template paths traversers
* extract path traverser for template paths and collection paths
* normalize traverser

implements: #1173

Change-Id: I85aa1d4274554d65f85a0deb7ac596e65dbb503b

hugegraph-api/pom.xml

@@ -125,7 +125,7 @@
                             </addDefaultSpecificationEntries>
                         </manifest>
                         <manifestEntries>
-                            <Implementation-Version>0.57.0.0</Implementation-Version>
+                            <Implementation-Version>0.58.0.0</Implementation-Version>
                         </manifestEntries>
                     </archive>
                 </configuration>

hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java

@@ -19,6 +19,8 @@
 
 package com.baidu.hugegraph.api.traversers;
 
+import java.util.List;
+
 import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY;
 import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_DEGREE;
 
@@ -45,6 +47,7 @@
 import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.util.Log;
 import com.codahale.metrics.annotation.Timed;
+import com.google.common.collect.ImmutableList;
 
 @Path("graphs/{graph}/traversers/allshortestpaths")
 @Singleton
@@ -81,9 +84,11 @@ public String get(@Context GraphManager manager,
         HugeGraph g = graph(manager, graph);
 
         ShortestPathTraverser traverser = new ShortestPathTraverser(g);
+        List<String> edgeLabels = edgeLabel == null ? ImmutableList.of() :
+                                  ImmutableList.of(edgeLabel);
         HugeTraverser.PathSet paths = traverser.allShortestPaths(
-                                      sourceId, targetId, dir, edgeLabel, depth,
-                                      degree, skipDegree, capacity);
+                                      sourceId, targetId, dir, edgeLabels,
+                                      depth, degree, skipDegree, capacity);
         return manager.serializer(g).writePaths("paths", paths, false);
     }
 }

hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CustomizedCrosspointsAPI.java

@@ -84,7 +84,7 @@ public String post(@Context GraphManager manager,
                   request.capacity, request.limit);
 
         HugeGraph g = graph(manager, graph);
-        Iterator<Vertex> sources = request.sources.sourcesVertices(g);
+        Iterator<Vertex> sources = request.sources.vertices(g);
         List<CustomizedCrosspointsTraverser.PathPattern> patterns;
         patterns = pathPatterns(g, request);
 
@@ -132,7 +132,7 @@ public String post(@Context GraphManager manager,
     private static class CrosspointsRequest {
 
         @JsonProperty("sources")
-        public SourceVertices sources;
+        public Vertices sources;
         @JsonProperty("path_patterns")
         public List<PathPattern> pathPatterns;
         @JsonProperty("capacity")

hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CustomizedPathsAPI.java

@@ -86,7 +86,7 @@ public String post(@Context GraphManager manager,
                   request.withVertex);
 
         HugeGraph g = graph(manager, graph);
-        Iterator<Vertex> sources = request.sources.sourcesVertices(g);
+        Iterator<Vertex> sources = request.sources.vertices(g);
         List<CustomizePathsTraverser.Step> steps = step(g, request);
         boolean sorted = request.sortBy != SortBy.NONE;
 
@@ -129,7 +129,7 @@ private static List<CustomizePathsTraverser.Step> step(HugeGraph graph,
     private static class PathRequest {
 
         @JsonProperty("sources")
-        public SourceVertices sources;
+        public Vertices sources;
         @JsonProperty("steps")
         public List<Step> steps;
         @JsonProperty("sort_by")

hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/FusiformSimilarityAPI.java

@@ -97,7 +97,7 @@ public String post(@Context GraphManager manager,
                   request.groupProperty, request.minGroups);
 
         HugeGraph g = graph(manager, graph);
-        Iterator<Vertex> sources = request.sources.sourcesVertices(g);
+        Iterator<Vertex> sources = request.sources.vertices(g);
         E.checkArgument(sources != null && sources.hasNext(),
                         "The source vertices can't be empty");
         EdgeLabel edgeLabel = request.label == null ?
@@ -125,7 +125,7 @@ public String post(@Context GraphManager manager,
     private static class FusiformSimilarityRequest {
 
         @JsonProperty("sources")
-        public SourceVertices sources;
+        public Vertices sources;
         @JsonProperty("label")
         public String label;
         @JsonProperty("direction")

hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/JaccardSimilarityAPI.java

@@ -19,11 +19,17 @@
 
 package com.baidu.hugegraph.api.traversers;
 
+import java.util.Map;
+
+import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY;
 import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_DEGREE;
+import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_LIMIT;
 
 import javax.inject.Singleton;
+import javax.ws.rs.Consumes;
 import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
+import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
@@ -33,22 +39,25 @@
 import org.slf4j.Logger;
 
 import com.baidu.hugegraph.HugeGraph;
-import com.baidu.hugegraph.api.API;
 import com.baidu.hugegraph.api.graph.EdgeAPI;
 import com.baidu.hugegraph.api.graph.VertexAPI;
 import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.core.GraphManager;
 import com.baidu.hugegraph.server.RestServer;
-import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
+import com.baidu.hugegraph.structure.HugeVertex;
+import com.baidu.hugegraph.traversal.algorithm.EdgeStep;
+import com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser;
 import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.E;
 import com.baidu.hugegraph.util.JsonUtil;
 import com.baidu.hugegraph.util.Log;
 import com.codahale.metrics.annotation.Timed;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.ImmutableMap;
 
 @Path("graphs/{graph}/traversers/jaccardsimilarity")
 @Singleton
-public class JaccardSimilarityAPI extends API {
+public class JaccardSimilarityAPI extends TraverserAPI {
 
     private static final Logger LOG = Log.logger(RestServer.class);
 
@@ -72,10 +81,61 @@ public String get(@Context GraphManager manager,
         Directions dir = Directions.convert(EdgeAPI.parseDirection(direction));
 
         HugeGraph g = graph(manager, graph);
-        HugeTraverser traverser = new HugeTraverser(g);
+        JaccardSimilarTraverser traverser = new JaccardSimilarTraverser(g);
         double similarity = traverser.jaccardSimilarity(sourceId, targetId, dir,
                                                         edgeLabel, degree);
         return JsonUtil.toJson(ImmutableMap.of("jaccard_similarity",
                                                similarity));
     }
+
+    @POST
+    @Timed
+    @Consumes(APPLICATION_JSON)
+    @Produces(APPLICATION_JSON_WITH_CHARSET)
+    public String post(@Context GraphManager manager,
+                       @PathParam("graph") String graph,
+                       Request request) {
+        E.checkArgumentNotNull(request, "The request body can't be null");
+        E.checkArgumentNotNull(request.vertex,
+                               "The source vertex of request can't be null");
+        E.checkArgument(request.step != null,
+                        "The steps of request can't be null");
+        E.checkArgument(request.top >= 0,
+                        "The top must be >= 0, but got: %s", request.top);
+
+        LOG.debug("Graph [{}] get jaccard similars from source vertex '{}', " +
+                  "with step '{}', top '{}' and capacity '{}'",
+                  graph, request.vertex, request.step,
+                  request.top, request.capacity);
+
+        HugeGraph g = graph(manager, graph);
+        Id sourceId = HugeVertex.getIdValue(request.vertex);
+
+        EdgeStep step = step(g, request.step);
+
+        JaccardSimilarTraverser traverser = new JaccardSimilarTraverser(g);
+        Map<Id, Double> results = traverser.jaccardSimilars(sourceId, step,
+                                                            request.top,
+                                                            request.capacity);
+        return manager.serializer(g).writeMap(results);
+    }
+
+    private static class Request {
+
+        @JsonProperty("vertex")
+        public Object vertex;
+        @JsonProperty("step")
+        public TraverserAPI.Step step;
+        @JsonProperty("top")
+        public int top = Integer.valueOf(DEFAULT_LIMIT);
+        @JsonProperty("capacity")
+        public long capacity = Long.valueOf(DEFAULT_CAPACITY);
+
+        @Override
+        public String toString() {
+            return String.format("Request{vertex=%s,step=%s,top=%s," +
+                                 "capacity=%s}", this.vertex, this.step,
+                                 this.top, this.capacity);
+        }
+    }
 }

hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java

@@ -21,35 +21,48 @@
 
 import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_DEGREE;
 import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_ELEMENTS_LIMIT;
+import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT;
 
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Set;
 
 import javax.inject.Singleton;
+import javax.ws.rs.Consumes;
 import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
+import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 
+import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.slf4j.Logger;
 
 import com.baidu.hugegraph.HugeGraph;
-import com.baidu.hugegraph.api.API;
 import com.baidu.hugegraph.api.graph.EdgeAPI;
 import com.baidu.hugegraph.api.graph.VertexAPI;
 import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.backend.query.QueryResults;
 import com.baidu.hugegraph.core.GraphManager;
 import com.baidu.hugegraph.server.RestServer;
+import com.baidu.hugegraph.structure.HugeVertex;
+import com.baidu.hugegraph.traversal.algorithm.KneighborTraverser;
+import com.baidu.hugegraph.traversal.algorithm.EdgeStep;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.E;
 import com.baidu.hugegraph.util.Log;
 import com.codahale.metrics.annotation.Timed;
+import com.fasterxml.jackson.annotation.JsonProperty;
 
 @Path("graphs/{graph}/traversers/kneighbor")
 @Singleton
-public class KneighborAPI extends API {
+public class KneighborAPI extends TraverserAPI {
 
     private static final Logger LOG = Log.logger(RestServer.class);
 
@@ -77,9 +90,98 @@ public String get(@Context GraphManager manager,
 
         HugeGraph g = graph(manager, graph);
 
-        HugeTraverser traverser = new HugeTraverser(g);
+        KneighborTraverser traverser = new KneighborTraverser(g);
         Set<Id> ids = traverser.kneighbor(source, dir, edgeLabel, depth,
                                           degree, limit);
         return manager.serializer(g).writeList("vertices", ids);
     }
+
+    @POST
+    @Timed
+    @Consumes(APPLICATION_JSON)
+    @Produces(APPLICATION_JSON_WITH_CHARSET)
+    public String post(@Context GraphManager manager,
+                       @PathParam("graph") String graph,
+                       Request request) {
+        E.checkArgumentNotNull(request, "The request body can't be null");
+        E.checkArgumentNotNull(request.source,
+                               "The source of request can't be null");
+        E.checkArgument(request.step != null,
+                        "The steps of request can't be null");
+        if (request.countOnly) {
+            E.checkArgument(!request.withVertex && !request.withPath,
+                            "Can't return vertex or path when count only");
+        }
+
+        LOG.debug("Graph [{}] get customized kneighbor from source vertex " +
+                  "'{}', with step '{}', limit '{}', count_only '{}', " +
+                  "with_vertex '{}' and with_path '{}'",
+                  graph, request.source, request.step, request.limit,
+                  request.countOnly, request.withVertex, request.withPath);
+
+        HugeGraph g = graph(manager, graph);
+        Id sourceId = HugeVertex.getIdValue(request.source);
+
+        EdgeStep step = step(g, request.step);
+
+        KneighborTraverser traverser = new KneighborTraverser(g);
+        Set<HugeTraverser.Node> results = traverser.customizedKneighbor(
+                                          sourceId, step, request.maxDepth,
+                                          request.limit);
+
+        Set<Id> neighbors = new HashSet<>();
+        for (HugeTraverser.Node node : results) {
+            neighbors.add(node.id());
+        }
+
+        List<HugeTraverser.Path> paths = new ArrayList<>();
+        if (request.withPath) {
+            for (HugeTraverser.Node node : results) {
+                paths.add(new HugeTraverser.Path(node.path()));
+            }
+        }
+        Iterator<Vertex> iter = QueryResults.emptyIterator();
+        if (request.withVertex) {
+            Set<Id> ids = new HashSet<>();
+            for (HugeTraverser.Node node : results) {
+                ids.add(node.id());
+            }
+            for (HugeTraverser.Path p : paths) {
+                ids.addAll(p.vertices());
+            }
+            if (!ids.isEmpty()) {
+                iter = g.vertices(ids.toArray());
+            }
+        }
+        return manager.serializer(g).writeNodesWithPath("kneighbor", neighbors,
+                                                        paths, iter,
+                                                        request.countOnly);
+    }
+
+    private static class Request {
+
+        @JsonProperty("source")
+        public Object source;
+        @JsonProperty("step")
+        public TraverserAPI.Step step;
+        @JsonProperty("max_depth")
+        public int maxDepth;
+        @JsonProperty("limit")
+        public long limit = Long.valueOf(DEFAULT_PATHS_LIMIT);
+        @JsonProperty("count_only")
+        public boolean countOnly = false;
+        @JsonProperty("with_vertex")
+        public boolean withVertex = false;
+        @JsonProperty("with_path")
+        public boolean withPath = false;
+
+        @Override
+        public String toString() {
+            return String.format("PathRequest{source=%s,step=%s,maxDepth=%s" +
+                                 "limit=%s,countOnly=%s,withVertex=%s," +
+                                 "withPath=%s}", this.source, this.step,
+                                 this.maxDepth, this.limit, this.countOnly,
+                                 this.withVertex, this.withPath);
+        }
+    }
 }