From eef7f2f8367f359c2de9922c32a4d7019b3217a7 Mon Sep 17 00:00:00 2001 From: liningrui Date: Fri, 14 Dec 2018 20:51:19 +0800 Subject: [PATCH 01/15] Add PersonalrRank and NeighborRank RESTful API Change-Id: I66e4224b5813aa2d5eadbfe540fb9de203db766b --- .../api/traversers/NeighborRankAPI.java | 126 +++++++++++ .../api/traversers/PersonalRankAPI.java | 86 ++++++++ .../traversal/algorithm/RankAlgorithm.java | 199 ++++++++++++++++++ .../com/baidu/hugegraph/example/Example2.java | 113 +++------- .../src/main/resources/hugegraph.properties | 7 +- 5 files changed, 448 insertions(+), 83 deletions(-) create mode 100644 hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java create mode 100644 hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankAlgorithm.java diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java new file mode 100644 index 0000000000..fdbd1830dd --- /dev/null +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java @@ -0,0 +1,126 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.api.traversers; + +import java.util.List; +import java.util.Map; + +import javax.inject.Singleton; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; + +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; + +import com.baidu.hugegraph.HugeException; +import com.baidu.hugegraph.HugeGraph; +import com.baidu.hugegraph.api.API; +import com.baidu.hugegraph.api.graph.VertexAPI; +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.core.GraphManager; +import com.baidu.hugegraph.schema.EdgeLabel; +import com.baidu.hugegraph.schema.VertexLabel; +import com.baidu.hugegraph.server.RestServer; +import com.baidu.hugegraph.traversal.algorithm.RankAlgorithm; +import com.baidu.hugegraph.type.define.Directions; +import com.baidu.hugegraph.util.CollectionUtil; +import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.InsertionOrderUtil; +import com.baidu.hugegraph.util.JsonUtil; +import com.baidu.hugegraph.util.Log; +import com.codahale.metrics.annotation.Timed; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +@Path("graphs/{graph}/traversers/neighborrank") +@Singleton +public class NeighborRankAPI extends API { + + private static final Logger LOG = Log.logger(RestServer.class); + + @GET + @Timed + @Produces(APPLICATION_JSON_WITH_CHARSET) + public String neighborRank(@Context GraphManager manager, + @PathParam("graph") String graph, + @QueryParam("source") String source, + @QueryParam("label_dirs") String labelDirs, + @QueryParam("alpha") double alpha, + @QueryParam("max_depth") int depth, + @QueryParam("sort_result") @DefaultValue("true") + boolean sortResult) { + LOG.debug("Graph [{}] get neighbor rank from '{}' with " + + "edge label '{}', alpha '{}', max depth '{}'", + graph, source, labelDirs, alpha, depth); + + E.checkNotNull(source, "source vertex id"); + E.checkNotNull(labelDirs, "label dirs"); + E.checkArgument(alpha >= 0.0 && alpha <= 1.0, + "The alpha must between [0, 1], but got '%s'", alpha); + E.checkArgument(depth >= 1, + "The max depth must >= 1, but got '%s'", depth); + + Id sourceId = VertexAPI.checkAndParseVertexId(source); + HugeGraph g = graph(manager, graph); + + List> dirs = this.parseLabelDirs(g, labelDirs); + E.checkArgument(depth <= dirs.size(), + "The label-dir pair size must equal with depth"); + + RankAlgorithm algorithm = new RankAlgorithm(g, alpha, depth); + Map ranks = algorithm.neighborRank(sourceId, dirs); + if (sortResult) { + ranks = CollectionUtil.sortByValue(ranks, false); + } + return JsonUtil.toJson(ranks); + } + + // TODO: 先假设labelDirs是有值的 + private List> parseLabelDirs(HugeGraph graph, + String labelDirStr) { + if (labelDirStr == null) { + return null; + } + if (labelDirStr.isEmpty()) { + return ImmutableList.of(); + } + try { + Map labelDirMap = JsonUtil.fromJson(labelDirStr, Map.class); + List> labelDirs = InsertionOrderUtil.newList(); + for (Map.Entry entry : labelDirMap.entrySet()) { + EdgeLabel label = graph.edgeLabel(entry.getKey()); + Directions dir = Directions.valueOf(entry.getValue().toString()); + labelDirs.add(Pair.of(label.id(), dir)); + } + return labelDirs; + } catch (Exception e) { + throw new HugeException("解析labelDirs失败", e); + } + // If properties is the string "null", props will be null +// E.checkArgument(dirs != null, "Invalid request with none properties"); +// return dirs; + } +} diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java new file mode 100644 index 0000000000..54eb6abb13 --- /dev/null +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java @@ -0,0 +1,86 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.api.traversers; + +import java.util.Map; + +import javax.inject.Singleton; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; + +import org.slf4j.Logger; + +import com.baidu.hugegraph.HugeGraph; +import com.baidu.hugegraph.api.API; +import com.baidu.hugegraph.api.graph.VertexAPI; +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.core.GraphManager; +import com.baidu.hugegraph.server.RestServer; +import com.baidu.hugegraph.traversal.algorithm.RankAlgorithm; +import com.baidu.hugegraph.util.CollectionUtil; +import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.JsonUtil; +import com.baidu.hugegraph.util.Log; +import com.codahale.metrics.annotation.Timed; + +@Path("graphs/{graph}/traversers/personalrank") +@Singleton +public class PersonalRankAPI extends API { + + private static final Logger LOG = Log.logger(RestServer.class); + + @GET + @Timed + @Produces(APPLICATION_JSON_WITH_CHARSET) + public String personalRank(@Context GraphManager manager, + @PathParam("graph") String graph, + @QueryParam("source") String source, + @QueryParam("label") String edgeLabel, + @QueryParam("alpha") double alpha, + @QueryParam("max_depth") int depth, + @QueryParam("sort_result") @DefaultValue("true") + boolean sortResult) { + LOG.debug("Graph [{}] get personal rank from '{}' with " + + "edge label '{}', alpha '{}', max depth '{}'", + graph, source, edgeLabel, alpha, depth); + + E.checkNotNull(source, "source vertex id"); + E.checkNotNull(edgeLabel, "edge label"); + E.checkArgument(alpha >= 0.0 && alpha <= 1.0, + "The alpha must between [0, 1], but got '%s'", alpha); + E.checkArgument(depth >= 1, + "The max depth must >= 1, but got '%s'", depth); + + Id sourceId = VertexAPI.checkAndParseVertexId(source); + HugeGraph g = graph(manager, graph); + + RankAlgorithm algorithm = new RankAlgorithm(g, alpha, depth); + Map ranks = algorithm.personalRank(sourceId, edgeLabel); + if (sortResult) { + ranks = CollectionUtil.sortByValue(ranks, false); + } + return JsonUtil.toJson(ranks); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankAlgorithm.java new file mode 100644 index 0000000000..e434ebdafb --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankAlgorithm.java @@ -0,0 +1,199 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiFunction; + +import org.apache.commons.lang3.tuple.Pair; + +import com.baidu.hugegraph.HugeGraph; +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.schema.EdgeLabel; +import com.baidu.hugegraph.schema.VertexLabel; +import com.baidu.hugegraph.structure.HugeVertex; +import com.baidu.hugegraph.traversal.optimize.HugeTraverser; +import com.baidu.hugegraph.type.define.Directions; +import com.baidu.hugegraph.util.E; +import com.google.common.collect.ImmutableMap; + +public class RankAlgorithm extends HugeTraverser { + + private final double alpha; + private final long maxDepth; + + public RankAlgorithm(HugeGraph graph, double alpha, long maxDepth) { + super(graph); + this.alpha = alpha; + this.maxDepth = maxDepth; + } + + public Map personalRank(Id source, String label) { + Id labelId = this.graph().edgeLabel(label).id(); + Directions dir = this.getStartDirection(source, label); + long degree = this.degreeOfVertex(source, dir, labelId); + if (degree < 1) { + return ImmutableMap.of(source, 1.0); + } + + long depth = 0; + Set outSeeds = new HashSet<>(); + Set inSeeds = new HashSet<>(); + if (dir == Directions.OUT) { + outSeeds.add(source); + } else { + inSeeds.add(source); + } + Map ranks = new HashMap<>(); + ranks.put(source, 1.0); + while (++depth <= this.maxDepth) { + Map incrRanks = this.getIncrRanks(outSeeds, inSeeds, + labelId, ranks); + ranks = this.compensateSourceVertex(source, incrRanks); + } + return ranks; + } + + private Map compensateSourceVertex(Id source, + Map incrRanks) { + double sourceRank = incrRanks.getOrDefault(source, 0.0); + sourceRank += (1 - this.alpha); + incrRanks.put(source, sourceRank); + return incrRanks; + } + + public Map neighborRank(Id source, List> labelDirs) { +// Id labelId = this.graph().edgeLabel(label).id(); +// Directions dir = this.getStartDirection(source, label); +// long degree = this.degreeOfVertex(source, dir, labelId); +// if (degree < 1) { +// return ImmutableMap.of(source, 1.0); +// } + + Set seeds = new HashSet<>(); + seeds.add(source); + Map ranks = new HashMap<>(); + ranks.put(source, 1.0); + for (int depth = 0; depth < this.maxDepth; depth++) { + Pair pair = labelDirs.get(depth); + Id labelId = pair.getKey(); + Directions dir = pair.getValue(); + + Map incrRanks = this.getIncrRanks(seeds, dir, + labelId, ranks); + this.combineIncrement(seeds, ranks, incrRanks); + seeds = incrRanks.keySet(); + } + return ranks; + } + + private void combineIncrement(Set seeds, Map ranks, + Map incrRanks) { + // Update the rank of the neighbor vertices + for (Map.Entry entry : incrRanks.entrySet()) { + double oldRank = ranks.getOrDefault(entry.getKey(), 0.0); + double incrRank = entry.getValue(); + double newRank = oldRank + incrRank; + ranks.put(entry.getKey(), newRank); + } +// // The rank of each seed vertex is attenuated to original * (1 - alpha) +// for (Id seed : seeds) { +// double oldRank = ranks.get(seed); +// double newRank = oldRank * (1 - alpha); +// ranks.put(seed, newRank); +// } + } + + private Map getIncrRanks(Set outSeeds, Set inSeeds, + Id label, Map ranks) { + Map incrRanks = new HashMap<>(); + BiFunction, Directions, Set> neighborIncrRanks = (seeds, dir) -> { + Set tmpSeeds = new HashSet<>(); + for (Id seed : seeds) { + long degree = this.degreeOfVertex(seed, dir, label); + assert degree > 0; + // Must be exist + double originRank = ranks.get(seed); + double spreadRank = originRank * alpha / degree; + + Set neighbors = this.adjacentVertices(seed, dir, label); + // Collect all neighbors increment + for (Id neighbor : neighbors) { + tmpSeeds.add(neighbor); + // Assign an initial value when firstly update neighbor rank + double incrRank = incrRanks.getOrDefault(neighbor, 0.0); + incrRank += spreadRank; + incrRanks.put(neighbor, incrRank); + } + } + return tmpSeeds; + }; + + Set tmpInSeeds = neighborIncrRanks.apply(outSeeds, Directions.OUT); + Set tmpOutSeeds = neighborIncrRanks.apply(inSeeds, Directions.IN); + + outSeeds.addAll(tmpOutSeeds); + inSeeds.addAll(tmpInSeeds); + return incrRanks; + } + + private Map getIncrRanks(Set seeds, Directions dir, + Id label, Map ranks) { + Map rankIncrs = new HashMap<>(); + for (Id seed : seeds) { + long degree = this.degreeOfVertex(seed, dir, label); + assert degree > 0; + // Must be exist + double originRank = ranks.get(seed); + double spreadRank = originRank * alpha / degree; + + Set neighbors = this.adjacentVertices(seed, dir, label); + // Collect all neighbors increment + for (Id neighbor : neighbors) { + // Assign an initial value when firstly update neighbor rank + double incrRank = rankIncrs.getOrDefault(neighbor, 0.0); + incrRank += spreadRank; + rankIncrs.put(neighbor, incrRank); + } + } + return rankIncrs; + } + + private Directions getStartDirection(Id source, String label) { + // NOTE: The outer layer needs to ensure that the vertex Id is valid + HugeVertex vertex = (HugeVertex) graph().vertices(source).next(); + VertexLabel vertexLabel = vertex.schemaLabel(); + EdgeLabel edgeLabel = this.graph().edgeLabel(label); + E.checkArgument(edgeLabel.linkWithLabel(vertexLabel.id()), + "The vertex '%s' doesn't link with edge label '%s'", + source, label); + + if (edgeLabel.sourceLabel().equals(vertexLabel.id())) { + return Directions.OUT; + } else { + assert edgeLabel.targetLabel().equals(vertexLabel.id()); + return Directions.IN; + } + } +} diff --git a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example2.java b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example2.java index aa2a958091..be57e6d85f 100644 --- a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example2.java +++ b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example2.java @@ -45,7 +45,7 @@ public static void main(String[] args) throws InterruptedException { HugeGraph graph = ExampleUtil.loadGraph(); Example2.load(graph); - traversal(graph); +// traversal(graph); graph.close(); @@ -151,99 +151,56 @@ public static void load(final HugeGraph graph) { schema.propertyKey("price").asInt().ifNotExist().create(); schema.vertexLabel("person") - .properties("name", "age", "city") - .primaryKeys("name") - .nullableKeys("age") + .properties("name") + .useCustomizeStringId() .ifNotExist() .create(); - schema.vertexLabel("software") - .properties("name", "lang", "price") - .primaryKeys("name") - .nullableKeys("price") + schema.vertexLabel("movie") + .properties("name") + .useCustomizeStringId() .ifNotExist() .create(); - schema.indexLabel("personByCity") - .onV("person") - .by("city") - .secondary() - .ifNotExist() - .create(); - - schema.indexLabel("personByAgeAndCity") - .onV("person") - .by("age", "city") - .secondary() - .ifNotExist() - .create(); - - schema.indexLabel("softwareByPrice") - .onV("software") - .by("price") - .range() - .ifNotExist() - .create(); - - schema.edgeLabel("knows") - .multiTimes() + schema.edgeLabel("follow") .sourceLabel("person") .targetLabel("person") - .properties("date", "weight") - .sortKeys("date") - .nullableKeys("weight") .ifNotExist() .create(); - schema.edgeLabel("created") - .sourceLabel("person").targetLabel("software") - .properties("date", "weight") - .nullableKeys("weight") - .ifNotExist() - .create(); - - schema.indexLabel("createdByDate") - .onE("created") - .by("date") - .secondary() - .ifNotExist() - .create(); - - schema.indexLabel("createdByWeight") - .onE("created") - .by("weight") - .range() - .ifNotExist() - .create(); - - schema.indexLabel("knowsByWeight") - .onE("knows") - .by("weight") - .range() + schema.edgeLabel("like") + .sourceLabel("person") + .targetLabel("movie") .ifNotExist() .create(); graph.tx().open(); - Vertex marko = graph.addVertex(T.label, "person", "name", "marko", - "age", 29, "city", "Beijing"); - Vertex vadas = graph.addVertex(T.label, "person", "name", "vadas", - "age", 27, "city", "Hongkong"); - Vertex lop = graph.addVertex(T.label, "software", "name", "lop", - "lang", "java", "price", 328); - Vertex josh = graph.addVertex(T.label, "person", "name", "josh", - "age", 32, "city", "Beijing"); - Vertex ripple = graph.addVertex(T.label, "software", "name", "ripple", - "lang", "java", "price", 199); - Vertex peter = graph.addVertex(T.label, "person", "name", "peter", - "age", 35, "city", "Shanghai"); - - marko.addEdge("knows", vadas, "date", "20160110", "weight", 0.5); - marko.addEdge("knows", josh, "date", "20130220", "weight", 1.0); - marko.addEdge("created", lop, "date", "20171210", "weight", 0.4); - josh.addEdge("created", lop, "date", "20091111", "weight", 0.4); - josh.addEdge("created", ripple, "date", "20171210", "weight", 1.0); - peter.addEdge("created", lop, "date", "20170324", "weight", 0.2); + Vertex A = graph.addVertex(T.label, "person", T.id, "A", "name", "A"); + Vertex B = graph.addVertex(T.label, "person", T.id, "B", "name", "B"); + Vertex C = graph.addVertex(T.label, "person", T.id, "C", "name", "C"); + Vertex a = graph.addVertex(T.label, "person", T.id, "a", "name", "a"); + Vertex b = graph.addVertex(T.label, "person", T.id, "b", "name", "b"); + Vertex c = graph.addVertex(T.label, "person", T.id, "c", "name", "c"); + Vertex d = graph.addVertex(T.label, "person", T.id, "d", "name", "d"); + + Vertex m1 = graph.addVertex(T.label, "movie", T.id, "m1", "name", "m1"); + Vertex m2 = graph.addVertex(T.label, "movie", T.id, "m2", "name", "m2"); + Vertex m3 = graph.addVertex(T.label, "movie", T.id, "m3", "name", "m3"); + + A.addEdge("follow", a); + A.addEdge("follow", c); + B.addEdge("follow", a); + B.addEdge("follow", b); + B.addEdge("follow", c); + B.addEdge("follow", d); + C.addEdge("follow", c); + C.addEdge("follow", d); + + a.addEdge("like", m1); + b.addEdge("like", m1); + c.addEdge("like", m2); + d.addEdge("like", m3); graph.tx().commit(); } diff --git a/hugegraph-example/src/main/resources/hugegraph.properties b/hugegraph-example/src/main/resources/hugegraph.properties index fa596fb7f5..0dfc92aecd 100644 --- a/hugegraph-example/src/main/resources/hugegraph.properties +++ b/hugegraph-example/src/main/resources/hugegraph.properties @@ -1,10 +1,7 @@ gremlin.graph=com.baidu.hugegraph.HugeFactory -backend=cassandra -serializer=cassandra - -#backend=rocksdb -#serializer=binary +backend=rocksdb +serializer=binary store=hugegraph rate_limit=0 From de15f5311ce78f719f322affafa2ef0c2b0e61f2 Mon Sep 17 00:00:00 2001 From: liningrui Date: Tue, 16 Apr 2019 11:42:03 +0800 Subject: [PATCH 02/15] Code optimization Change-Id: I8cb701940a323883796c6cbc60dd8b17d4f6ba88 --- .../api/traversers/NeighborRankAPI.java | 156 +++++++---- .../api/traversers/PersonalRankAPI.java | 86 ------ .../traversal/algorithm/RankAlgorithm.java | 199 -------------- .../traversal/algorithm/RankTraverser.java | 258 ++++++++++++++++++ .../baidu/hugegraph/util/OrderLimitMap.java | 98 +++++++ .../com/baidu/hugegraph/example/Example2.java | 113 +++++--- .../com/baidu/hugegraph/example/Example3.java | 130 +++++++++ .../src/main/resources/hugegraph.properties | 7 +- .../baidu/hugegraph/unit/UnitTestSuite.java | 2 + .../unit/core/OrderLimitMapTest.java | 135 +++++++++ 10 files changed, 803 insertions(+), 381 deletions(-) delete mode 100644 hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java delete mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankAlgorithm.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankTraverser.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/util/OrderLimitMap.java create mode 100644 hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java create mode 100644 hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/OrderLimitMapTest.java diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java index fdbd1830dd..dff6b99c24 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java @@ -19,41 +19,40 @@ package com.baidu.hugegraph.api.traversers; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_DEGREE; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.NO_LIMIT; +import static com.baidu.hugegraph.traversal.algorithm.RankTraverser.Step.MAX_TOP; + +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import javax.inject.Singleton; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; -import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; -import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.api.API; import com.baidu.hugegraph.api.graph.VertexAPI; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.core.GraphManager; import com.baidu.hugegraph.schema.EdgeLabel; -import com.baidu.hugegraph.schema.VertexLabel; import com.baidu.hugegraph.server.RestServer; -import com.baidu.hugegraph.traversal.algorithm.RankAlgorithm; +import com.baidu.hugegraph.traversal.algorithm.RankTraverser; import com.baidu.hugegraph.type.define.Directions; -import com.baidu.hugegraph.util.CollectionUtil; import com.baidu.hugegraph.util.E; -import com.baidu.hugegraph.util.InsertionOrderUtil; import com.baidu.hugegraph.util.JsonUtil; import com.baidu.hugegraph.util.Log; import com.codahale.metrics.annotation.Timed; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import com.fasterxml.jackson.annotation.JsonProperty; @Path("graphs/{graph}/traversers/neighborrank") @Singleton @@ -61,66 +60,105 @@ public class NeighborRankAPI extends API { private static final Logger LOG = Log.logger(RestServer.class); - @GET + @POST @Timed @Produces(APPLICATION_JSON_WITH_CHARSET) public String neighborRank(@Context GraphManager manager, @PathParam("graph") String graph, - @QueryParam("source") String source, - @QueryParam("label_dirs") String labelDirs, - @QueryParam("alpha") double alpha, - @QueryParam("max_depth") int depth, - @QueryParam("sort_result") @DefaultValue("true") - boolean sortResult) { - LOG.debug("Graph [{}] get neighbor rank from '{}' with " + - "edge label '{}', alpha '{}', max depth '{}'", - graph, source, labelDirs, alpha, depth); - - E.checkNotNull(source, "source vertex id"); - E.checkNotNull(labelDirs, "label dirs"); - E.checkArgument(alpha >= 0.0 && alpha <= 1.0, - "The alpha must between [0, 1], but got '%s'", alpha); - E.checkArgument(depth >= 1, - "The max depth must >= 1, but got '%s'", depth); - - Id sourceId = VertexAPI.checkAndParseVertexId(source); + RankRequest request) { + E.checkArgumentNotNull(request, "The rank request body can't be null"); + E.checkArgumentNotNull(request.source, + "The sources of rank request can't be null"); + E.checkArgument(request.steps != null && !request.steps.isEmpty(), + "The steps of rank request can't be empty"); + E.checkArgument(request.alpha > 0 && request.alpha <= 1.0, + "The alpha of rank request must belong (0, 1], " + + "but got '%s'", request.alpha); + + LOG.debug("Graph [{}] get neighbor rank from '{}' with steps '{}', " + + "alpha '{}', capacity '{}', limit '{}'", + graph, request.source, request.steps, request.alpha, + request.capacity, request.limit); + + Id sourceId = VertexAPI.checkAndParseVertexId(request.source); HugeGraph g = graph(manager, graph); - List> dirs = this.parseLabelDirs(g, labelDirs); - E.checkArgument(depth <= dirs.size(), - "The label-dir pair size must equal with depth"); + List steps = step(g, request); - RankAlgorithm algorithm = new RankAlgorithm(g, alpha, depth); - Map ranks = algorithm.neighborRank(sourceId, dirs); - if (sortResult) { - ranks = CollectionUtil.sortByValue(ranks, false); - } + RankTraverser traverser = new RankTraverser(g, request.alpha); + List> ranks = traverser.neighborRank(sourceId, steps, + request.capacity, + request.limit); return JsonUtil.toJson(ranks); } - // TODO: 先假设labelDirs是有值的 - private List> parseLabelDirs(HugeGraph graph, - String labelDirStr) { - if (labelDirStr == null) { - return null; + private static List step(HugeGraph graph, + RankRequest req) { + List steps = new ArrayList<>(req.steps.size()); + for (Step step : req.steps) { + steps.add(step.jsonToStep(graph)); + } + return steps; + } + + private static class RankRequest { + + @JsonProperty("source") + private String source; + @JsonProperty("steps") + private List steps; + @JsonProperty("alpha") + private double alpha; + @JsonProperty("capacity") + public long capacity = Long.valueOf(DEFAULT_CAPACITY); + @JsonProperty("limit") + public long limit = Long.valueOf(DEFAULT_PATHS_LIMIT); + + @Override + public String toString() { + return String.format("RankRequest{source=%s,steps=%s," + + "alpha=%s,capacity=%s,limit=%s}", + this.source, this.steps, this.alpha, + this.capacity, this.limit); } - if (labelDirStr.isEmpty()) { - return ImmutableList.of(); + } + + private static class Step { + + @JsonProperty("direction") + public Directions direction; + @JsonProperty("labels") + public List labels; + @JsonProperty("degree") + public long degree = Long.valueOf(DEFAULT_DEGREE); + @JsonProperty("top") + public int top = MAX_TOP; + + @Override + public String toString() { + return String.format("Step{direction=%s,labels=%s,degree=%s}", + this.direction, this.labels, this.degree); } - try { - Map labelDirMap = JsonUtil.fromJson(labelDirStr, Map.class); - List> labelDirs = InsertionOrderUtil.newList(); - for (Map.Entry entry : labelDirMap.entrySet()) { - EdgeLabel label = graph.edgeLabel(entry.getKey()); - Directions dir = Directions.valueOf(entry.getValue().toString()); - labelDirs.add(Pair.of(label.id(), dir)); + + private RankTraverser.Step jsonToStep(HugeGraph graph) { + E.checkArgument(this.degree > 0 || this.degree == NO_LIMIT, + "The degree must be > 0, but got: %s", + this.degree); + E.checkArgument(this.degree == NO_LIMIT, + "Degree must be greater than or equal to sample, " + + "but got degree %s", degree); + E.checkArgument(this.top > 0 && this.top <= MAX_TOP, + "The recommended number of each layer cannot " + + "exceed '%s'", MAX_TOP); + Map labelIds = new HashMap<>(); + if (this.labels != null) { + for (String label : this.labels) { + EdgeLabel el = graph.edgeLabel(label); + labelIds.put(el.id(), label); + } } - return labelDirs; - } catch (Exception e) { - throw new HugeException("解析labelDirs失败", e); + return new RankTraverser.Step(this.direction, labelIds, + this.degree, this.top); } - // If properties is the string "null", props will be null -// E.checkArgument(dirs != null, "Invalid request with none properties"); -// return dirs; } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java deleted file mode 100644 index 54eb6abb13..0000000000 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2017 HugeGraph Authors - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.baidu.hugegraph.api.traversers; - -import java.util.Map; - -import javax.inject.Singleton; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Context; - -import org.slf4j.Logger; - -import com.baidu.hugegraph.HugeGraph; -import com.baidu.hugegraph.api.API; -import com.baidu.hugegraph.api.graph.VertexAPI; -import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.core.GraphManager; -import com.baidu.hugegraph.server.RestServer; -import com.baidu.hugegraph.traversal.algorithm.RankAlgorithm; -import com.baidu.hugegraph.util.CollectionUtil; -import com.baidu.hugegraph.util.E; -import com.baidu.hugegraph.util.JsonUtil; -import com.baidu.hugegraph.util.Log; -import com.codahale.metrics.annotation.Timed; - -@Path("graphs/{graph}/traversers/personalrank") -@Singleton -public class PersonalRankAPI extends API { - - private static final Logger LOG = Log.logger(RestServer.class); - - @GET - @Timed - @Produces(APPLICATION_JSON_WITH_CHARSET) - public String personalRank(@Context GraphManager manager, - @PathParam("graph") String graph, - @QueryParam("source") String source, - @QueryParam("label") String edgeLabel, - @QueryParam("alpha") double alpha, - @QueryParam("max_depth") int depth, - @QueryParam("sort_result") @DefaultValue("true") - boolean sortResult) { - LOG.debug("Graph [{}] get personal rank from '{}' with " + - "edge label '{}', alpha '{}', max depth '{}'", - graph, source, edgeLabel, alpha, depth); - - E.checkNotNull(source, "source vertex id"); - E.checkNotNull(edgeLabel, "edge label"); - E.checkArgument(alpha >= 0.0 && alpha <= 1.0, - "The alpha must between [0, 1], but got '%s'", alpha); - E.checkArgument(depth >= 1, - "The max depth must >= 1, but got '%s'", depth); - - Id sourceId = VertexAPI.checkAndParseVertexId(source); - HugeGraph g = graph(manager, graph); - - RankAlgorithm algorithm = new RankAlgorithm(g, alpha, depth); - Map ranks = algorithm.personalRank(sourceId, edgeLabel); - if (sortResult) { - ranks = CollectionUtil.sortByValue(ranks, false); - } - return JsonUtil.toJson(ranks); - } -} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankAlgorithm.java deleted file mode 100644 index e434ebdafb..0000000000 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankAlgorithm.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright 2017 HugeGraph Authors - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.baidu.hugegraph.traversal.algorithm; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.BiFunction; - -import org.apache.commons.lang3.tuple.Pair; - -import com.baidu.hugegraph.HugeGraph; -import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.schema.EdgeLabel; -import com.baidu.hugegraph.schema.VertexLabel; -import com.baidu.hugegraph.structure.HugeVertex; -import com.baidu.hugegraph.traversal.optimize.HugeTraverser; -import com.baidu.hugegraph.type.define.Directions; -import com.baidu.hugegraph.util.E; -import com.google.common.collect.ImmutableMap; - -public class RankAlgorithm extends HugeTraverser { - - private final double alpha; - private final long maxDepth; - - public RankAlgorithm(HugeGraph graph, double alpha, long maxDepth) { - super(graph); - this.alpha = alpha; - this.maxDepth = maxDepth; - } - - public Map personalRank(Id source, String label) { - Id labelId = this.graph().edgeLabel(label).id(); - Directions dir = this.getStartDirection(source, label); - long degree = this.degreeOfVertex(source, dir, labelId); - if (degree < 1) { - return ImmutableMap.of(source, 1.0); - } - - long depth = 0; - Set outSeeds = new HashSet<>(); - Set inSeeds = new HashSet<>(); - if (dir == Directions.OUT) { - outSeeds.add(source); - } else { - inSeeds.add(source); - } - Map ranks = new HashMap<>(); - ranks.put(source, 1.0); - while (++depth <= this.maxDepth) { - Map incrRanks = this.getIncrRanks(outSeeds, inSeeds, - labelId, ranks); - ranks = this.compensateSourceVertex(source, incrRanks); - } - return ranks; - } - - private Map compensateSourceVertex(Id source, - Map incrRanks) { - double sourceRank = incrRanks.getOrDefault(source, 0.0); - sourceRank += (1 - this.alpha); - incrRanks.put(source, sourceRank); - return incrRanks; - } - - public Map neighborRank(Id source, List> labelDirs) { -// Id labelId = this.graph().edgeLabel(label).id(); -// Directions dir = this.getStartDirection(source, label); -// long degree = this.degreeOfVertex(source, dir, labelId); -// if (degree < 1) { -// return ImmutableMap.of(source, 1.0); -// } - - Set seeds = new HashSet<>(); - seeds.add(source); - Map ranks = new HashMap<>(); - ranks.put(source, 1.0); - for (int depth = 0; depth < this.maxDepth; depth++) { - Pair pair = labelDirs.get(depth); - Id labelId = pair.getKey(); - Directions dir = pair.getValue(); - - Map incrRanks = this.getIncrRanks(seeds, dir, - labelId, ranks); - this.combineIncrement(seeds, ranks, incrRanks); - seeds = incrRanks.keySet(); - } - return ranks; - } - - private void combineIncrement(Set seeds, Map ranks, - Map incrRanks) { - // Update the rank of the neighbor vertices - for (Map.Entry entry : incrRanks.entrySet()) { - double oldRank = ranks.getOrDefault(entry.getKey(), 0.0); - double incrRank = entry.getValue(); - double newRank = oldRank + incrRank; - ranks.put(entry.getKey(), newRank); - } -// // The rank of each seed vertex is attenuated to original * (1 - alpha) -// for (Id seed : seeds) { -// double oldRank = ranks.get(seed); -// double newRank = oldRank * (1 - alpha); -// ranks.put(seed, newRank); -// } - } - - private Map getIncrRanks(Set outSeeds, Set inSeeds, - Id label, Map ranks) { - Map incrRanks = new HashMap<>(); - BiFunction, Directions, Set> neighborIncrRanks = (seeds, dir) -> { - Set tmpSeeds = new HashSet<>(); - for (Id seed : seeds) { - long degree = this.degreeOfVertex(seed, dir, label); - assert degree > 0; - // Must be exist - double originRank = ranks.get(seed); - double spreadRank = originRank * alpha / degree; - - Set neighbors = this.adjacentVertices(seed, dir, label); - // Collect all neighbors increment - for (Id neighbor : neighbors) { - tmpSeeds.add(neighbor); - // Assign an initial value when firstly update neighbor rank - double incrRank = incrRanks.getOrDefault(neighbor, 0.0); - incrRank += spreadRank; - incrRanks.put(neighbor, incrRank); - } - } - return tmpSeeds; - }; - - Set tmpInSeeds = neighborIncrRanks.apply(outSeeds, Directions.OUT); - Set tmpOutSeeds = neighborIncrRanks.apply(inSeeds, Directions.IN); - - outSeeds.addAll(tmpOutSeeds); - inSeeds.addAll(tmpInSeeds); - return incrRanks; - } - - private Map getIncrRanks(Set seeds, Directions dir, - Id label, Map ranks) { - Map rankIncrs = new HashMap<>(); - for (Id seed : seeds) { - long degree = this.degreeOfVertex(seed, dir, label); - assert degree > 0; - // Must be exist - double originRank = ranks.get(seed); - double spreadRank = originRank * alpha / degree; - - Set neighbors = this.adjacentVertices(seed, dir, label); - // Collect all neighbors increment - for (Id neighbor : neighbors) { - // Assign an initial value when firstly update neighbor rank - double incrRank = rankIncrs.getOrDefault(neighbor, 0.0); - incrRank += spreadRank; - rankIncrs.put(neighbor, incrRank); - } - } - return rankIncrs; - } - - private Directions getStartDirection(Id source, String label) { - // NOTE: The outer layer needs to ensure that the vertex Id is valid - HugeVertex vertex = (HugeVertex) graph().vertices(source).next(); - VertexLabel vertexLabel = vertex.schemaLabel(); - EdgeLabel edgeLabel = this.graph().edgeLabel(label); - E.checkArgument(edgeLabel.linkWithLabel(vertexLabel.id()), - "The vertex '%s' doesn't link with edge label '%s'", - source, label); - - if (edgeLabel.sourceLabel().equals(vertexLabel.id())) { - return Directions.OUT; - } else { - assert edgeLabel.targetLabel().equals(vertexLabel.id()); - return Directions.IN; - } - } -} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankTraverser.java new file mode 100644 index 0000000000..133326e792 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankTraverser.java @@ -0,0 +1,258 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.ws.rs.core.MultivaluedMap; + +import org.apache.tinkerpop.gremlin.structure.Edge; + +import com.baidu.hugegraph.HugeGraph; +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.structure.HugeEdge; +import com.baidu.hugegraph.type.define.Directions; +import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.OrderLimitMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +public class RankTraverser extends HugeTraverser { + + private final double alpha; + + public RankTraverser(HugeGraph graph, double alpha) { + super(graph); + this.alpha = alpha; + } + + public List> neighborRank(Id source, List steps, + long capacity, long limit) { + E.checkArgument(!steps.isEmpty(), "The steps can't be empty"); + checkCapacity(capacity); + checkLimit(limit); + + MultivaluedMap sources = newMultivalueMap(); + sources.add(source, new Node(source, null)); + + boolean sameLayerTransfer = true; + int stepNum = steps.size(); + int pathCount = 0; + long access = 0; + // Result + List> ranks = new ArrayList<>(stepNum + 1); + ranks.add(ImmutableMap.of(source, 1.0)); + + MultivaluedMap newVertices; + root : for (Step step : steps) { + stepNum--; + newVertices = newMultivalueMap(); + + Map lastLayerRanks = ranks.get(ranks.size() - 1); + Map sameLayerIncrRanks = new HashMap<>(); + MultivaluedMap adjacencies = newMultivalueMap(); + // Traversal vertices of previous level + for (Map.Entry> entry : sources.entrySet()) { + Id vertex = entry.getKey(); + Iterator edges = edgesOfVertex(vertex, step.direction, + step.labels, null, + step.degree); + + long degree = 0L; + Set sameLayerNodes = new HashSet<>(); + Map> prevLayerNodes = new HashMap<>(); + while (edges.hasNext()) { + degree++; + HugeEdge edge = (HugeEdge) edges.next(); + Id target = edge.id().otherVertexId(); + // Determine whether it belongs to the same layer + if (belongToSameLayer(sources, target, sameLayerNodes)) { + continue; + } + /* + * Determine whether it belongs to the previous layers, + * if it belongs to, update the weight, but don't pass + * any more + */ + if (belongToPrevLayers(ranks, target, prevLayerNodes)) { + continue; + } + + for (Node n : entry.getValue()) { + // If have loop, skip target + if (n.contains(target)) { + continue; + } + Node newNode = new Node(target, n); + adjacencies.add(vertex, newNode); + + checkCapacity(capacity, ++access, "neighbor ranks"); + } + } + List adjacency = adjacencies.getOrDefault(vertex, + ImmutableList.of()); + assert degree == sameLayerNodes.size() + prevLayerNodes.size() + + adjacency.size(); + + // Add current node's adjacent nodes + for (Node node : adjacency) { + newVertices.add(node.id(), node); + // Avoid exceeding limit + if (stepNum == 0) { + if (limit != NO_LIMIT && ++pathCount >= limit) { + break root; + } + } + } + double incr = lastLayerRanks.getOrDefault(vertex, 0.0) * + this.alpha / degree; + // Save the contribution of the same layer node(increment) + for (Id node : sameLayerNodes) { + sameLayerIncrRanks.put(node, incr); + } + // Adding contributions to the previous layers + for (Map.Entry> e : prevLayerNodes.entrySet()) { + Map prevLayerRanks = ranks.get(e.getKey()); + for (Id node : e.getValue()) { + double oldRank = prevLayerRanks.get(node); + prevLayerRanks.put(node, oldRank + incr); + } + } + } + + Map newLayerRanks; + if (sameLayerTransfer) { + // First contribute to last layer, then pass to the new layer + this.contributeLastLayer(sameLayerIncrRanks, lastLayerRanks); + newLayerRanks = this.contributeNewLayer(adjacencies, + lastLayerRanks, step); + } else { + // First pass to the new layer, then contribute to last layer + newLayerRanks = this.contributeNewLayer(adjacencies, + lastLayerRanks, step); + this.contributeLastLayer(sameLayerIncrRanks, lastLayerRanks); + } + ranks.add(newLayerRanks); + + // Re-init sources + sources = newVertices; + } + if (stepNum != 0) { + return ImmutableList.of(ImmutableMap.of()); + } + return this.topRanks(ranks, steps); + } + + private boolean belongToSameLayer(MultivaluedMap sources, + Id target, Set sameLayerNodes) { + if (sources.containsKey(target)) { + sameLayerNodes.add(target); + return true; + } else { + return false; + } + } + + private boolean belongToPrevLayers(List> ranks, Id target, + Map> prevLayerNodes) { + for (int i = ranks.size() - 2; i > 0; i--) { + Map prevLayerRanks = ranks.get(i); + if (prevLayerRanks.containsKey(target)) { + Set nodes = prevLayerNodes.computeIfAbsent(i, + k -> new HashSet<>()); + nodes.add(target); + return true; + } + } + return false; + } + + private void contributeLastLayer(Map rankIncrs, + Map lastLayerRanks) { + for (Map.Entry entry : rankIncrs.entrySet()) { + double originRank = lastLayerRanks.get(entry.getKey()); + double incrRank = entry.getValue(); + lastLayerRanks.put(entry.getKey(), originRank + incrRank); + } + } + + private Map contributeNewLayer( + MultivaluedMap adjacencies, + Map lastLayerRanks, + Step step) { + Map newLayerRanks = new OrderLimitMap<>(step.capacity); + for (Map.Entry> entry : adjacencies.entrySet()) { + Id parent = entry.getKey(); + long size = entry.getValue().size(); + for (Node node : entry.getValue()) { + double rank = newLayerRanks.getOrDefault(node.id(), 0.0); + rank += (lastLayerRanks.get(parent) * this.alpha / size); + newLayerRanks.put(node.id(), rank); + } + } + return newLayerRanks; + } + + private List> topRanks(List> ranks, + List steps) { + assert ranks.size() > 0; + List> results = new ArrayList<>(ranks.size()); + // The first layer is root node so skip i=0 + results.add(ranks.get(0)); + for (int i = 1; i < ranks.size(); i++) { + Step step = steps.get(i - 1); + OrderLimitMap origin = (OrderLimitMap) + ranks.get(i); + if (origin.size() > step.top) { + results.add(origin.topN(step.top)); + } else { + results.add(origin); + } + } + return results; + } + + public static class Step { + + public static final int MAX_TOP = 1000; + public static final int DEFAULT_CAPACITY_PER_LAYER = 100000; + + private final Directions direction; + private final Map labels; + private final long degree; + private final int top; + private final int capacity; + + public Step(Directions direction, Map labels, + long degree, int top) { + this.direction = direction; + this.labels = labels; + this.degree = degree; + this.top = top; + this.capacity = DEFAULT_CAPACITY_PER_LAYER; + } + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/OrderLimitMap.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/OrderLimitMap.java new file mode 100644 index 0000000000..fc24816c41 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/OrderLimitMap.java @@ -0,0 +1,98 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.util; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +import com.google.common.base.Functions; +import com.google.common.collect.Ordering; + +/** + * Reference: https://stackoverflow.com/questions/109383/sort-a-mapkey-value-by-values + * TODO: move to common + */ +public class OrderLimitMap, V extends Comparable> + extends TreeMap { + + private final int capacity; + private final Map valueMap; + + public static > Ordering incr() { + return Ordering.from((V o1, V o2) -> o1.compareTo(o2)); + } + + public static > Ordering decr() { + return Ordering.from((V o1, V o2) -> -o1.compareTo(o2)); + } + + public OrderLimitMap(int capacity) { + this(capacity, decr(), new HashMap<>()); + } + + private OrderLimitMap(int capacity, Ordering ordering, + HashMap valueMap) { + /* + * onResultOf: for getting the value for the key from value map + * compound: keep insertion order + */ + super(ordering.onResultOf(Functions.forMap(valueMap)) + .compound(Ordering.natural())); + this.capacity = capacity; + this.valueMap = valueMap; + } + + @Override + public V put(K k, V v) { + if (this.valueMap.containsKey(k)) { + super.remove(k); + } else if (this.valueMap.size() >= this.capacity) { + K key = super.lastKey(); + super.remove(key); + this.valueMap.remove(key); + } + this.valueMap.put(k, v); + return super.put(k, v); + } + + @Override + public V getOrDefault(Object key, V defaultValue) { + return this.valueMap.getOrDefault(key, defaultValue); + } + + @Override + public boolean containsKey(Object key) { + return this.valueMap.containsKey(key); + } + + public Map topN(int n) { + E.checkArgument(n > 0, "'N' Must be positive, but got '%s'", n); + Map subMap = InsertionOrderUtil.newMap(); + int i = 0; + for (Map.Entry entry : this.entrySet()) { + subMap.put(entry.getKey(), entry.getValue()); + if (++i >= n) { + break; + } + } + return subMap; + } +} diff --git a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example2.java b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example2.java index be57e6d85f..aa2a958091 100644 --- a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example2.java +++ b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example2.java @@ -45,7 +45,7 @@ public static void main(String[] args) throws InterruptedException { HugeGraph graph = ExampleUtil.loadGraph(); Example2.load(graph); -// traversal(graph); + traversal(graph); graph.close(); @@ -151,56 +151,99 @@ public static void load(final HugeGraph graph) { schema.propertyKey("price").asInt().ifNotExist().create(); schema.vertexLabel("person") - .properties("name") - .useCustomizeStringId() + .properties("name", "age", "city") + .primaryKeys("name") + .nullableKeys("age") .ifNotExist() .create(); - schema.vertexLabel("movie") - .properties("name") - .useCustomizeStringId() + schema.vertexLabel("software") + .properties("name", "lang", "price") + .primaryKeys("name") + .nullableKeys("price") .ifNotExist() .create(); - schema.edgeLabel("follow") + schema.indexLabel("personByCity") + .onV("person") + .by("city") + .secondary() + .ifNotExist() + .create(); + + schema.indexLabel("personByAgeAndCity") + .onV("person") + .by("age", "city") + .secondary() + .ifNotExist() + .create(); + + schema.indexLabel("softwareByPrice") + .onV("software") + .by("price") + .range() + .ifNotExist() + .create(); + + schema.edgeLabel("knows") + .multiTimes() .sourceLabel("person") .targetLabel("person") + .properties("date", "weight") + .sortKeys("date") + .nullableKeys("weight") .ifNotExist() .create(); - schema.edgeLabel("like") - .sourceLabel("person") - .targetLabel("movie") + schema.edgeLabel("created") + .sourceLabel("person").targetLabel("software") + .properties("date", "weight") + .nullableKeys("weight") + .ifNotExist() + .create(); + + schema.indexLabel("createdByDate") + .onE("created") + .by("date") + .secondary() + .ifNotExist() + .create(); + + schema.indexLabel("createdByWeight") + .onE("created") + .by("weight") + .range() + .ifNotExist() + .create(); + + schema.indexLabel("knowsByWeight") + .onE("knows") + .by("weight") + .range() .ifNotExist() .create(); graph.tx().open(); - Vertex A = graph.addVertex(T.label, "person", T.id, "A", "name", "A"); - Vertex B = graph.addVertex(T.label, "person", T.id, "B", "name", "B"); - Vertex C = graph.addVertex(T.label, "person", T.id, "C", "name", "C"); - Vertex a = graph.addVertex(T.label, "person", T.id, "a", "name", "a"); - Vertex b = graph.addVertex(T.label, "person", T.id, "b", "name", "b"); - Vertex c = graph.addVertex(T.label, "person", T.id, "c", "name", "c"); - Vertex d = graph.addVertex(T.label, "person", T.id, "d", "name", "d"); - - Vertex m1 = graph.addVertex(T.label, "movie", T.id, "m1", "name", "m1"); - Vertex m2 = graph.addVertex(T.label, "movie", T.id, "m2", "name", "m2"); - Vertex m3 = graph.addVertex(T.label, "movie", T.id, "m3", "name", "m3"); - - A.addEdge("follow", a); - A.addEdge("follow", c); - B.addEdge("follow", a); - B.addEdge("follow", b); - B.addEdge("follow", c); - B.addEdge("follow", d); - C.addEdge("follow", c); - C.addEdge("follow", d); - - a.addEdge("like", m1); - b.addEdge("like", m1); - c.addEdge("like", m2); - d.addEdge("like", m3); + Vertex marko = graph.addVertex(T.label, "person", "name", "marko", + "age", 29, "city", "Beijing"); + Vertex vadas = graph.addVertex(T.label, "person", "name", "vadas", + "age", 27, "city", "Hongkong"); + Vertex lop = graph.addVertex(T.label, "software", "name", "lop", + "lang", "java", "price", 328); + Vertex josh = graph.addVertex(T.label, "person", "name", "josh", + "age", 32, "city", "Beijing"); + Vertex ripple = graph.addVertex(T.label, "software", "name", "ripple", + "lang", "java", "price", 199); + Vertex peter = graph.addVertex(T.label, "person", "name", "peter", + "age", 35, "city", "Shanghai"); + + marko.addEdge("knows", vadas, "date", "20160110", "weight", 0.5); + marko.addEdge("knows", josh, "date", "20130220", "weight", 1.0); + marko.addEdge("created", lop, "date", "20171210", "weight", 0.4); + josh.addEdge("created", lop, "date", "20091111", "weight", 0.4); + josh.addEdge("created", ripple, "date", "20171210", "weight", 1.0); + peter.addEdge("created", lop, "date", "20170324", "weight", 0.2); graph.tx().commit(); } diff --git a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java new file mode 100644 index 0000000000..290b77f3c5 --- /dev/null +++ b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java @@ -0,0 +1,130 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.example; + +import org.apache.tinkerpop.gremlin.structure.T; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.slf4j.Logger; + +import com.baidu.hugegraph.HugeGraph; +import com.baidu.hugegraph.schema.SchemaManager; +import com.baidu.hugegraph.util.Log; + +public class Example3 { + + private static final Logger LOG = Log.logger(Example3.class); + + public static void main(String[] args) throws InterruptedException { + LOG.info("Example3 start!"); + + HugeGraph graph = ExampleUtil.loadGraph(); + + Example3.load(graph); + + graph.close(); + + HugeGraph.shutdown(30L); + } + + public static void load(final HugeGraph graph) { + SchemaManager schema = graph.schema(); + + schema.propertyKey("name").asText().ifNotExist().create(); + schema.propertyKey("age").asInt().ifNotExist().create(); + schema.propertyKey("city").asText().ifNotExist().create(); + schema.propertyKey("weight").asDouble().ifNotExist().create(); + schema.propertyKey("lang").asText().ifNotExist().create(); + schema.propertyKey("date").asText().ifNotExist().create(); + schema.propertyKey("price").asInt().ifNotExist().create(); + + schema.vertexLabel("person") + .properties("name") + .useCustomizeStringId() + .ifNotExist() + .create(); + + schema.vertexLabel("movie") + .properties("name") + .useCustomizeStringId() + .ifNotExist() + .create(); + + schema.edgeLabel("follow") + .sourceLabel("person") + .targetLabel("person") + .ifNotExist() + .create(); + + schema.edgeLabel("like") + .sourceLabel("person") + .targetLabel("movie") + .ifNotExist() + .create(); + + schema.edgeLabel("directedby") + .sourceLabel("movie") + .targetLabel("person") + .ifNotExist() + .create(); + + graph.tx().open(); + + Vertex O = graph.addVertex(T.label, "person", T.id, "O", "name", "O"); + + Vertex A = graph.addVertex(T.label, "person", T.id, "A", "name", "A"); + Vertex B = graph.addVertex(T.label, "person", T.id, "B", "name", "B"); + Vertex C = graph.addVertex(T.label, "person", T.id, "C", "name", "C"); + Vertex D = graph.addVertex(T.label, "person", T.id, "D", "name", "D"); + + Vertex E = graph.addVertex(T.label, "movie", T.id, "E", "name", "E"); + Vertex F = graph.addVertex(T.label, "movie", T.id, "F", "name", "F"); + Vertex G = graph.addVertex(T.label, "movie", T.id, "G", "name", "G"); + Vertex H = graph.addVertex(T.label, "movie", T.id, "H", "name", "H"); + Vertex I = graph.addVertex(T.label, "movie", T.id, "I", "name", "I"); + Vertex J = graph.addVertex(T.label, "movie", T.id, "J", "name", "J"); + + Vertex K = graph.addVertex(T.label, "person", T.id, "K", "name", "K"); + Vertex L = graph.addVertex(T.label, "person", T.id, "L", "name", "L"); + Vertex M = graph.addVertex(T.label, "person", T.id, "M", "name", "M"); + + O.addEdge("follow", A); + O.addEdge("follow", B); + O.addEdge("follow", C); + D.addEdge("follow", O); + + A.addEdge("follow", B); + A.addEdge("like", E); + A.addEdge("like", F); + + B.addEdge("like", G); + B.addEdge("like", H); + + C.addEdge("like", I); + C.addEdge("like", J); + + E.addEdge("directedby", K); + F.addEdge("directedby", B); + F.addEdge("directedby", L); + + G.addEdge("directedby", M); + + graph.tx().commit(); + } +} diff --git a/hugegraph-example/src/main/resources/hugegraph.properties b/hugegraph-example/src/main/resources/hugegraph.properties index 0dfc92aecd..fa596fb7f5 100644 --- a/hugegraph-example/src/main/resources/hugegraph.properties +++ b/hugegraph-example/src/main/resources/hugegraph.properties @@ -1,7 +1,10 @@ gremlin.graph=com.baidu.hugegraph.HugeFactory -backend=rocksdb -serializer=binary +backend=cassandra +serializer=cassandra + +#backend=rocksdb +#serializer=binary store=hugegraph rate_limit=0 diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java index 98d0297fbe..8951d01de9 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java @@ -30,6 +30,7 @@ import com.baidu.hugegraph.unit.core.ConditionQueryFlattenTest; import com.baidu.hugegraph.unit.core.EdgeIdTest; import com.baidu.hugegraph.unit.core.JsonUtilTest; +import com.baidu.hugegraph.unit.core.OrderLimitMapTest; import com.baidu.hugegraph.unit.core.VersionTest; import com.baidu.hugegraph.unit.rocksdb.RocksDBCountersTest; import com.baidu.hugegraph.unit.rocksdb.RocksDBSessionsTest; @@ -46,6 +47,7 @@ EdgeIdTest.class, AnalyzerTest.class, JsonUtilTest.class, + OrderLimitMapTest.class, RocksDBSessionsTest.class, RocksDBCountersTest.class diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/OrderLimitMapTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/OrderLimitMapTest.java new file mode 100644 index 0000000000..dbeaf5c1b9 --- /dev/null +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/OrderLimitMapTest.java @@ -0,0 +1,135 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.unit.core; + +import org.junit.Test; + +import com.baidu.hugegraph.testutil.Assert; +import com.baidu.hugegraph.util.OrderLimitMap; +import com.google.common.collect.ImmutableList; + +public class OrderLimitMapTest { + + @Test + public void testOrder() { + OrderLimitMap map = new OrderLimitMap<>(5); + map.put(1, 0.1); + map.put(2, 0.2); + map.put(3, 0.3); + map.put(4, 0.4); + map.put(5, 0.5); + + Assert.assertEquals(5, map.size()); + Assert.assertEquals(ImmutableList.of(5, 4, 3, 2, 1), + ImmutableList.copyOf(map.keySet())); + } + + @Test + public void testOrderWithDupValue() { + OrderLimitMap map = new OrderLimitMap<>(5); + map.put(1, 0.1); + map.put(2, 0.2); + map.put(3, 0.3); + map.put(4, 0.2); + map.put(5, 0.3); + + Assert.assertEquals(5, map.size()); + Assert.assertEquals(ImmutableList.of(3, 5, 2, 4, 1), + ImmutableList.copyOf(map.keySet())); + } + + @Test + public void testOrderWithDupValueAndKeyIncrOrder() { + OrderLimitMap map = new OrderLimitMap<>(5); + map.put(4, 0.2); + map.put(2, 0.2); + map.put(1, 0.1); + map.put(5, 0.3); + map.put(3, 0.3); + + Assert.assertEquals(5, map.size()); + Assert.assertEquals(ImmutableList.of(3, 5, 2, 4, 1), + ImmutableList.copyOf(map.keySet())); + } + + @Test + public void testOrderWithDupKey() { + OrderLimitMap map = new OrderLimitMap<>(5); + map.put(1, 0.1); + map.put(2, 0.2); + map.put(3, 0.3); + map.put(2, 0.4); + map.put(3, 0.2); + + Assert.assertEquals(3, map.size()); + Assert.assertEquals(ImmutableList.of(2, 3, 1), + ImmutableList.copyOf(map.keySet())); + } + + @Test + public void testLimit() { + OrderLimitMap map = new OrderLimitMap<>(5); + map.put(1, 0.1); + map.put(2, 0.2); + map.put(3, 0.3); + map.put(4, 0.4); + map.put(5, 0.5); + + map.put(6, 0.6); + Assert.assertEquals(5, map.size()); + Assert.assertEquals(ImmutableList.of(6, 5, 4, 3, 2), + ImmutableList.copyOf(map.keySet())); + + map.put(7, 0.7); + Assert.assertEquals(5, map.size()); + Assert.assertEquals(ImmutableList.of(7, 6, 5, 4, 3), + ImmutableList.copyOf(map.keySet())); + } + + @Test + public void testLimitWithDupValue() { + OrderLimitMap map = new OrderLimitMap<>(5); + map.put(1, 0.1); + map.put(2, 0.2); + map.put(3, 0.3); + map.put(4, 0.4); + map.put(5, 0.5); + + map.put(6, 0.1); + Assert.assertEquals(5, map.size()); + Assert.assertEquals(ImmutableList.of(5, 4, 3, 2, 6), + ImmutableList.copyOf(map.keySet())); + + map.put(7, 0.3); + Assert.assertEquals(5, map.size()); + Assert.assertEquals(ImmutableList.of(5, 4, 3, 7, 2), + ImmutableList.copyOf(map.keySet())); + + map.put(8, 0.5); + Assert.assertEquals(5, map.size()); + Assert.assertEquals(ImmutableList.of(5, 8, 4, 3, 7), + ImmutableList.copyOf(map.keySet())); + + map.put(0, 0.5); + Assert.assertEquals(5, map.size()); + Assert.assertEquals(ImmutableList.of(0, 5, 8, 4, 3), + ImmutableList.copyOf(map.keySet())); + } +} From 19c49c8d2822d36340c6f4d660203912d73a2c94 Mon Sep 17 00:00:00 2001 From: liningrui Date: Tue, 16 Apr 2019 15:38:29 +0800 Subject: [PATCH 03/15] Encapsulate some methods Change-Id: I8f02506cbc3acfa62072f46e4f2000e5edd8502c --- .../api/traversers/NeighborRankAPI.java | 41 ++++--- .../traversal/algorithm/RankTraverser.java | 108 +++++++++--------- 2 files changed, 76 insertions(+), 73 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java index dff6b99c24..01feb76f44 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java @@ -23,7 +23,8 @@ import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_DEGREE; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.NO_LIMIT; -import static com.baidu.hugegraph.traversal.algorithm.RankTraverser.Step.MAX_TOP; +import static com.baidu.hugegraph.traversal.algorithm.RankTraverser.MAX_STEPS; +import static com.baidu.hugegraph.traversal.algorithm.RankTraverser.MAX_TOP; import java.util.ArrayList; import java.util.HashMap; @@ -71,29 +72,29 @@ public String neighborRank(@Context GraphManager manager, "The sources of rank request can't be null"); E.checkArgument(request.steps != null && !request.steps.isEmpty(), "The steps of rank request can't be empty"); + E.checkArgument(request.steps.size() <= MAX_STEPS, + "The steps length of rank request can't exceed %s", + MAX_STEPS); E.checkArgument(request.alpha > 0 && request.alpha <= 1.0, "The alpha of rank request must belong (0, 1], " + "but got '%s'", request.alpha); LOG.debug("Graph [{}] get neighbor rank from '{}' with steps '{}', " + - "alpha '{}', capacity '{}', limit '{}'", - graph, request.source, request.steps, request.alpha, - request.capacity, request.limit); + "alpha '{}', capacity '{}'", graph, request.source, + request.steps, request.alpha, request.capacity); Id sourceId = VertexAPI.checkAndParseVertexId(request.source); HugeGraph g = graph(manager, graph); - List steps = step(g, request); - - RankTraverser traverser = new RankTraverser(g, request.alpha); - List> ranks = traverser.neighborRank(sourceId, steps, - request.capacity, - request.limit); + List steps = steps(g, request); + RankTraverser traverser = new RankTraverser(g, request.alpha, + request.capacity); + List> ranks = traverser.neighborRank(sourceId, steps); return JsonUtil.toJson(ranks); } - private static List step(HugeGraph graph, - RankRequest req) { + private static List steps(HugeGraph graph, + RankRequest req) { List steps = new ArrayList<>(req.steps.size()); for (Step step : req.steps) { steps.add(step.jsonToStep(graph)); @@ -111,15 +112,12 @@ private static class RankRequest { private double alpha; @JsonProperty("capacity") public long capacity = Long.valueOf(DEFAULT_CAPACITY); - @JsonProperty("limit") - public long limit = Long.valueOf(DEFAULT_PATHS_LIMIT); @Override public String toString() { - return String.format("RankRequest{source=%s,steps=%s," + - "alpha=%s,capacity=%s,limit=%s}", - this.source, this.steps, this.alpha, - this.capacity, this.limit); + return String.format("RankRequest{source=%s,steps=%s,alpha=%s," + + "capacity=%s}", this.source, this.steps, + this.alpha, this.capacity); } } @@ -132,12 +130,13 @@ private static class Step { @JsonProperty("degree") public long degree = Long.valueOf(DEFAULT_DEGREE); @JsonProperty("top") - public int top = MAX_TOP; + public int top = Integer.valueOf(DEFAULT_PATHS_LIMIT); @Override public String toString() { - return String.format("Step{direction=%s,labels=%s,degree=%s}", - this.direction, this.labels, this.degree); + return String.format("Step{direction=%s,labels=%s,degree=%s," + + "top=%s}", this.direction, this.labels, + this.degree, this.top); } private RankTraverser.Step jsonToStep(HugeGraph graph) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankTraverser.java index 133326e792..d82c4663ca 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankTraverser.java @@ -42,38 +42,38 @@ public class RankTraverser extends HugeTraverser { + public static final int MAX_STEPS = 100; + public static final int MAX_TOP = 1000; + public static final int DEFAULT_CAPACITY_PER_LAYER = 100000; + private final double alpha; + private final long capacity; - public RankTraverser(HugeGraph graph, double alpha) { + public RankTraverser(HugeGraph graph, double alpha, long capacity) { super(graph); + checkCapacity(capacity); this.alpha = alpha; + this.capacity = capacity; } - public List> neighborRank(Id source, List steps, - long capacity, long limit) { + public List> neighborRank(Id source, List steps) { + E.checkArgumentNotNull(source, "The source vertex id can't be null"); E.checkArgument(!steps.isEmpty(), "The steps can't be empty"); - checkCapacity(capacity); - checkLimit(limit); MultivaluedMap sources = newMultivalueMap(); sources.add(source, new Node(source, null)); boolean sameLayerTransfer = true; - int stepNum = steps.size(); - int pathCount = 0; long access = 0; // Result - List> ranks = new ArrayList<>(stepNum + 1); + List> ranks = new ArrayList<>(); ranks.add(ImmutableMap.of(source, 1.0)); - MultivaluedMap newVertices; root : for (Step step : steps) { - stepNum--; - newVertices = newMultivalueMap(); - Map lastLayerRanks = ranks.get(ranks.size() - 1); Map sameLayerIncrRanks = new HashMap<>(); MultivaluedMap adjacencies = newMultivalueMap(); + MultivaluedMap newVertices = newMultivalueMap(); // Traversal vertices of previous level for (Map.Entry> entry : sources.entrySet()) { Id vertex = entry.getKey(); @@ -82,14 +82,15 @@ public List> neighborRank(Id source, List steps, step.degree); long degree = 0L; - Set sameLayerNodes = new HashSet<>(); - Map> prevLayerNodes = new HashMap<>(); + Set sameLayerNodesV = new HashSet<>(); + Map> prevLayerNodesV = new HashMap<>(); while (edges.hasNext()) { degree++; HugeEdge edge = (HugeEdge) edges.next(); Id target = edge.id().otherVertexId(); // Determine whether it belongs to the same layer - if (belongToSameLayer(sources, target, sameLayerNodes)) { + if (this.belongToSameLayer(sources.keySet(), target, + sameLayerNodesV)) { continue; } /* @@ -97,7 +98,8 @@ public List> neighborRank(Id source, List steps, * if it belongs to, update the weight, but don't pass * any more */ - if (belongToPrevLayers(ranks, target, prevLayerNodes)) { + if (this.belongToPrevLayers(ranks, target, + prevLayerNodesV)) { continue; } @@ -112,35 +114,22 @@ public List> neighborRank(Id source, List steps, checkCapacity(capacity, ++access, "neighbor ranks"); } } - List adjacency = adjacencies.getOrDefault(vertex, - ImmutableList.of()); - assert degree == sameLayerNodes.size() + prevLayerNodes.size() + - adjacency.size(); + List adjacenciesV = adjacencies.getOrDefault(vertex, + ImmutableList.of()); + assert degree == sameLayerNodesV.size() + + prevLayerNodesV.size() + adjacenciesV.size(); - // Add current node's adjacent nodes - for (Node node : adjacency) { + // Add adjacent nodes of current node to sources of next step + for (Node node : adjacenciesV) { newVertices.add(node.id(), node); - // Avoid exceeding limit - if (stepNum == 0) { - if (limit != NO_LIMIT && ++pathCount >= limit) { - break root; - } - } } double incr = lastLayerRanks.getOrDefault(vertex, 0.0) * this.alpha / degree; - // Save the contribution of the same layer node(increment) - for (Id node : sameLayerNodes) { - sameLayerIncrRanks.put(node, incr); - } + // Merge the increment of the same layer node + this.mergeSameLayerIncrRanks(sameLayerNodesV, incr, + sameLayerIncrRanks); // Adding contributions to the previous layers - for (Map.Entry> e : prevLayerNodes.entrySet()) { - Map prevLayerRanks = ranks.get(e.getKey()); - for (Id node : e.getValue()) { - double oldRank = prevLayerRanks.get(node); - prevLayerRanks.put(node, oldRank + incr); - } - } + this.contributePrevLayers(ranks, incr, prevLayerNodesV); } Map newLayerRanks; @@ -148,11 +137,13 @@ public List> neighborRank(Id source, List steps, // First contribute to last layer, then pass to the new layer this.contributeLastLayer(sameLayerIncrRanks, lastLayerRanks); newLayerRanks = this.contributeNewLayer(adjacencies, - lastLayerRanks, step); + lastLayerRanks, + step.capacity); } else { // First pass to the new layer, then contribute to last layer newLayerRanks = this.contributeNewLayer(adjacencies, - lastLayerRanks, step); + lastLayerRanks, + step.capacity); this.contributeLastLayer(sameLayerIncrRanks, lastLayerRanks); } ranks.add(newLayerRanks); @@ -160,15 +151,12 @@ public List> neighborRank(Id source, List steps, // Re-init sources sources = newVertices; } - if (stepNum != 0) { - return ImmutableList.of(ImmutableMap.of()); - } return this.topRanks(ranks, steps); } - private boolean belongToSameLayer(MultivaluedMap sources, - Id target, Set sameLayerNodes) { - if (sources.containsKey(target)) { + private boolean belongToSameLayer(Set sources, Id target, + Set sameLayerNodes) { + if (sources.contains(target)) { sameLayerNodes.add(target); return true; } else { @@ -190,6 +178,25 @@ private boolean belongToPrevLayers(List> ranks, Id target, return false; } + private void mergeSameLayerIncrRanks(Set sameLayerNodesV, double incr, + Map sameLayerIncrRanks) { + for (Id node : sameLayerNodesV) { + double oldRank = sameLayerIncrRanks.getOrDefault(node, 0.0); + sameLayerIncrRanks.put(node, oldRank + incr); + } + } + + private void contributePrevLayers(List> ranks, double incr, + Map> prevLayerNodesV) { + for (Map.Entry> e : prevLayerNodesV.entrySet()) { + Map prevLayerRanks = ranks.get(e.getKey()); + for (Id node : e.getValue()) { + double oldRank = prevLayerRanks.get(node); + prevLayerRanks.put(node, oldRank + incr); + } + } + } + private void contributeLastLayer(Map rankIncrs, Map lastLayerRanks) { for (Map.Entry entry : rankIncrs.entrySet()) { @@ -202,8 +209,8 @@ private void contributeLastLayer(Map rankIncrs, private Map contributeNewLayer( MultivaluedMap adjacencies, Map lastLayerRanks, - Step step) { - Map newLayerRanks = new OrderLimitMap<>(step.capacity); + int capacity) { + Map newLayerRanks = new OrderLimitMap<>(capacity); for (Map.Entry> entry : adjacencies.entrySet()) { Id parent = entry.getKey(); long size = entry.getValue().size(); @@ -237,9 +244,6 @@ private List> topRanks(List> ranks, public static class Step { - public static final int MAX_TOP = 1000; - public static final int DEFAULT_CAPACITY_PER_LAYER = 100000; - private final Directions direction; private final Map labels; private final long degree; From 2a95cbeaa235bd0c9a79a911333d7bf249e00dcb Mon Sep 17 00:00:00 2001 From: liningrui Date: Tue, 16 Apr 2019 16:50:04 +0800 Subject: [PATCH 04/15] tiny improve Change-Id: Ib66fce20642e89b49a7bf45edb223ab4bd5c27dd --- hugegraph-api/pom.xml | 2 +- .../baidu/hugegraph/api/traversers/NeighborRankAPI.java | 8 ++------ .../main/java/com/baidu/hugegraph/version/ApiVersion.java | 3 ++- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/hugegraph-api/pom.xml b/hugegraph-api/pom.xml index 8ea074ee5e..d90316d514 100644 --- a/hugegraph-api/pom.xml +++ b/hugegraph-api/pom.xml @@ -86,7 +86,7 @@ - 0.37.0.0 + 0.38.0.0 diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java index 01feb76f44..5f5ae0d740 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java @@ -69,7 +69,7 @@ public String neighborRank(@Context GraphManager manager, RankRequest request) { E.checkArgumentNotNull(request, "The rank request body can't be null"); E.checkArgumentNotNull(request.source, - "The sources of rank request can't be null"); + "The source of rank request can't be null"); E.checkArgument(request.steps != null && !request.steps.isEmpty(), "The steps of rank request can't be empty"); E.checkArgument(request.steps.size() <= MAX_STEPS, @@ -143,12 +143,8 @@ private RankTraverser.Step jsonToStep(HugeGraph graph) { E.checkArgument(this.degree > 0 || this.degree == NO_LIMIT, "The degree must be > 0, but got: %s", this.degree); - E.checkArgument(this.degree == NO_LIMIT, - "Degree must be greater than or equal to sample, " + - "but got degree %s", degree); E.checkArgument(this.top > 0 && this.top <= MAX_TOP, - "The recommended number of each layer cannot " + - "exceed '%s'", MAX_TOP); + "The top of each layer cannot exceed %s", MAX_TOP); Map labelIds = new HashMap<>(); if (this.labels != null) { for (String label : this.labels) { diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java index ed51000a36..7cdbe85da4 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java @@ -83,10 +83,11 @@ public final class ApiVersion { * [0.35] Issue-287: Support pagination when do index query * [0.36] Issue-360: Support paging for scan api * [0.37] Issue-391: Add skip_super_node for shortest path + * [0.38] Issue-274: Add neighborrank API */ // The second parameter of Version.of() is for IDE running without JAR - public static final Version VERSION = Version.of(ApiVersion.class, "0.37"); + public static final Version VERSION = Version.of(ApiVersion.class, "0.38"); public static final void check() { // Check version of hugegraph-core. Firstly do check from version 0.3 From 206aab45e5b88ebaffc84a16b2da1a7923749420 Mon Sep 17 00:00:00 2001 From: liningrui Date: Tue, 16 Apr 2019 17:03:32 +0800 Subject: [PATCH 05/15] Add personalrank Change-Id: I606d0059f093ca5ae4721dce05ad75ef31b26e20 --- .../api/traversers/NeighborRankAPI.java | 25 +-- .../api/traversers/PersonalRankAPI.java | 115 +++++++++++++ .../baidu/hugegraph/version/ApiVersion.java | 2 +- .../traversal/algorithm/HugeTraverser.java | 10 ++ ...verser.java => NeighborRankTraverser.java} | 6 +- .../algorithm/PersonalRankTraverser.java | 153 ++++++++++++++++++ .../com/baidu/hugegraph/example/Example3.java | 16 +- 7 files changed, 300 insertions(+), 27 deletions(-) create mode 100644 hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/{RankTraverser.java => NeighborRankTraverser.java} (97%) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java index 5f5ae0d740..f1dcecd146 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java @@ -23,8 +23,8 @@ import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_DEGREE; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.NO_LIMIT; -import static com.baidu.hugegraph.traversal.algorithm.RankTraverser.MAX_STEPS; -import static com.baidu.hugegraph.traversal.algorithm.RankTraverser.MAX_TOP; +import static com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser.MAX_STEPS; +import static com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser.MAX_TOP; import java.util.ArrayList; import java.util.HashMap; @@ -47,7 +47,7 @@ import com.baidu.hugegraph.core.GraphManager; import com.baidu.hugegraph.schema.EdgeLabel; import com.baidu.hugegraph.server.RestServer; -import com.baidu.hugegraph.traversal.algorithm.RankTraverser; +import com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.JsonUtil; @@ -86,16 +86,17 @@ public String neighborRank(@Context GraphManager manager, Id sourceId = VertexAPI.checkAndParseVertexId(request.source); HugeGraph g = graph(manager, graph); - List steps = steps(g, request); - RankTraverser traverser = new RankTraverser(g, request.alpha, - request.capacity); + List steps = steps(g, request); + NeighborRankTraverser traverser; + traverser = new NeighborRankTraverser(g, request.alpha, + request.capacity); List> ranks = traverser.neighborRank(sourceId, steps); return JsonUtil.toJson(ranks); } - private static List steps(HugeGraph graph, - RankRequest req) { - List steps = new ArrayList<>(req.steps.size()); + private static List steps(HugeGraph graph, + RankRequest req) { + List steps = new ArrayList<>(); for (Step step : req.steps) { steps.add(step.jsonToStep(graph)); } @@ -139,7 +140,7 @@ public String toString() { this.degree, this.top); } - private RankTraverser.Step jsonToStep(HugeGraph graph) { + private NeighborRankTraverser.Step jsonToStep(HugeGraph graph) { E.checkArgument(this.degree > 0 || this.degree == NO_LIMIT, "The degree must be > 0, but got: %s", this.degree); @@ -152,8 +153,8 @@ private RankTraverser.Step jsonToStep(HugeGraph graph) { labelIds.put(el.id(), label); } } - return new RankTraverser.Step(this.direction, labelIds, - this.degree, this.top); + return new NeighborRankTraverser.Step(this.direction, labelIds, + this.degree, this.top); } } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java new file mode 100644 index 0000000000..9eb65be19f --- /dev/null +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java @@ -0,0 +1,115 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.api.traversers; + +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_DEGREE; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.NO_LIMIT; + +import java.util.Map; + +import javax.inject.Singleton; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; + +import org.slf4j.Logger; + +import com.baidu.hugegraph.HugeGraph; +import com.baidu.hugegraph.api.API; +import com.baidu.hugegraph.api.graph.VertexAPI; +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.core.GraphManager; +import com.baidu.hugegraph.server.RestServer; +import com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser; +import com.baidu.hugegraph.util.CollectionUtil; +import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.JsonUtil; +import com.baidu.hugegraph.util.Log; +import com.codahale.metrics.annotation.Timed; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Path("graphs/{graph}/traversers/personalrank") +@Singleton +public class PersonalRankAPI extends API { + + private static final Logger LOG = Log.logger(RestServer.class); + + @POST + @Timed + @Produces(APPLICATION_JSON_WITH_CHARSET) + public String personalRank(@Context GraphManager manager, + @PathParam("graph") String graph, + RankRequest request) { + LOG.debug("Graph [{}] get personal rank from '{}' with " + + "edge label '{}', alpha '{}', max depth '{}'", + graph, request.source, request.label, + request.alpha, request.maxDepth); + + E.checkNotNull(request.source, "source vertex id"); + E.checkNotNull(request.label, "edge label"); + E.checkArgument(request.alpha > 0 && request.alpha <= 1.0, + "The alpha of rank request must belong (0, 1], " + + "but got '%s'", request.alpha); + E.checkArgument(request.degree > 0 || request.degree == NO_LIMIT, + "The degree of rank request must be > 0, but got: %s", + request.degree); + E.checkArgument(request.maxDepth >= 1, + "The max depth of rank request must >= 1, but got '%s'", + request.maxDepth); + + Id sourceId = VertexAPI.checkAndParseVertexId(request.source); + HugeGraph g = graph(manager, graph); + + PersonalRankTraverser traverser; + traverser = new PersonalRankTraverser(g, request.alpha, request.degree, + request.maxDepth); + Map ranks = traverser.personalRank(sourceId, request.label); + if (request.sorted) { + ranks = CollectionUtil.sortByValue(ranks, false); + } + return JsonUtil.toJson(ranks); + } + + private static class RankRequest { + + @JsonProperty("source") + private String source; + @JsonProperty("label") + private String label; + @JsonProperty("alpha") + private double alpha; + @JsonProperty("degree") + public long degree = Long.valueOf(DEFAULT_DEGREE); + @JsonProperty("max_depth") + private int maxDepth; + @JsonProperty("sorted") + private boolean sorted = true; + + @Override + public String toString() { + return String.format("RankRequest{source=%s,label=%s," + + "alpha=%s,degree=%s,maxDepth=%s,sorted=%s}", + this.source, this.label, this.alpha, + this.degree, this.maxDepth, this.sorted); + } + } +} diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java index 7cdbe85da4..37fd336ea1 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java @@ -83,7 +83,7 @@ public final class ApiVersion { * [0.35] Issue-287: Support pagination when do index query * [0.36] Issue-360: Support paging for scan api * [0.37] Issue-391: Add skip_super_node for shortest path - * [0.38] Issue-274: Add neighborrank API + * [0.38] Issue-274: Add personalrank and neighborrank RESTful API */ // The second parameter of Version.of() is for IDE running without JAR diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java index 059b715942..ade72e1c73 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java @@ -44,6 +44,7 @@ import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.tx.GraphTransaction; import com.baidu.hugegraph.iterator.ExtendableIterator; +import com.baidu.hugegraph.iterator.MapperIterator; import com.baidu.hugegraph.schema.SchemaLabel; import com.baidu.hugegraph.structure.HugeEdge; import com.baidu.hugegraph.type.HugeType; @@ -191,6 +192,15 @@ private Set adjacentVertices(Set vertices, Directions dir, return neighbors; } + protected Iterator adjacentVertices(Id source, Directions dir, + Id label, long limit) { + Iterator edges = this.edgesOfVertex(source, dir, label, limit); + return new MapperIterator<>(edges, e -> { + HugeEdge edge = (HugeEdge) e; + return edge.id().otherVertexId(); + }); + } + protected Iterator edgesOfVertex(Id source, Directions dir, Id label, long limit) { Id[] labels = {}; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java similarity index 97% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankTraverser.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java index d82c4663ca..2722cd1a91 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/RankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java @@ -40,7 +40,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -public class RankTraverser extends HugeTraverser { +public class NeighborRankTraverser extends HugeTraverser { public static final int MAX_STEPS = 100; public static final int MAX_TOP = 1000; @@ -49,7 +49,7 @@ public class RankTraverser extends HugeTraverser { private final double alpha; private final long capacity; - public RankTraverser(HugeGraph graph, double alpha, long capacity) { + public NeighborRankTraverser(HugeGraph graph, double alpha, long capacity) { super(graph); checkCapacity(capacity); this.alpha = alpha; @@ -111,7 +111,7 @@ public List> neighborRank(Id source, List steps) { Node newNode = new Node(target, n); adjacencies.add(vertex, newNode); - checkCapacity(capacity, ++access, "neighbor ranks"); + checkCapacity(this.capacity, ++access, "neighbor rank"); } } List adjacenciesV = adjacencies.getOrDefault(vertex, diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java new file mode 100644 index 0000000000..ee85bffdc4 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java @@ -0,0 +1,153 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.function.BiFunction; + +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; + +import com.baidu.hugegraph.HugeGraph; +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.iterator.MapperIterator; +import com.baidu.hugegraph.schema.EdgeLabel; +import com.baidu.hugegraph.schema.VertexLabel; +import com.baidu.hugegraph.structure.HugeEdge; +import com.baidu.hugegraph.structure.HugeVertex; +import com.baidu.hugegraph.type.define.Directions; +import com.baidu.hugegraph.util.E; +import com.google.common.collect.ImmutableMap; + +public class PersonalRankTraverser extends HugeTraverser { + + private final double alpha; + private final long degree; + private final int maxDepth; + + public PersonalRankTraverser(HugeGraph graph, double alpha, + long degree, int maxDepth) { + super(graph); + this.alpha = alpha; + this.degree = degree; + this.maxDepth = maxDepth; + } + + public Map personalRank(Id source, String label) { + Id labelId = this.graph().edgeLabel(label).id(); + Directions dir = this.getStartDirection(source, label); + long degree = this.degreeOfVertex(source, dir, labelId); + if (degree <= 0) { + return ImmutableMap.of(source, 1.0); + } + + Set outSeeds = new HashSet<>(); + Set inSeeds = new HashSet<>(); + if (dir == Directions.OUT) { + outSeeds.add(source); + } else { + inSeeds.add(source); + } + + Map ranks = new HashMap<>(); + ranks.put(source, 1.0); + + for (long i = 0; i < this.maxDepth; i++) { + Map incrRanks = this.getIncrRanks(outSeeds, inSeeds, + labelId, ranks); + ranks = this.compensateSource(source, incrRanks); + } + return ranks; + } + + private Map getIncrRanks(Set outSeeds, Set inSeeds, + Id label, Map ranks) { + Map incrRanks = new HashMap<>(); + BiFunction, Directions, Set> neighborIncrRanks; + neighborIncrRanks = (seeds, dir) -> { + Set tmpSeeds = new HashSet<>(); + for (Id seed : seeds) { + long degree = this.degreeOfVertex(seed, dir, label); + assert degree > 0; + // Must be exist + double originRank = ranks.get(seed); + double spreadRank = originRank * alpha / degree; + + Iterator neighbors = this.adjacentVertices(seed, dir, label, + this.degree); + // Collect all neighbors increment + while (neighbors.hasNext()) { + Id neighbor = neighbors.next(); + tmpSeeds.add(neighbor); + // Assign an initial value when firstly update neighbor rank + double incrRank = incrRanks.getOrDefault(neighbor, 0.0); + incrRank += spreadRank; + incrRanks.put(neighbor, incrRank); + } + } + return tmpSeeds; + }; + + Set tmpInSeeds = neighborIncrRanks.apply(outSeeds, Directions.OUT); + Set tmpOutSeeds = neighborIncrRanks.apply(inSeeds, Directions.IN); + + outSeeds.addAll(tmpOutSeeds); + inSeeds.addAll(tmpInSeeds); + return incrRanks; + } + + private Map compensateSource(Id source, Map incrRanks) { + double sourceRank = incrRanks.getOrDefault(source, 0.0); + sourceRank += (1 - this.alpha); + incrRanks.put(source, sourceRank); + return incrRanks; + } + + private Directions getStartDirection(Id source, String label) { + // NOTE: The outer layer needs to ensure that the vertex Id is valid + HugeVertex vertex = (HugeVertex) graph().vertices(source).next(); + VertexLabel vertexLabel = vertex.schemaLabel(); + EdgeLabel edgeLabel = this.graph().edgeLabel(label); + Id sourceLabel = edgeLabel.sourceLabel(); + Id targetLabel = edgeLabel.targetLabel(); + + E.checkArgument(edgeLabel.linkWithLabel(vertexLabel.id()), + "The vertex '%s' doesn't link with edge label '%s'", + source, label); + E.checkArgument(!sourceLabel.equals(targetLabel), + "The edge label for personal rank must " + + "link different vertex labels"); + if (sourceLabel.equals(vertexLabel.id())) { + return Directions.OUT; + } else { + assert targetLabel.equals(vertexLabel.id()); + return Directions.IN; + } + } + + private long degreeOfVertex(Id source, Directions dir, Id label) { + return IteratorUtils.count(this.edgesOfVertex(source, dir, label, + this.degree)); + } +} diff --git a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java index 290b77f3c5..3185625dd0 100644 --- a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java +++ b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java @@ -47,12 +47,6 @@ public static void load(final HugeGraph graph) { SchemaManager schema = graph.schema(); schema.propertyKey("name").asText().ifNotExist().create(); - schema.propertyKey("age").asInt().ifNotExist().create(); - schema.propertyKey("city").asText().ifNotExist().create(); - schema.propertyKey("weight").asDouble().ifNotExist().create(); - schema.propertyKey("lang").asText().ifNotExist().create(); - schema.propertyKey("date").asText().ifNotExist().create(); - schema.propertyKey("price").asInt().ifNotExist().create(); schema.vertexLabel("person") .properties("name") @@ -78,7 +72,7 @@ public static void load(final HugeGraph graph) { .ifNotExist() .create(); - schema.edgeLabel("directedby") + schema.edgeLabel("directedBy") .sourceLabel("movie") .targetLabel("person") .ifNotExist() @@ -119,11 +113,11 @@ public static void load(final HugeGraph graph) { C.addEdge("like", I); C.addEdge("like", J); - E.addEdge("directedby", K); - F.addEdge("directedby", B); - F.addEdge("directedby", L); + E.addEdge("directedBy", K); + F.addEdge("directedBy", B); + F.addEdge("directedBy", L); - G.addEdge("directedby", M); + G.addEdge("directedBy", M); graph.tx().commit(); } From 98b18bd314ccc2a3defbb40e76b72d3002fe3f00 Mon Sep 17 00:00:00 2001 From: liningrui Date: Tue, 16 Apr 2019 20:09:16 +0800 Subject: [PATCH 06/15] tiny improve Change-Id: I84abff91af2a2df447c8a02c5c8ee942e75fbf9b --- .../hugegraph/api/traversers/EdgesAPI.java | 2 +- .../api/traversers/KneighborAPI.java | 2 +- .../hugegraph/api/traversers/KoutAPI.java | 2 +- .../api/traversers/NeighborRankAPI.java | 3 +- .../api/traversers/PersonalRankAPI.java | 3 +- .../api/traversers/ShortestPathAPI.java | 2 +- .../hugegraph/api/traversers/VerticesAPI.java | 2 +- .../hugegraph/serializer/JsonSerializer.java | 22 +++----- .../hugegraph/serializer/Serializer.java | 9 ++-- .../com/baidu/hugegraph/example/Example3.java | 53 ++++++++++++++++++- 10 files changed, 70 insertions(+), 30 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java index 3ac78446d6..34ac0ffc70 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java @@ -93,7 +93,7 @@ public String shards(@Context GraphManager manager, HugeGraph g = graph(manager, graph); List shards = g.graphTransaction() .metadata(HugeType.EDGE_OUT, "splits", splitSize); - return manager.serializer(g).writeShards(shards); + return manager.serializer(g).writeList("shards", shards); } @GET diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java index b4e1d9fcb0..7c01217173 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java @@ -80,6 +80,6 @@ public String get(@Context GraphManager manager, HugeTraverser traverser = new HugeTraverser(g); Set ids = traverser.kneighbor(source, dir, edgeLabel, depth, degree, limit); - return manager.serializer(g).writeIds("vertices", ids); + return manager.serializer(g).writeList("vertices", ids); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KoutAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KoutAPI.java index 4825637778..191fcbb164 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KoutAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KoutAPI.java @@ -85,6 +85,6 @@ public String get(@Context GraphManager manager, HugeTraverser traverser = new HugeTraverser(g); Set ids = traverser.kout(sourceId, dir, edgeLabel, depth, nearest, degree, capacity, limit); - return manager.serializer(g).writeIds("vertices", ids); + return manager.serializer(g).writeList("vertices", ids); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java index f1dcecd146..e931fa3a1d 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java @@ -50,7 +50,6 @@ import com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; -import com.baidu.hugegraph.util.JsonUtil; import com.baidu.hugegraph.util.Log; import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.annotation.JsonProperty; @@ -91,7 +90,7 @@ public String neighborRank(@Context GraphManager manager, traverser = new NeighborRankTraverser(g, request.alpha, request.capacity); List> ranks = traverser.neighborRank(sourceId, steps); - return JsonUtil.toJson(ranks); + return manager.serializer(g).writeList("ranks", ranks); } private static List steps(HugeGraph graph, diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java index 9eb65be19f..c55829d88c 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java @@ -42,7 +42,6 @@ import com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser; import com.baidu.hugegraph.util.CollectionUtil; import com.baidu.hugegraph.util.E; -import com.baidu.hugegraph.util.JsonUtil; import com.baidu.hugegraph.util.Log; import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.annotation.JsonProperty; @@ -86,7 +85,7 @@ public String personalRank(@Context GraphManager manager, if (request.sorted) { ranks = CollectionUtil.sortByValue(ranks, false); } - return JsonUtil.toJson(ranks); + return manager.serializer(g).writeMap(ranks); } private static class RankRequest { diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java index b606d15766..8ce94a5775 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java @@ -85,6 +85,6 @@ public String get(@Context GraphManager manager, List path = traverser.shortestPath(sourceId, targetId, dir, edgeLabel, depth, degree, skipDegree, capacity); - return manager.serializer(g).writeIds("path", path); + return manager.serializer(g).writeList("path", path); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java index 2c6aaf8812..df2aaacd63 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java @@ -93,7 +93,7 @@ public String shards(@Context GraphManager manager, HugeGraph g = graph(manager, graph); List shards = g.graphTransaction() .metadata(HugeType.VERTEX, "splits", splitSize); - return manager.serializer(g).writeShards(shards); + return manager.serializer(g).writeList("shards", shards); } @GET diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/serializer/JsonSerializer.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/serializer/JsonSerializer.java index 03c88f8828..1be4062245 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/serializer/JsonSerializer.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/serializer/JsonSerializer.java @@ -61,7 +61,13 @@ public static JsonSerializer instance() { return INSTANCE; } - private String writeList(String label, List list) { + @Override + public String writeMap(Map map) { + return JsonUtil.toJson(map); + } + + @Override + public String writeList(String label, Collection list) { try (ByteArrayOutputStream out = new ByteArrayOutputStream(LBUF_SIZE)) { out.write(String.format("{\"%s\": ", label).getBytes(API.CHARSET)); out.write(JsonUtil.toJson(list).getBytes(API.CHARSET)); @@ -197,15 +203,6 @@ public String writeEdges(Iterator edges, boolean paging) { return this.writeIterator("edges", edges, paging); } - @Override - public String writeIds(String name, Collection ids) { - if (ids instanceof List) { - return this.writeList(name, (List) ids); - } else { - return this.writeList(name, new ArrayList<>(ids)); - } - } - @Override public String writePaths(String name, Collection paths, boolean withCrossPoint, @@ -243,9 +240,4 @@ public String writeCrosspoints(CrosspointsPaths paths, "vertices", iterator); return JsonUtil.toJson(results); } - - @Override - public String writeShards(List shards) { - return this.writeList("shards", shards); - } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/serializer/Serializer.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/serializer/Serializer.java index 9e1e5198a6..7e51a9efd5 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/serializer/Serializer.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/serializer/Serializer.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Map; import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.Vertex; @@ -37,6 +38,10 @@ public interface Serializer { + public String writeMap(Map map); + + public String writeList(String label, Collection list); + public String writePropertyKey(PropertyKey propertyKey); public String writePropertyKeys(List propertyKeys); @@ -63,8 +68,6 @@ public interface Serializer { public String writeEdges(Iterator edges, boolean paging); - public String writeIds(String name, Collection ids); - public String writePaths(String name, Collection paths, boolean withCrossPoint, Iterator vertices); @@ -76,6 +79,4 @@ public default String writePaths(String name, public String writeCrosspoints(CrosspointsPaths paths, Iterator iterator, boolean withPath); - - public String writeShards(List shards); } diff --git a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java index 3185625dd0..1b61b3f18a 100644 --- a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java +++ b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java @@ -36,14 +36,15 @@ public static void main(String[] args) throws InterruptedException { HugeGraph graph = ExampleUtil.loadGraph(); - Example3.load(graph); + Example3.loadNeighborRankData(graph); +// Example3.loadPersonalRankData(graph); graph.close(); HugeGraph.shutdown(30L); } - public static void load(final HugeGraph graph) { + public static void loadNeighborRankData(final HugeGraph graph) { SchemaManager schema = graph.schema(); schema.propertyKey("name").asText().ifNotExist().create(); @@ -121,4 +122,52 @@ public static void load(final HugeGraph graph) { graph.tx().commit(); } + + public static void loadPersonalRankData(final HugeGraph graph) { + SchemaManager schema = graph.schema(); + + schema.propertyKey("name").asText().ifNotExist().create(); + + schema.vertexLabel("person") + .properties("name") + .useCustomizeStringId() + .ifNotExist() + .create(); + + schema.vertexLabel("movie") + .properties("name") + .useCustomizeStringId() + .ifNotExist() + .create(); + + schema.edgeLabel("like") + .sourceLabel("person") + .targetLabel("movie") + .ifNotExist() + .create(); + + graph.tx().open(); + + Vertex A = graph.addVertex(T.label, "person", T.id, "A", "name", "A"); + Vertex B = graph.addVertex(T.label, "person", T.id, "B", "name", "B"); + Vertex C = graph.addVertex(T.label, "person", T.id, "C", "name", "C"); + + Vertex a = graph.addVertex(T.label, "movie", T.id, "a", "name", "a"); + Vertex b = graph.addVertex(T.label, "movie", T.id, "b", "name", "b"); + Vertex c = graph.addVertex(T.label, "movie", T.id, "c", "name", "c"); + Vertex d = graph.addVertex(T.label, "movie", T.id, "d", "name", "d"); + + A.addEdge("like", a); + A.addEdge("like", c); + + B.addEdge("like", a); + B.addEdge("like", b); + B.addEdge("like", c); + B.addEdge("like", d); + + C.addEdge("like", c); + C.addEdge("like", d); + + graph.tx().commit(); + } } From da7be43faeeafc70f6d5531898df6456a2b77eff Mon Sep 17 00:00:00 2001 From: liningrui Date: Tue, 16 Apr 2019 21:17:07 +0800 Subject: [PATCH 07/15] Add dependency hugegraph-common 1.6.0 Change-Id: I79b2d6ae1f780728cc935c147ca8d96f3a786bf8 --- .../api/traversers/PersonalRankAPI.java | 17 ++- hugegraph-core/pom.xml | 2 +- .../algorithm/NeighborRankTraverser.java | 43 +++--- .../algorithm/PersonalRankTraverser.java | 11 +- .../baidu/hugegraph/util/OrderLimitMap.java | 98 ------------- .../baidu/hugegraph/version/CoreVersion.java | 2 +- .../baidu/hugegraph/unit/UnitTestSuite.java | 1 - .../unit/core/OrderLimitMapTest.java | 135 ------------------ 8 files changed, 45 insertions(+), 264 deletions(-) delete mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/util/OrderLimitMap.java delete mode 100644 hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/OrderLimitMapTest.java diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java index c55829d88c..f5aa815270 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java @@ -58,13 +58,10 @@ public class PersonalRankAPI extends API { public String personalRank(@Context GraphManager manager, @PathParam("graph") String graph, RankRequest request) { - LOG.debug("Graph [{}] get personal rank from '{}' with " + - "edge label '{}', alpha '{}', max depth '{}'", - graph, request.source, request.label, - request.alpha, request.maxDepth); - - E.checkNotNull(request.source, "source vertex id"); - E.checkNotNull(request.label, "edge label"); + E.checkArgument(request.source != null, + "The source vertex id of rank request can't be null"); + E.checkArgument(request.label != null, + "The edge label of rank request can't be null"); E.checkArgument(request.alpha > 0 && request.alpha <= 1.0, "The alpha of rank request must belong (0, 1], " + "but got '%s'", request.alpha); @@ -75,6 +72,12 @@ public String personalRank(@Context GraphManager manager, "The max depth of rank request must >= 1, but got '%s'", request.maxDepth); + LOG.debug("Graph [{}] get personal rank from '{}' with " + + "edge label '{}', alpha '{}', degree '{}', " + + "max depth '{}', sorted '{}'", + graph, request.source, request.label, request.alpha, + request.degree, request.maxDepth, request.sorted); + Id sourceId = VertexAPI.checkAndParseVertexId(request.source); HugeGraph g = graph(manager, graph); diff --git a/hugegraph-core/pom.xml b/hugegraph-core/pom.xml index d2e3dc69aa..b192b4819b 100644 --- a/hugegraph-core/pom.xml +++ b/hugegraph-core/pom.xml @@ -19,7 +19,7 @@ com.baidu.hugegraph hugegraph-common - 1.5.8 + 1.6.0 diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java index 2722cd1a91..32acd24d4e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java @@ -38,7 +38,6 @@ import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.OrderLimitMap; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; public class NeighborRankTraverser extends HugeTraverser { @@ -66,8 +65,8 @@ public List> neighborRank(Id source, List steps) { boolean sameLayerTransfer = true; long access = 0; // Result - List> ranks = new ArrayList<>(); - ranks.add(ImmutableMap.of(source, 1.0)); + List ranks = new ArrayList<>(); + ranks.add(Ranks.of(source, 1.0)); root : for (Step step : steps) { Map lastLayerRanks = ranks.get(ranks.size() - 1); @@ -132,7 +131,7 @@ public List> neighborRank(Id source, List steps) { this.contributePrevLayers(ranks, incr, prevLayerNodesV); } - Map newLayerRanks; + Ranks newLayerRanks; if (sameLayerTransfer) { // First contribute to last layer, then pass to the new layer this.contributeLastLayer(sameLayerIncrRanks, lastLayerRanks); @@ -164,13 +163,12 @@ private boolean belongToSameLayer(Set sources, Id target, } } - private boolean belongToPrevLayers(List> ranks, Id target, + private boolean belongToPrevLayers(List ranks, Id target, Map> prevLayerNodes) { for (int i = ranks.size() - 2; i > 0; i--) { - Map prevLayerRanks = ranks.get(i); + Ranks prevLayerRanks = ranks.get(i); if (prevLayerRanks.containsKey(target)) { - Set nodes = prevLayerNodes.computeIfAbsent(i, - k -> new HashSet<>()); + Set nodes = prevLayerNodes.computeIfAbsent(i, HashSet::new); nodes.add(target); return true; } @@ -186,7 +184,7 @@ private void mergeSameLayerIncrRanks(Set sameLayerNodesV, double incr, } } - private void contributePrevLayers(List> ranks, double incr, + private void contributePrevLayers(List ranks, double incr, Map> prevLayerNodesV) { for (Map.Entry> e : prevLayerNodesV.entrySet()) { Map prevLayerRanks = ranks.get(e.getKey()); @@ -206,11 +204,10 @@ private void contributeLastLayer(Map rankIncrs, } } - private Map contributeNewLayer( - MultivaluedMap adjacencies, - Map lastLayerRanks, - int capacity) { - Map newLayerRanks = new OrderLimitMap<>(capacity); + private Ranks contributeNewLayer(MultivaluedMap adjacencies, + Map lastLayerRanks, + int capacity) { + Ranks newLayerRanks = new Ranks(capacity); for (Map.Entry> entry : adjacencies.entrySet()) { Id parent = entry.getKey(); long size = entry.getValue().size(); @@ -223,7 +220,7 @@ private Map contributeNewLayer( return newLayerRanks; } - private List> topRanks(List> ranks, + private List> topRanks(List ranks, List steps) { assert ranks.size() > 0; List> results = new ArrayList<>(ranks.size()); @@ -231,8 +228,7 @@ private List> topRanks(List> ranks, results.add(ranks.get(0)); for (int i = 1; i < ranks.size(); i++) { Step step = steps.get(i - 1); - OrderLimitMap origin = (OrderLimitMap) - ranks.get(i); + Ranks origin = ranks.get(i); if (origin.size() > step.top) { results.add(origin.topN(step.top)); } else { @@ -259,4 +255,17 @@ public Step(Directions direction, Map labels, this.capacity = DEFAULT_CAPACITY_PER_LAYER; } } + + private static class Ranks extends OrderLimitMap { + + public Ranks(int capacity) { + super(capacity); + } + + public static Ranks of(Id key, Double value) { + Ranks ranks = new Ranks(1); + ranks.put(key, value); + return ranks; + } + } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java index ee85bffdc4..25c7a391bf 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java @@ -55,11 +55,17 @@ public PersonalRankTraverser(HugeGraph graph, double alpha, } public Map personalRank(Id source, String label) { + E.checkArgumentNotNull(source, "The source vertex id can't be null"); + E.checkArgumentNotNull(label, "The edge label can't be null"); + + Map ranks = new HashMap<>(); + ranks.put(source, 1.0); + Id labelId = this.graph().edgeLabel(label).id(); Directions dir = this.getStartDirection(source, label); long degree = this.degreeOfVertex(source, dir, labelId); if (degree <= 0) { - return ImmutableMap.of(source, 1.0); + return ranks; } Set outSeeds = new HashSet<>(); @@ -70,9 +76,6 @@ public Map personalRank(Id source, String label) { inSeeds.add(source); } - Map ranks = new HashMap<>(); - ranks.put(source, 1.0); - for (long i = 0; i < this.maxDepth; i++) { Map incrRanks = this.getIncrRanks(outSeeds, inSeeds, labelId, ranks); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/OrderLimitMap.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/OrderLimitMap.java deleted file mode 100644 index fc24816c41..0000000000 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/OrderLimitMap.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2017 HugeGraph Authors - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.baidu.hugegraph.util; - -import java.util.HashMap; -import java.util.Map; -import java.util.TreeMap; - -import com.google.common.base.Functions; -import com.google.common.collect.Ordering; - -/** - * Reference: https://stackoverflow.com/questions/109383/sort-a-mapkey-value-by-values - * TODO: move to common - */ -public class OrderLimitMap, V extends Comparable> - extends TreeMap { - - private final int capacity; - private final Map valueMap; - - public static > Ordering incr() { - return Ordering.from((V o1, V o2) -> o1.compareTo(o2)); - } - - public static > Ordering decr() { - return Ordering.from((V o1, V o2) -> -o1.compareTo(o2)); - } - - public OrderLimitMap(int capacity) { - this(capacity, decr(), new HashMap<>()); - } - - private OrderLimitMap(int capacity, Ordering ordering, - HashMap valueMap) { - /* - * onResultOf: for getting the value for the key from value map - * compound: keep insertion order - */ - super(ordering.onResultOf(Functions.forMap(valueMap)) - .compound(Ordering.natural())); - this.capacity = capacity; - this.valueMap = valueMap; - } - - @Override - public V put(K k, V v) { - if (this.valueMap.containsKey(k)) { - super.remove(k); - } else if (this.valueMap.size() >= this.capacity) { - K key = super.lastKey(); - super.remove(key); - this.valueMap.remove(key); - } - this.valueMap.put(k, v); - return super.put(k, v); - } - - @Override - public V getOrDefault(Object key, V defaultValue) { - return this.valueMap.getOrDefault(key, defaultValue); - } - - @Override - public boolean containsKey(Object key) { - return this.valueMap.containsKey(key); - } - - public Map topN(int n) { - E.checkArgument(n > 0, "'N' Must be positive, but got '%s'", n); - Map subMap = InsertionOrderUtil.newMap(); - int i = 0; - for (Map.Entry entry : this.entrySet()) { - subMap.put(entry.getKey(), entry.getValue()); - if (++i >= n) { - break; - } - } - return subMap; - } -} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/version/CoreVersion.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/version/CoreVersion.java index 6548edd6f0..12e2707f82 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/version/CoreVersion.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/version/CoreVersion.java @@ -39,7 +39,7 @@ public class CoreVersion { public static void check() { // Check version of hugegraph-common - VersionUtil.check(CommonVersion.VERSION, "1.5.0", "1.6", + VersionUtil.check(CommonVersion.VERSION, "1.6.0", "1.7", CommonVersion.NAME); } } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java index 8951d01de9..afbb687f8e 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java @@ -47,7 +47,6 @@ EdgeIdTest.class, AnalyzerTest.class, JsonUtilTest.class, - OrderLimitMapTest.class, RocksDBSessionsTest.class, RocksDBCountersTest.class diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/OrderLimitMapTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/OrderLimitMapTest.java deleted file mode 100644 index dbeaf5c1b9..0000000000 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/OrderLimitMapTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2017 HugeGraph Authors - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.baidu.hugegraph.unit.core; - -import org.junit.Test; - -import com.baidu.hugegraph.testutil.Assert; -import com.baidu.hugegraph.util.OrderLimitMap; -import com.google.common.collect.ImmutableList; - -public class OrderLimitMapTest { - - @Test - public void testOrder() { - OrderLimitMap map = new OrderLimitMap<>(5); - map.put(1, 0.1); - map.put(2, 0.2); - map.put(3, 0.3); - map.put(4, 0.4); - map.put(5, 0.5); - - Assert.assertEquals(5, map.size()); - Assert.assertEquals(ImmutableList.of(5, 4, 3, 2, 1), - ImmutableList.copyOf(map.keySet())); - } - - @Test - public void testOrderWithDupValue() { - OrderLimitMap map = new OrderLimitMap<>(5); - map.put(1, 0.1); - map.put(2, 0.2); - map.put(3, 0.3); - map.put(4, 0.2); - map.put(5, 0.3); - - Assert.assertEquals(5, map.size()); - Assert.assertEquals(ImmutableList.of(3, 5, 2, 4, 1), - ImmutableList.copyOf(map.keySet())); - } - - @Test - public void testOrderWithDupValueAndKeyIncrOrder() { - OrderLimitMap map = new OrderLimitMap<>(5); - map.put(4, 0.2); - map.put(2, 0.2); - map.put(1, 0.1); - map.put(5, 0.3); - map.put(3, 0.3); - - Assert.assertEquals(5, map.size()); - Assert.assertEquals(ImmutableList.of(3, 5, 2, 4, 1), - ImmutableList.copyOf(map.keySet())); - } - - @Test - public void testOrderWithDupKey() { - OrderLimitMap map = new OrderLimitMap<>(5); - map.put(1, 0.1); - map.put(2, 0.2); - map.put(3, 0.3); - map.put(2, 0.4); - map.put(3, 0.2); - - Assert.assertEquals(3, map.size()); - Assert.assertEquals(ImmutableList.of(2, 3, 1), - ImmutableList.copyOf(map.keySet())); - } - - @Test - public void testLimit() { - OrderLimitMap map = new OrderLimitMap<>(5); - map.put(1, 0.1); - map.put(2, 0.2); - map.put(3, 0.3); - map.put(4, 0.4); - map.put(5, 0.5); - - map.put(6, 0.6); - Assert.assertEquals(5, map.size()); - Assert.assertEquals(ImmutableList.of(6, 5, 4, 3, 2), - ImmutableList.copyOf(map.keySet())); - - map.put(7, 0.7); - Assert.assertEquals(5, map.size()); - Assert.assertEquals(ImmutableList.of(7, 6, 5, 4, 3), - ImmutableList.copyOf(map.keySet())); - } - - @Test - public void testLimitWithDupValue() { - OrderLimitMap map = new OrderLimitMap<>(5); - map.put(1, 0.1); - map.put(2, 0.2); - map.put(3, 0.3); - map.put(4, 0.4); - map.put(5, 0.5); - - map.put(6, 0.1); - Assert.assertEquals(5, map.size()); - Assert.assertEquals(ImmutableList.of(5, 4, 3, 2, 6), - ImmutableList.copyOf(map.keySet())); - - map.put(7, 0.3); - Assert.assertEquals(5, map.size()); - Assert.assertEquals(ImmutableList.of(5, 4, 3, 7, 2), - ImmutableList.copyOf(map.keySet())); - - map.put(8, 0.5); - Assert.assertEquals(5, map.size()); - Assert.assertEquals(ImmutableList.of(5, 8, 4, 3, 7), - ImmutableList.copyOf(map.keySet())); - - map.put(0, 0.5); - Assert.assertEquals(5, map.size()); - Assert.assertEquals(ImmutableList.of(0, 5, 8, 4, 3), - ImmutableList.copyOf(map.keySet())); - } -} From 7146434c2dfe5237227d0a694eddddf24f6b0747 Mon Sep 17 00:00:00 2001 From: liningrui Date: Tue, 16 Apr 2019 21:28:47 +0800 Subject: [PATCH 08/15] tiny improve Change-Id: I5012e7e7be649ccf7c7de160354a39b980fecf48 --- .../traversal/algorithm/NeighborRankTraverser.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java index 32acd24d4e..d9c6a01db5 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java @@ -64,12 +64,12 @@ public List> neighborRank(Id source, List steps) { boolean sameLayerTransfer = true; long access = 0; - // Result + // Results: ranks of each layer List ranks = new ArrayList<>(); ranks.add(Ranks.of(source, 1.0)); root : for (Step step : steps) { - Map lastLayerRanks = ranks.get(ranks.size() - 1); + Ranks lastLayerRanks = ranks.get(ranks.size() - 1); Map sameLayerIncrRanks = new HashMap<>(); MultivaluedMap adjacencies = newMultivalueMap(); MultivaluedMap newVertices = newMultivalueMap(); @@ -187,7 +187,7 @@ private void mergeSameLayerIncrRanks(Set sameLayerNodesV, double incr, private void contributePrevLayers(List ranks, double incr, Map> prevLayerNodesV) { for (Map.Entry> e : prevLayerNodesV.entrySet()) { - Map prevLayerRanks = ranks.get(e.getKey()); + Ranks prevLayerRanks = ranks.get(e.getKey()); for (Id node : e.getValue()) { double oldRank = prevLayerRanks.get(node); prevLayerRanks.put(node, oldRank + incr); @@ -196,7 +196,7 @@ private void contributePrevLayers(List ranks, double incr, } private void contributeLastLayer(Map rankIncrs, - Map lastLayerRanks) { + Ranks lastLayerRanks) { for (Map.Entry entry : rankIncrs.entrySet()) { double originRank = lastLayerRanks.get(entry.getKey()); double incrRank = entry.getValue(); @@ -205,8 +205,7 @@ private void contributeLastLayer(Map rankIncrs, } private Ranks contributeNewLayer(MultivaluedMap adjacencies, - Map lastLayerRanks, - int capacity) { + Ranks lastLayerRanks, int capacity) { Ranks newLayerRanks = new Ranks(capacity); for (Map.Entry> entry : adjacencies.entrySet()) { Id parent = entry.getKey(); From fa1a42794651469dcbffa20afebd0eb96f8c0363 Mon Sep 17 00:00:00 2001 From: liningrui Date: Wed, 17 Apr 2019 13:23:26 +0800 Subject: [PATCH 09/15] fix some bugs Change-Id: I8c27b23216bd9ee97c2afa613890a6a23b66df5d --- .../api/traversers/NeighborRankAPI.java | 2 +- .../api/traversers/PersonalRankAPI.java | 42 +++++++++++++--- .../traversal/algorithm/HugeTraverser.java | 1 + .../algorithm/NeighborRankTraverser.java | 49 +++++++++++++++---- .../algorithm/PersonalRankTraverser.java | 27 +++++++++- .../com/baidu/hugegraph/example/Example3.java | 2 +- .../baidu/hugegraph/unit/UnitTestSuite.java | 1 - 7 files changed, 104 insertions(+), 20 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java index e931fa3a1d..3a1c4667d0 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java @@ -79,7 +79,7 @@ public String neighborRank(@Context GraphManager manager, "but got '%s'", request.alpha); LOG.debug("Graph [{}] get neighbor rank from '{}' with steps '{}', " + - "alpha '{}', capacity '{}'", graph, request.source, + "alpha '{}' and capacity '{}'", graph, request.source, request.steps, request.alpha, request.capacity); Id sourceId = VertexAPI.checkAndParseVertexId(request.source); diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java index f5aa815270..13e35628e0 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java @@ -20,6 +20,7 @@ package com.baidu.hugegraph.api.traversers; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_DEGREE; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_LIMIT; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.NO_LIMIT; import java.util.Map; @@ -42,6 +43,7 @@ import com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser; import com.baidu.hugegraph.util.CollectionUtil; import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.InsertionOrderUtil; import com.baidu.hugegraph.util.Log; import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.annotation.JsonProperty; @@ -58,6 +60,7 @@ public class PersonalRankAPI extends API { public String personalRank(@Context GraphManager manager, @PathParam("graph") String graph, RankRequest request) { + E.checkArgumentNotNull(request, "The rank request body can't be null"); E.checkArgument(request.source != null, "The source vertex id of rank request can't be null"); E.checkArgument(request.label != null, @@ -68,13 +71,16 @@ public String personalRank(@Context GraphManager manager, E.checkArgument(request.degree > 0 || request.degree == NO_LIMIT, "The degree of rank request must be > 0, but got: %s", request.degree); + E.checkArgument(request.limit > 0 || request.limit == NO_LIMIT, + "The limit of rank request must be > 0, but got: %s", + request.limit); E.checkArgument(request.maxDepth >= 1, "The max depth of rank request must >= 1, but got '%s'", request.maxDepth); LOG.debug("Graph [{}] get personal rank from '{}' with " + "edge label '{}', alpha '{}', degree '{}', " + - "max depth '{}', sorted '{}'", + "max depth '{}' and sorted '{}'", graph, request.source, request.label, request.alpha, request.degree, request.maxDepth, request.sorted); @@ -84,11 +90,26 @@ public String personalRank(@Context GraphManager manager, PersonalRankTraverser traverser; traverser = new PersonalRankTraverser(g, request.alpha, request.degree, request.maxDepth); - Map ranks = traverser.personalRank(sourceId, request.label); - if (request.sorted) { + Map ranks = traverser.personalRank(sourceId, request.label, + request.withLabel); + ranks = topN(ranks, request.sorted, request.limit); + return manager.serializer(g).writeMap(ranks); + } + + private static Map topN(Map ranks, + boolean sorted, long limit) { + if (sorted) { ranks = CollectionUtil.sortByValue(ranks, false); } - return manager.serializer(g).writeMap(ranks); + Map results = InsertionOrderUtil.newMap(); + long count = 0; + for (Map.Entry entry : ranks.entrySet()) { + results.put(entry.getKey(), entry.getValue()); + if (++count >= limit) { + break; + } + } + return results; } private static class RankRequest { @@ -100,18 +121,25 @@ private static class RankRequest { @JsonProperty("alpha") private double alpha; @JsonProperty("degree") - public long degree = Long.valueOf(DEFAULT_DEGREE); + private long degree = Long.valueOf(DEFAULT_DEGREE); + @JsonProperty("limit") + private long limit = Long.valueOf(DEFAULT_LIMIT); @JsonProperty("max_depth") private int maxDepth; + @JsonProperty("with_label") + private PersonalRankTraverser.WithLabel withLabel = + PersonalRankTraverser.WithLabel.BOTH_LABEL; @JsonProperty("sorted") private boolean sorted = true; @Override public String toString() { return String.format("RankRequest{source=%s,label=%s," + - "alpha=%s,degree=%s,maxDepth=%s,sorted=%s}", + "alpha=%s,degree=%s,limit=%s, maxDepth=%s," + + "withLabel=%s,sorted=%s}", this.source, this.label, this.alpha, - this.degree, this.maxDepth, this.sorted); + this.degree, this.limit, this.maxDepth, + this.withLabel, this.sorted); } } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java index ade72e1c73..d5adb24d81 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java @@ -63,6 +63,7 @@ public class HugeTraverser { public static final String DEFAULT_CAPACITY = "10000000"; public static final String DEFAULT_ELEMENTS_LIMIT = "10000000"; public static final String DEFAULT_PATHS_LIMIT = "10"; + public static final String DEFAULT_LIMIT = "100"; public static final String DEFAULT_DEGREE = "10000"; public static final String DEFAULT_SAMPLE = "100"; public static final String DEFAULT_WEIGHT = "0"; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java index d9c6a01db5..67a699a6b9 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java @@ -71,7 +71,7 @@ public List> neighborRank(Id source, List steps) { root : for (Step step : steps) { Ranks lastLayerRanks = ranks.get(ranks.size() - 1); Map sameLayerIncrRanks = new HashMap<>(); - MultivaluedMap adjacencies = newMultivalueMap(); + Adjacencies adjacencies = new Adjacencies(); MultivaluedMap newVertices = newMultivalueMap(); // Traversal vertices of previous level for (Map.Entry> entry : sources.entrySet()) { @@ -113,10 +113,10 @@ public List> neighborRank(Id source, List steps) { checkCapacity(this.capacity, ++access, "neighbor rank"); } } - List adjacenciesV = adjacencies.getOrDefault(vertex, - ImmutableList.of()); + List adjacenciesV = adjacencies.get(vertex); assert degree == sameLayerNodesV.size() + prevLayerNodesV.size() + adjacenciesV.size(); + adjacencies.put(vertex, degree); // Add adjacent nodes of current node to sources of next step for (Node node : adjacenciesV) { @@ -168,8 +168,8 @@ private boolean belongToPrevLayers(List ranks, Id target, for (int i = ranks.size() - 2; i > 0; i--) { Ranks prevLayerRanks = ranks.get(i); if (prevLayerRanks.containsKey(target)) { - Set nodes = prevLayerNodes.computeIfAbsent(i, HashSet::new); - nodes.add(target); + Set get = prevLayerNodes.computeIfAbsent(i, HashSet::new); + get.add(target); return true; } } @@ -204,15 +204,15 @@ private void contributeLastLayer(Map rankIncrs, } } - private Ranks contributeNewLayer(MultivaluedMap adjacencies, + private Ranks contributeNewLayer(Adjacencies adjacencies, Ranks lastLayerRanks, int capacity) { Ranks newLayerRanks = new Ranks(capacity); - for (Map.Entry> entry : adjacencies.entrySet()) { + for (Map.Entry> entry : adjacencies.nodeEntrySet()) { Id parent = entry.getKey(); - long size = entry.getValue().size(); + long degree = adjacencies.degree(parent); for (Node node : entry.getValue()) { double rank = newLayerRanks.getOrDefault(node.id(), 0.0); - rank += (lastLayerRanks.get(parent) * this.alpha / size); + rank += (lastLayerRanks.get(parent) * this.alpha / degree); newLayerRanks.put(node.id(), rank); } } @@ -255,6 +255,37 @@ public Step(Directions direction, Map labels, } } + private static class Adjacencies { + + private final MultivaluedMap adjacencies; + private final Map degrees; + + public Adjacencies() { + this.adjacencies = newMultivalueMap(); + this.degrees = new HashMap<>(); + } + + public void add(Id id, Node node) { + this.adjacencies.add(id, node); + } + + public List get(Id id) { + return this.adjacencies.getOrDefault(id, ImmutableList.of()); + } + + public Iterable>> nodeEntrySet() { + return this.adjacencies.entrySet(); + } + + public void put(Id id, long degree) { + this.degrees.put(id, degree); + } + + public long degree(Id id) { + return this.degrees.get(id); + } + } + private static class Ranks extends OrderLimitMap { public Ranks(int capacity) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java index 25c7a391bf..8b263da4a2 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java @@ -54,7 +54,8 @@ public PersonalRankTraverser(HugeGraph graph, double alpha, this.maxDepth = maxDepth; } - public Map personalRank(Id source, String label) { + public Map personalRank(Id source, String label, + WithLabel withLabel) { E.checkArgumentNotNull(source, "The source vertex id can't be null"); E.checkArgumentNotNull(label, "The edge label can't be null"); @@ -76,10 +77,22 @@ public Map personalRank(Id source, String label) { inSeeds.add(source); } + Set firstAdjacencies = new HashSet<>(); for (long i = 0; i < this.maxDepth; i++) { Map incrRanks = this.getIncrRanks(outSeeds, inSeeds, labelId, ranks); ranks = this.compensateSource(source, incrRanks); + if (i == 0) { + firstAdjacencies.addAll(ranks.keySet()); + } + } + // Remove directly connected neighbors + removeAll(ranks, firstAdjacencies); + // Remove unnecessary label + if (withLabel == WithLabel.SAME_LABEL) { + removeAll(ranks, dir == Directions.OUT ? inSeeds : outSeeds); + } else if (withLabel == WithLabel.OTHER_LABEL) { + removeAll(ranks, dir == Directions.OUT ? outSeeds : inSeeds); } return ranks; } @@ -153,4 +166,16 @@ private long degreeOfVertex(Id source, Directions dir, Id label) { return IteratorUtils.count(this.edgesOfVertex(source, dir, label, this.degree)); } + + private static void removeAll(Map map, Set keys) { + for (Id key : keys) { + map.remove(key); + } + } + + public enum WithLabel { + SAME_LABEL, + OTHER_LABEL, + BOTH_LABEL + } } diff --git a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java index 1b61b3f18a..caa212228c 100644 --- a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java +++ b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example3.java @@ -37,7 +37,7 @@ public static void main(String[] args) throws InterruptedException { HugeGraph graph = ExampleUtil.loadGraph(); Example3.loadNeighborRankData(graph); -// Example3.loadPersonalRankData(graph); + Example3.loadPersonalRankData(graph); graph.close(); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java index afbb687f8e..98d0297fbe 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java @@ -30,7 +30,6 @@ import com.baidu.hugegraph.unit.core.ConditionQueryFlattenTest; import com.baidu.hugegraph.unit.core.EdgeIdTest; import com.baidu.hugegraph.unit.core.JsonUtilTest; -import com.baidu.hugegraph.unit.core.OrderLimitMapTest; import com.baidu.hugegraph.unit.core.VersionTest; import com.baidu.hugegraph.unit.rocksdb.RocksDBCountersTest; import com.baidu.hugegraph.unit.rocksdb.RocksDBSessionsTest; From f8f817d86719fdb4577bc443194aafee677427db Mon Sep 17 00:00:00 2001 From: liningrui Date: Wed, 17 Apr 2019 15:32:31 +0800 Subject: [PATCH 10/15] Refactor class Adjacencies Change-Id: I2bd0d75f0a798015ea302679cdf2635b7fc80745 --- .../api/traversers/PersonalRankAPI.java | 3 + .../algorithm/NeighborRankTraverser.java | 69 +++++++++---------- .../algorithm/PersonalRankTraverser.java | 4 -- 3 files changed, 37 insertions(+), 39 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java index 13e35628e0..5b62d015b6 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java @@ -101,6 +101,9 @@ private static Map topN(Map ranks, if (sorted) { ranks = CollectionUtil.sortByValue(ranks, false); } + if (limit == NO_LIMIT) { + return ranks; + } Map results = InsertionOrderUtil.newMap(); long count = 0; for (Map.Entry entry : ranks.entrySet()) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java index 67a699a6b9..484c7f6302 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java @@ -68,10 +68,10 @@ public List> neighborRank(Id source, List steps) { List ranks = new ArrayList<>(); ranks.add(Ranks.of(source, 1.0)); - root : for (Step step : steps) { + for (Step step : steps) { Ranks lastLayerRanks = ranks.get(ranks.size() - 1); Map sameLayerIncrRanks = new HashMap<>(); - Adjacencies adjacencies = new Adjacencies(); + List adjacencies = new ArrayList<>(); MultivaluedMap newVertices = newMultivalueMap(); // Traversal vertices of previous level for (Map.Entry> entry : sources.entrySet()) { @@ -80,11 +80,10 @@ public List> neighborRank(Id source, List steps) { step.labels, null, step.degree); - long degree = 0L; + Adjacencies adjacenciesV = new Adjacencies(vertex); Set sameLayerNodesV = new HashSet<>(); Map> prevLayerNodesV = new HashMap<>(); while (edges.hasNext()) { - degree++; HugeEdge edge = (HugeEdge) edges.next(); Id target = edge.id().otherVertexId(); // Determine whether it belongs to the same layer @@ -108,20 +107,18 @@ public List> neighborRank(Id source, List steps) { continue; } Node newNode = new Node(target, n); - adjacencies.add(vertex, newNode); + adjacenciesV.add(newNode); + // Add adjacent nodes to sources of next step + newVertices.add(target, newNode); checkCapacity(this.capacity, ++access, "neighbor rank"); } } - List adjacenciesV = adjacencies.get(vertex); - assert degree == sameLayerNodesV.size() + - prevLayerNodesV.size() + adjacenciesV.size(); - adjacencies.put(vertex, degree); - - // Add adjacent nodes of current node to sources of next step - for (Node node : adjacenciesV) { - newVertices.add(node.id(), node); - } + long degree = sameLayerNodesV.size() + prevLayerNodesV.size() + + adjacenciesV.nodes().size(); + adjacenciesV.degree(degree); + adjacencies.add(adjacenciesV); + double incr = lastLayerRanks.getOrDefault(vertex, 0.0) * this.alpha / degree; // Merge the increment of the same layer node @@ -204,15 +201,15 @@ private void contributeLastLayer(Map rankIncrs, } } - private Ranks contributeNewLayer(Adjacencies adjacencies, + private Ranks contributeNewLayer(List adjacencies, Ranks lastLayerRanks, int capacity) { Ranks newLayerRanks = new Ranks(capacity); - for (Map.Entry> entry : adjacencies.nodeEntrySet()) { - Id parent = entry.getKey(); - long degree = adjacencies.degree(parent); - for (Node node : entry.getValue()) { + for (Adjacencies adjacenciesV : adjacencies) { + Id source = adjacenciesV.source(); + long degree = adjacenciesV.degree(); + for (Node node : adjacenciesV.nodes()) { double rank = newLayerRanks.getOrDefault(node.id(), 0.0); - rank += (lastLayerRanks.get(parent) * this.alpha / degree); + rank += (lastLayerRanks.get(source) * this.alpha / degree); newLayerRanks.put(node.id(), rank); } } @@ -257,32 +254,34 @@ public Step(Directions direction, Map labels, private static class Adjacencies { - private final MultivaluedMap adjacencies; - private final Map degrees; + private final Id source; + private final List nodes; + private long degree; - public Adjacencies() { - this.adjacencies = newMultivalueMap(); - this.degrees = new HashMap<>(); + public Adjacencies(Id source) { + this.source = source; + this.nodes = new ArrayList<>(); + this.degree = 0L; } - public void add(Id id, Node node) { - this.adjacencies.add(id, node); + public Id source() { + return this.source; } - public List get(Id id) { - return this.adjacencies.getOrDefault(id, ImmutableList.of()); + public List nodes() { + return this.nodes; } - public Iterable>> nodeEntrySet() { - return this.adjacencies.entrySet(); + public void add(Node node) { + this.nodes.add(node); } - public void put(Id id, long degree) { - this.degrees.put(id, degree); + public long degree() { + return this.degree; } - public long degree(Id id) { - return this.degrees.get(id); + public void degree(long degree) { + this.degree = degree; } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java index 8b263da4a2..e1783184bb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java @@ -64,10 +64,6 @@ public Map personalRank(Id source, String label, Id labelId = this.graph().edgeLabel(label).id(); Directions dir = this.getStartDirection(source, label); - long degree = this.degreeOfVertex(source, dir, labelId); - if (degree <= 0) { - return ranks; - } Set outSeeds = new HashSet<>(); Set inSeeds = new HashSet<>(); From 37ffcfd676d26f6a04f5da8daac0004b416ecca3 Mon Sep 17 00:00:00 2001 From: liningrui Date: Wed, 17 Apr 2019 16:12:30 +0800 Subject: [PATCH 11/15] tiny improve Change-Id: I4ae50946c46a016872848dbcd9f69e70b6f290a2 --- .../algorithm/NeighborRankTraverser.java | 7 ++- .../algorithm/PersonalRankTraverser.java | 48 ++++++++++--------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java index 484c7f6302..823820a5dd 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java @@ -116,6 +116,9 @@ public List> neighborRank(Id source, List steps) { } long degree = sameLayerNodesV.size() + prevLayerNodesV.size() + adjacenciesV.nodes().size(); + if (degree == 0L) { + continue; + } adjacenciesV.degree(degree); adjacencies.add(adjacenciesV); @@ -261,7 +264,7 @@ private static class Adjacencies { public Adjacencies(Id source) { this.source = source; this.nodes = new ArrayList<>(); - this.degree = 0L; + this.degree = -1L; } public Id source() { @@ -277,6 +280,8 @@ public void add(Node node) { } public long degree() { + E.checkArgument(degree > 0, + "The degree must be > 0, but got %s", degree); return this.degree; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java index e1783184bb..70f503ad5b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java @@ -22,23 +22,20 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.BiFunction; -import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.iterator.MapperIterator; import com.baidu.hugegraph.schema.EdgeLabel; import com.baidu.hugegraph.schema.VertexLabel; -import com.baidu.hugegraph.structure.HugeEdge; import com.baidu.hugegraph.structure.HugeVertex; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; -import com.google.common.collect.ImmutableMap; public class PersonalRankTraverser extends HugeTraverser { @@ -75,9 +72,9 @@ public Map personalRank(Id source, String label, Set firstAdjacencies = new HashSet<>(); for (long i = 0; i < this.maxDepth; i++) { - Map incrRanks = this.getIncrRanks(outSeeds, inSeeds, - labelId, ranks); - ranks = this.compensateSource(source, incrRanks); + Map incrRanks = this.calcIncrRanks(outSeeds, inSeeds, + labelId, ranks); + ranks = this.compensateRoot(source, incrRanks); if (i == 0) { firstAdjacencies.addAll(ranks.keySet()); } @@ -93,24 +90,29 @@ public Map personalRank(Id source, String label, return ranks; } - private Map getIncrRanks(Set outSeeds, Set inSeeds, - Id label, Map ranks) { + private Map calcIncrRanks(Set outSeeds, Set inSeeds, + Id label, Map ranks) { Map incrRanks = new HashMap<>(); BiFunction, Directions, Set> neighborIncrRanks; neighborIncrRanks = (seeds, dir) -> { Set tmpSeeds = new HashSet<>(); for (Id seed : seeds) { - long degree = this.degreeOfVertex(seed, dir, label); - assert degree > 0; - // Must be exist - double originRank = ranks.get(seed); - double spreadRank = originRank * alpha / degree; - - Iterator neighbors = this.adjacentVertices(seed, dir, label, - this.degree); + Double oldRank = ranks.get(seed); + E.checkState(oldRank != null, "Expect rank of seed exists"); + + Iterator iter = this.adjacentVertices(seed, dir, label, + this.degree); + List neighbors = IteratorUtils.list(iter); + + long degree = neighbors.size(); + if (degree == 0L) { + incrRanks.put(seed, oldRank); + continue; + } + double spreadRank = oldRank * alpha / degree; + // Collect all neighbors increment - while (neighbors.hasNext()) { - Id neighbor = neighbors.next(); + for (Id neighbor : neighbors) { tmpSeeds.add(neighbor); // Assign an initial value when firstly update neighbor rank double incrRank = incrRanks.getOrDefault(neighbor, 0.0); @@ -129,10 +131,10 @@ private Map getIncrRanks(Set outSeeds, Set inSeeds, return incrRanks; } - private Map compensateSource(Id source, Map incrRanks) { - double sourceRank = incrRanks.getOrDefault(source, 0.0); - sourceRank += (1 - this.alpha); - incrRanks.put(source, sourceRank); + private Map compensateRoot(Id root, Map incrRanks) { + double oldRank = incrRanks.getOrDefault(root, 0.0); + oldRank += (1 - this.alpha); + incrRanks.put(root, oldRank); return incrRanks; } From 66af119654102182e496cf2856b0e8e350bf719b Mon Sep 17 00:00:00 2001 From: liningrui Date: Wed, 17 Apr 2019 17:25:39 +0800 Subject: [PATCH 12/15] fix degree limit lost Change-Id: I9829eacea9e8598a4475b39bd9699ea951de6a97 --- .../baidu/hugegraph/backend/query/IdPrefixQuery.java | 1 + .../baidu/hugegraph/backend/query/IdRangeQuery.java | 1 + .../com/baidu/hugegraph/backend/query/Query.java | 9 +++++++++ .../backend/serializer/BinarySerializer.java | 12 +----------- .../hugegraph/traversal/algorithm/HugeTraverser.java | 1 + 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdPrefixQuery.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdPrefixQuery.java index a6e7ca3101..3a0119a649 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdPrefixQuery.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdPrefixQuery.java @@ -55,6 +55,7 @@ public IdPrefixQuery(HugeType resultType, Query originQuery, this.start = start; this.inclusiveStart = inclusive; this.prefix = prefix; + this.copyBasic(originQuery); } public Id start() { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdRangeQuery.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdRangeQuery.java index 7547254ea8..4d1bc2660c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdRangeQuery.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdRangeQuery.java @@ -57,6 +57,7 @@ public IdRangeQuery(HugeType resultType, Query originQuery, this.end = end; this.inclusiveStart = inclusiveStart; this.inclusiveEnd = inclusiveEnd; + this.copyBasic(originQuery); } public Id start() { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java index bdd35bc6c8..15dc48aa9d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java @@ -74,6 +74,15 @@ public Query(HugeType resultType, Query originQuery) { this.showDeleting = false; } + public void copyBasic(Query query) { + this.offset = query.offset(); + this.limit = query.limit(); + this.page = query.page(); + this.capacity = query.capacity(); + this.showHidden = query.showHidden(); + this.showDeleting = query.showDeleting(); + } + public HugeType resultType() { return this.resultType; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java index be4a27fd1f..95f3362128 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java @@ -661,9 +661,6 @@ private Query writeStringIndexQuery(ConditionQuery query) { } else { newQuery = new IdPrefixQuery(query, prefix); } - newQuery.page(query.page()); - newQuery.limit(query.limit()); - newQuery.offset(query.offset()); return newQuery; } @@ -707,11 +704,7 @@ private Query writeRangeIndexQuery(ConditionQuery query) { HugeType type = query.resultType(); if (keyEq != null) { Id id = formatIndexId(type, index, keyEq); - Query newQuery = new IdPrefixQuery(query, id); - newQuery.page(query.page()); - newQuery.limit(query.limit()); - newQuery.offset(query.offset()); - return newQuery; + return new IdPrefixQuery(query, id); } if (keyMin == null) { @@ -754,9 +747,6 @@ private Query writeRangeIndexQuery(ConditionQuery query) { } newQuery = new IdRangeQuery(query, start, keyMinEq, max, keyMaxEq); } - newQuery.page(query.page()); - newQuery.limit(query.limit()); - newQuery.offset(query.offset()); return newQuery; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java index d5adb24d81..b09638020d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java @@ -224,6 +224,7 @@ protected Iterator edgesOfVertex(Id source, Directions dir, ExtendableIterator results = new ExtendableIterator<>(); for (Id label : labels) { E.checkNotNull(label, "edge label"); + // TODO: limit should be applied to all labels results.extend(this.edgesOfVertex(source, dir, label, limit)); } return results; From 0f6df0f77311bf1d4ce5373288262af48fa7a803 Mon Sep 17 00:00:00 2001 From: liningrui Date: Wed, 17 Apr 2019 22:17:00 +0800 Subject: [PATCH 13/15] tiny improve Change-Id: Ib8c2da1f85c9e78504d704b0b3629a24b0666e42 --- .../api/traversers/NeighborRankAPI.java | 8 ++-- .../api/traversers/PersonalRankAPI.java | 15 ++++--- .../traversal/algorithm/HugeTraverser.java | 1 + .../algorithm/NeighborRankTraverser.java | 5 +-- .../algorithm/PersonalRankTraverser.java | 39 +++++++++---------- 5 files changed, 35 insertions(+), 33 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java index 3a1c4667d0..040bc7269f 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java @@ -21,9 +21,9 @@ import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_DEGREE; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEPTH; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.NO_LIMIT; -import static com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser.MAX_STEPS; import static com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser.MAX_TOP; import java.util.ArrayList; @@ -71,11 +71,11 @@ public String neighborRank(@Context GraphManager manager, "The source of rank request can't be null"); E.checkArgument(request.steps != null && !request.steps.isEmpty(), "The steps of rank request can't be empty"); - E.checkArgument(request.steps.size() <= MAX_STEPS, + E.checkArgument(request.steps.size() <= Long.valueOf(DEFAULT_MAX_DEPTH), "The steps length of rank request can't exceed %s", - MAX_STEPS); + DEFAULT_MAX_DEPTH); E.checkArgument(request.alpha > 0 && request.alpha <= 1.0, - "The alpha of rank request must belong (0, 1], " + + "The alpha of rank request must be in range (0, 1], " + "but got '%s'", request.alpha); LOG.debug("Graph [{}] get neighbor rank from '{}' with steps '{}', " + diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java index 5b62d015b6..17d6ae35e1 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java @@ -21,6 +21,7 @@ import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_DEGREE; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_LIMIT; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEPTH; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.NO_LIMIT; import java.util.Map; @@ -66,7 +67,7 @@ public String personalRank(@Context GraphManager manager, E.checkArgument(request.label != null, "The edge label of rank request can't be null"); E.checkArgument(request.alpha > 0 && request.alpha <= 1.0, - "The alpha of rank request must belong (0, 1], " + + "The alpha of rank request must be in range (0, 1], " + "but got '%s'", request.alpha); E.checkArgument(request.degree > 0 || request.degree == NO_LIMIT, "The degree of rank request must be > 0, but got: %s", @@ -74,9 +75,11 @@ public String personalRank(@Context GraphManager manager, E.checkArgument(request.limit > 0 || request.limit == NO_LIMIT, "The limit of rank request must be > 0, but got: %s", request.limit); - E.checkArgument(request.maxDepth >= 1, - "The max depth of rank request must >= 1, but got '%s'", - request.maxDepth); + E.checkArgument(request.maxDepth > 0 && + request.maxDepth <= Long.valueOf(DEFAULT_MAX_DEPTH), + "The max depth of rank request must be " + + "in range (0, %s], but got '%s'", + DEFAULT_MAX_DEPTH, request.maxDepth); LOG.debug("Graph [{}] get personal rank from '{}' with " + "edge label '{}', alpha '{}', degree '{}', " + @@ -137,8 +140,8 @@ private static class RankRequest { @Override public String toString() { - return String.format("RankRequest{source=%s,label=%s," + - "alpha=%s,degree=%s,limit=%s, maxDepth=%s," + + return String.format("RankRequest{source=%s,label=%s,alpha=%s" + + "degree=%s,limit=%s,maxDepth=%s," + "withLabel=%s,sorted=%s}", this.source, this.label, this.alpha, this.degree, this.limit, this.maxDepth, diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java index b09638020d..279064fcaa 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java @@ -66,6 +66,7 @@ public class HugeTraverser { public static final String DEFAULT_LIMIT = "100"; public static final String DEFAULT_DEGREE = "10000"; public static final String DEFAULT_SAMPLE = "100"; + public static final String DEFAULT_MAX_DEPTH = "50"; public static final String DEFAULT_WEIGHT = "0"; // Empirical value of scan limit, with which results can be returned in 3s diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java index 823820a5dd..0766811d49 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java @@ -41,7 +41,6 @@ public class NeighborRankTraverser extends HugeTraverser { - public static final int MAX_STEPS = 100; public static final int MAX_TOP = 1000; public static final int DEFAULT_CAPACITY_PER_LAYER = 100000; @@ -168,8 +167,8 @@ private boolean belongToPrevLayers(List ranks, Id target, for (int i = ranks.size() - 2; i > 0; i--) { Ranks prevLayerRanks = ranks.get(i); if (prevLayerRanks.containsKey(target)) { - Set get = prevLayerNodes.computeIfAbsent(i, HashSet::new); - get.add(target); + Set nodes = prevLayerNodes.computeIfAbsent(i, HashSet::new); + nodes.add(target); return true; } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java index 70f503ad5b..ab96e984af 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java @@ -70,17 +70,17 @@ public Map personalRank(Id source, String label, inSeeds.add(source); } - Set firstAdjacencies = new HashSet<>(); + Set rootAdjacencies = new HashSet<>(); for (long i = 0; i < this.maxDepth; i++) { - Map incrRanks = this.calcIncrRanks(outSeeds, inSeeds, - labelId, ranks); - ranks = this.compensateRoot(source, incrRanks); + Map newRanks = this.calcNewRanks(outSeeds, inSeeds, + labelId, ranks); + ranks = this.compensateRoot(source, newRanks); if (i == 0) { - firstAdjacencies.addAll(ranks.keySet()); + rootAdjacencies.addAll(ranks.keySet()); } } // Remove directly connected neighbors - removeAll(ranks, firstAdjacencies); + removeAll(ranks, rootAdjacencies); // Remove unnecessary label if (withLabel == WithLabel.SAME_LABEL) { removeAll(ranks, dir == Directions.OUT ? inSeeds : outSeeds); @@ -90,9 +90,9 @@ public Map personalRank(Id source, String label, return ranks; } - private Map calcIncrRanks(Set outSeeds, Set inSeeds, - Id label, Map ranks) { - Map incrRanks = new HashMap<>(); + private Map calcNewRanks(Set outSeeds, Set inSeeds, + Id label, Map ranks) { + Map newRanks = new HashMap<>(); BiFunction, Directions, Set> neighborIncrRanks; neighborIncrRanks = (seeds, dir) -> { Set tmpSeeds = new HashSet<>(); @@ -106,18 +106,17 @@ private Map calcIncrRanks(Set outSeeds, Set inSeeds, long degree = neighbors.size(); if (degree == 0L) { - incrRanks.put(seed, oldRank); + newRanks.put(seed, oldRank); continue; } - double spreadRank = oldRank * alpha / degree; + double incrRank = oldRank * alpha / degree; // Collect all neighbors increment for (Id neighbor : neighbors) { tmpSeeds.add(neighbor); // Assign an initial value when firstly update neighbor rank - double incrRank = incrRanks.getOrDefault(neighbor, 0.0); - incrRank += spreadRank; - incrRanks.put(neighbor, incrRank); + double rank = newRanks.getOrDefault(neighbor, 0.0); + newRanks.put(neighbor, rank + incrRank); } } return tmpSeeds; @@ -128,14 +127,14 @@ private Map calcIncrRanks(Set outSeeds, Set inSeeds, outSeeds.addAll(tmpOutSeeds); inSeeds.addAll(tmpInSeeds); - return incrRanks; + return newRanks; } - private Map compensateRoot(Id root, Map incrRanks) { - double oldRank = incrRanks.getOrDefault(root, 0.0); - oldRank += (1 - this.alpha); - incrRanks.put(root, oldRank); - return incrRanks; + private Map compensateRoot(Id root, Map newRanks) { + double rank = newRanks.getOrDefault(root, 0.0); + rank += (1 - this.alpha); + newRanks.put(root, rank); + return newRanks; } private Directions getStartDirection(Id source, String label) { From c498f562657f0551f5c3c654ee151008f733da3e Mon Sep 17 00:00:00 2001 From: liningrui Date: Thu, 18 Apr 2019 20:08:29 +0800 Subject: [PATCH 14/15] remove status created annotation Change-Id: Iccda41b7869a53a77177d342754f080f6e4612e4 --- .../api/traversers/CustomizedCrosspointsAPI.java | 9 +++++---- .../hugegraph/api/traversers/CustomizedPathsAPI.java | 11 +++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CustomizedCrosspointsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CustomizedCrosspointsAPI.java index 797e6ec9ff..76b33a385d 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CustomizedCrosspointsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CustomizedCrosspointsAPI.java @@ -19,6 +19,11 @@ package com.baidu.hugegraph.api.traversers; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_DEGREE; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.NO_LIMIT; + import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -41,7 +46,6 @@ import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.api.API; -import com.baidu.hugegraph.api.filter.StatusFilter.Status; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.core.GraphManager; import com.baidu.hugegraph.schema.EdgeLabel; @@ -55,8 +59,6 @@ import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.annotation.JsonProperty; -import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.*; - @Path("graphs/{graph}/traversers/customizedcrosspoints") @Singleton public class CustomizedCrosspointsAPI extends API { @@ -65,7 +67,6 @@ public class CustomizedCrosspointsAPI extends API { @POST @Timed - @Status(Status.CREATED) @Consumes(APPLICATION_JSON) @Produces(APPLICATION_JSON_WITH_CHARSET) public String post(@Context GraphManager manager, diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CustomizedPathsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CustomizedPathsAPI.java index 462aa13fa1..8b9c58a838 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CustomizedPathsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CustomizedPathsAPI.java @@ -19,6 +19,13 @@ package com.baidu.hugegraph.api.traversers; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_DEGREE; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_SAMPLE; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_WEIGHT; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.NO_LIMIT; + import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -41,7 +48,6 @@ import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.api.API; -import com.baidu.hugegraph.api.filter.StatusFilter.Status; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.core.GraphManager; import com.baidu.hugegraph.schema.EdgeLabel; @@ -56,8 +62,6 @@ import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.annotation.JsonProperty; -import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.*; - @Path("graphs/{graph}/traversers/customizedpaths") @Singleton public class CustomizedPathsAPI extends API { @@ -66,7 +70,6 @@ public class CustomizedPathsAPI extends API { @POST @Timed - @Status(Status.CREATED) @Consumes(APPLICATION_JSON) @Produces(APPLICATION_JSON_WITH_CHARSET) public String post(@Context GraphManager manager, From 6b2ab2828121052a395179e29bc75b55d88576f2 Mon Sep 17 00:00:00 2001 From: liningrui Date: Thu, 18 Apr 2019 22:26:06 +0800 Subject: [PATCH 15/15] use HugeVertex.getIdValue Change-Id: I9cb451b4612ff540da9e1045a44970e119d35d6e --- .../com/baidu/hugegraph/api/traversers/NeighborRankAPI.java | 4 ++-- .../com/baidu/hugegraph/api/traversers/PersonalRankAPI.java | 6 +++--- .../traversal/algorithm/NeighborRankTraverser.java | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java index 040bc7269f..4b7b38e51b 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java @@ -42,11 +42,11 @@ import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.api.API; -import com.baidu.hugegraph.api.graph.VertexAPI; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.core.GraphManager; import com.baidu.hugegraph.schema.EdgeLabel; import com.baidu.hugegraph.server.RestServer; +import com.baidu.hugegraph.structure.HugeVertex; import com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; @@ -82,7 +82,7 @@ public String neighborRank(@Context GraphManager manager, "alpha '{}' and capacity '{}'", graph, request.source, request.steps, request.alpha, request.capacity); - Id sourceId = VertexAPI.checkAndParseVertexId(request.source); + Id sourceId = HugeVertex.getIdValue(request.source); HugeGraph g = graph(manager, graph); List steps = steps(g, request); diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java index 17d6ae35e1..8f6445ffaa 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java @@ -37,10 +37,10 @@ import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.api.API; -import com.baidu.hugegraph.api.graph.VertexAPI; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.core.GraphManager; import com.baidu.hugegraph.server.RestServer; +import com.baidu.hugegraph.structure.HugeVertex; import com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser; import com.baidu.hugegraph.util.CollectionUtil; import com.baidu.hugegraph.util.E; @@ -87,7 +87,7 @@ public String personalRank(@Context GraphManager manager, graph, request.source, request.label, request.alpha, request.degree, request.maxDepth, request.sorted); - Id sourceId = VertexAPI.checkAndParseVertexId(request.source); + Id sourceId = HugeVertex.getIdValue(request.source); HugeGraph g = graph(manager, graph); PersonalRankTraverser traverser; @@ -140,7 +140,7 @@ private static class RankRequest { @Override public String toString() { - return String.format("RankRequest{source=%s,label=%s,alpha=%s" + + return String.format("RankRequest{source=%s,label=%s,alpha=%s," + "degree=%s,limit=%s,maxDepth=%s," + "withLabel=%s,sorted=%s}", this.source, this.label, this.alpha, diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java index 0766811d49..5bebb02287 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java @@ -37,7 +37,6 @@ import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.OrderLimitMap; -import com.google.common.collect.ImmutableList; public class NeighborRankTraverser extends HugeTraverser {