From 8fa9ca632d42cc526334f92c162f01f670c1d39e Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 26 Nov 2020 15:15:24 +0800 Subject: [PATCH 01/38] add Eclipse Collection dependency Change-Id: I501a18a70054fad3c40bd6b9e852a82a5c185949 --- hugegraph-core/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hugegraph-core/pom.xml b/hugegraph-core/pom.xml index d9fb1a1700..9853350b53 100644 --- a/hugegraph-core/pom.xml +++ b/hugegraph-core/pom.xml @@ -157,6 +157,16 @@ commons-compress 1.20 + + org.eclipse.collections + eclipse-collections-api + 10.4.0 + + + org.eclipse.collections + eclipse-collections + 10.4.0 + From 520490165d11e34f8ce2b30779bfb49f7f9a0be3 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 26 Nov 2020 21:45:29 +0800 Subject: [PATCH 02/38] use primitive map for properties of HugeElement from Map> to MutableIntObjectMap> Change-Id: Icd0a394c1ac110193fa244e818bce85b9cd78e08 --- .../backend/cache/CachedBackendStore.java | 6 +++ .../baidu/hugegraph/backend/id/EdgeId.java | 5 ++ .../com/baidu/hugegraph/backend/id/Id.java | 2 + .../hugegraph/backend/id/IdGenerator.java | 20 ++++++++ .../serializer/BinaryBackendEntry.java | 5 ++ .../hugegraph/structure/HugeElement.java | 50 +++++++++++-------- 6 files changed, 68 insertions(+), 20 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedBackendStore.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedBackendStore.java index b2daec42fb..e27f4899f6 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedBackendStore.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedBackendStore.java @@ -225,6 +225,12 @@ public long asLong() { return 0L; } + @Override + public int asInt() { + // TODO: improve + return 0; + } + @Override public byte[] asBytes() { return StringEncoding.encode(this.query); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java index 1c69e55cc4..0673888d3c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java @@ -155,6 +155,11 @@ public long asLong() { throw new UnsupportedOperationException(); } + @Override + public int asInt() { + throw new UnsupportedOperationException(); + } + @Override public byte[] asBytes() { return StringEncoding.encode(this.asString()); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/Id.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/Id.java index 78afad4823..689fd5b74d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/Id.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/Id.java @@ -31,6 +31,8 @@ public interface Id extends Comparable { public long asLong(); + public int asInt(); + public byte[] asBytes(); public int length(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java index 1858fb4c0b..224de5da8e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java @@ -160,6 +160,11 @@ public long asLong() { return Long.parseLong(this.id); } + @Override + public int asInt() { + return Integer.parseInt(this.id); + } + @Override public byte[] asBytes() { return StringEncoding.encode(this.id); @@ -233,6 +238,11 @@ public long asLong() { return this.id; } + @Override + public int asInt() { + return (int) this.id; + } + @Override public byte[] asBytes() { return NumericUtil.longToBytes(this.id); @@ -328,6 +338,11 @@ public long asLong() { throw new UnsupportedOperationException(); } + @Override + public int asInt() { + throw new UnsupportedOperationException(); + } + @Override public byte[] asBytes() { BytesBuffer buffer = BytesBuffer.allocate(16); @@ -410,6 +425,11 @@ public long asLong() { throw new UnsupportedOperationException(); } + @Override + public int asInt() { + throw new UnsupportedOperationException(); + } + @Override public byte[] asBytes() { throw new UnsupportedOperationException(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryBackendEntry.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryBackendEntry.java index 6b77141d7a..3bfbbbca8a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryBackendEntry.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryBackendEntry.java @@ -208,6 +208,11 @@ public long asLong() { throw new UnsupportedOperationException(); } + @Override + public int asInt() { + throw new UnsupportedOperationException(); + } + @Override public int compareTo(Id other) { return Bytes.compare(this.bytes, other.asBytes()); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java index 517fc38679..6188576a06 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -35,6 +34,9 @@ import org.apache.tinkerpop.gremlin.structure.Property; import org.apache.tinkerpop.gremlin.structure.T; import org.apache.tinkerpop.gremlin.structure.util.ElementHelper; +import org.eclipse.collections.api.map.primitive.MutableIntObjectMap; +import org.eclipse.collections.api.tuple.primitive.IntObjectPair; +import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.id.EdgeId; @@ -52,15 +54,15 @@ import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.util.CollectionUtil; import com.baidu.hugegraph.util.E; -import com.google.common.collect.ImmutableMap; public abstract class HugeElement implements Element, GraphType, Idfiable { - private static final Map> EMPTY_MAP = ImmutableMap.of(); + private static final MutableIntObjectMap> EMPTY_MAP = + new IntObjectHashMap<>(); private static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX; private final HugeGraph graph; - private Map> properties; + private MutableIntObjectMap> properties; private long expiredTime; // TODO: move into properties to keep small object @@ -98,7 +100,7 @@ protected void updateToDefaultValueIfNone() { this.defaultValueUpdated = true; // Set default value if needed for (Id pkeyId : this.schemaLabel().properties()) { - if (this.properties.containsKey(pkeyId)) { + if (this.properties.containsKey(pkeyId.asInt())) { continue; } PropertyKey pkey = this.graph().propertyKey(pkeyId); @@ -198,28 +200,36 @@ public boolean hasTtl() { return this.schemaLabel().ttl() > 0L; } + // TODO: return MutableIntObjectMap> public Map> getProperties() { - return Collections.unmodifiableMap(this.properties); + Map> props = new HashMap<>(); + for (IntObjectPair> e : this.properties.keyValuesView()) { + props.put(IdGenerator.of(e.getOne()), e.getTwo()); + } + return props; } + // TODO: return MutableIntObjectMap> public Map> getFilledProperties() { this.ensureFilledProperties(true); return this.getProperties(); } + // TODO: return MutableIntObjectMap> public Map getPropertiesMap() { Map props = new HashMap<>(); - for (Map.Entry> e : this.properties.entrySet()) { - props.put(e.getKey(), e.getValue().value()); + for (IntObjectPair> e : this.properties.keyValuesView()) { + props.put(IdGenerator.of(e.getOne()), e.getTwo().value()); } return props; } + // TODO: return MutableIntObjectMap> public Map> getAggregateProperties() { Map> aggrProps = new HashMap<>(); - for (Map.Entry> e : this.properties.entrySet()) { - if (e.getValue().type().isAggregateProperty()) { - aggrProps.put(e.getKey(), e.getValue()); + for (IntObjectPair> e : this.properties.keyValuesView()) { + if (e.getTwo().type().isAggregateProperty()) { + aggrProps.put(IdGenerator.of(e.getOne()), e.getTwo()); } } return aggrProps; @@ -227,12 +237,12 @@ public Map> getAggregateProperties() { @SuppressWarnings("unchecked") public HugeProperty getProperty(Id key) { - return (HugeProperty) this.properties.get(key); + return (HugeProperty) this.properties.get(key.asInt()); } @SuppressWarnings("unchecked") public V getPropertyValue(Id key) { - HugeProperty prop = this.properties.get(key); + HugeProperty prop = this.properties.get(key.asInt()); if (prop == null) { return null; } @@ -240,7 +250,7 @@ public V getPropertyValue(Id key) { } public boolean hasProperty(Id key) { - return this.properties.containsKey(key); + return this.properties.containsKey(key.asInt()); } public boolean hasProperties() { @@ -266,17 +276,17 @@ public int sizeOfSubProperties() { @Watched(prefix = "element") public HugeProperty setProperty(HugeProperty prop) { if (this.properties == EMPTY_MAP) { - this.properties = new HashMap<>(); + this.properties = new IntObjectHashMap<>(); } PropertyKey pkey = prop.propertyKey(); - E.checkArgument(this.properties.containsKey(pkey.id()) || + E.checkArgument(this.properties.containsKey(pkey.id().asInt()) || this.properties.size() < MAX_PROPERTIES, "Exceeded the maximum number of properties"); - return this.properties.put(pkey.id(), prop); + return this.properties.put(pkey.id().asInt(), prop); } public HugeProperty removeProperty(Id key) { - return this.properties.remove(key); + return this.properties.remove(key.asInt()); } public HugeProperty addProperty(PropertyKey pkey, V value) { @@ -353,7 +363,7 @@ private HugeProperty addProperty(PropertyKey pkey, V value, } public void resetProperties() { - this.properties = new HashMap<>(); + this.properties = new IntObjectHashMap<>(); this.propLoaded = false; } @@ -361,7 +371,7 @@ protected void copyProperties(HugeElement element) { if (element.properties == EMPTY_MAP) { this.properties = EMPTY_MAP; } else { - this.properties = new HashMap<>(element.properties); + this.properties = new IntObjectHashMap<>(element.properties); } this.propLoaded = true; } From 33c82508fb24ffbb5888761725ca2854b56646da Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Wed, 24 Mar 2021 21:58:10 +0800 Subject: [PATCH 03/38] allow config oltp collections(JCF or EC) Change-Id: Iaa5eb686d2c0b2aeec17cfcf5c742f6cbfe02d35 --- .../api/traversers/AllShortestPathsAPI.java | 3 +- .../api/traversers/CrosspointsAPI.java | 3 +- .../hugegraph/api/traversers/PathsAPI.java | 3 +- .../hugegraph/api/traversers/RaysAPI.java | 2 +- .../hugegraph/api/traversers/RingsAPI.java | 3 +- .../baidu/hugegraph/StandardHugeGraph.java | 1 + .../backend/store/ram/IntLongMap.java | 2 + .../baidu/hugegraph/config/CoreOptions.java | 9 + .../baidu/hugegraph/structure/HugeEdge.java | 4 +- .../baidu/hugegraph/structure/HugeVertex.java | 2 + .../algorithm/CollectionPathsTraverser.java | 5 +- .../traversal/algorithm/CountTraverser.java | 3 +- .../algorithm/CustomizePathsTraverser.java | 9 +- .../CustomizedCrosspointsTraverser.java | 16 +- .../FusiformSimilarityTraverser.java | 36 ++-- .../traversal/algorithm/HugeTraverser.java | 90 ++++++-- .../algorithm/JaccardSimilarTraverser.java | 8 +- .../traversal/algorithm/KoutTraverser.java | 4 +- .../MultiNodeShortestPathTraverser.java | 7 +- .../algorithm/NeighborRankTraverser.java | 20 +- .../traversal/algorithm/PathsTraverser.java | 2 +- .../algorithm/PersonalRankTraverser.java | 14 +- .../algorithm/ShortestPathTraverser.java | 10 +- .../SingleSourceShortestPathTraverser.java | 9 +- .../algorithm/SubGraphTraverser.java | 7 +- .../algorithm/TemplatePathsTraverser.java | 5 +- .../strategy/SingleTraverseStrategy.java | 2 +- .../type/define/CollectionImplType.java | 60 ++++++ .../util/collection/CollectionFactory.java | 197 ++++++++++++++++++ .../hugegraph/util/collection/IdList.java | 133 ++++++++++++ .../hugegraph/util/collection/IdSet.java | 131 ++++++++++++ 31 files changed, 695 insertions(+), 105 deletions(-) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdList.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java index 21371d06cb..d26aef7048 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java @@ -89,6 +89,7 @@ public String get(@Context GraphManager manager, HugeTraverser.PathSet paths = traverser.allShortestPaths( sourceId, targetId, dir, edgeLabels, depth, maxDegree, skipDegree, capacity); - return manager.serializer(g).writePaths("paths", paths, false); + return manager.serializer(g).writePaths("paths", paths.paths(), + false); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CrosspointsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CrosspointsAPI.java index 30ca26ac52..59f054d18a 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CrosspointsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CrosspointsAPI.java @@ -84,6 +84,7 @@ public String get(@Context GraphManager manager, HugeTraverser.PathSet paths = traverser.paths(sourceId, dir, targetId, dir, edgeLabel, depth, maxDegree, capacity, limit); - return manager.serializer(g).writePaths("crosspoints", paths, true); + return manager.serializer(g).writePaths("crosspoints", + paths.paths(),true); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java index cd1096e29d..e98e1384ec 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java @@ -97,7 +97,8 @@ public String get(@Context GraphManager manager, dir.opposite(), edgeLabel, depth, maxDegree, capacity, limit); - return manager.serializer(g).writePaths("paths", paths, false); + return manager.serializer(g).writePaths("paths", paths.paths(), + false); } @POST diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RaysAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RaysAPI.java index bb36766782..8ba11cd892 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RaysAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RaysAPI.java @@ -83,6 +83,6 @@ public String get(@Context GraphManager manager, HugeTraverser.PathSet paths = traverser.rays(source, dir, edgeLabel, depth, maxDegree, capacity, limit); - return manager.serializer(g).writePaths("rays", paths, false); + return manager.serializer(g).writePaths("rays", paths.paths(), false); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RingsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RingsAPI.java index 6879da007e..b576785eec 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RingsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RingsAPI.java @@ -86,6 +86,7 @@ public String get(@Context GraphManager manager, HugeTraverser.PathSet paths = traverser.rings(source, dir, edgeLabel, depth, sourceInRing, maxDegree, capacity, limit); - return manager.serializer(g).writePaths("rings", paths, false); + return manager.serializer(g).writePaths("rings", paths.paths(), + false); } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java index cfa1aec501..d09b553035 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java @@ -123,6 +123,7 @@ public class StandardHugeGraph implements HugeGraph { CoreOptions.TASK_RESULT_SIZE_LIMIT, CoreOptions.OLTP_CONCURRENT_THREADS, CoreOptions.OLTP_CONCURRENT_DEPTH, + CoreOptions.OLTP_COLLECTION_IMPL_TYPE, CoreOptions.VERTEX_DEFAULT_LABEL, CoreOptions.VERTEX_ENCODE_PK_NUMBER ); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/IntLongMap.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/IntLongMap.java index 7bbb7b98c9..36ef9428fa 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/IntLongMap.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/IntLongMap.java @@ -64,6 +64,7 @@ public long get(int key) { @Override public void clear() { Arrays.fill(this.array, 0L); + this.size = 0; } @Override @@ -90,5 +91,6 @@ public void readFrom(DataInputStream buffer) throws IOException { long value = buffer.readLong(); this.array[i] = value; } + this.size = size; } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java index 24fcf606ae..6ca6976b15 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java @@ -623,4 +623,13 @@ public static synchronized CoreOptions instance() { rangeInt(0, 65535), 10 ); + + public static final ConfigOption OLTP_COLLECTION_IMPL_TYPE = + new ConfigOption<>( + "oltp.collection_impl_type", + "The implementation type of collections " + + "used in oltp algorithm.", + allowValues("jcf", "ec"), + "ec" + ); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java index 70118634c7..3503916e17 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java @@ -499,10 +499,10 @@ public static HugeEdge constructEdge(HugeVertex ownerVertex, VertexLabel otherVertexLabel; if (isOutEdge) { - ownerVertex.correctVertexLabel(srcLabel); +// ownerVertex.correctVertexLabel(srcLabel); otherVertexLabel = tgtLabel; } else { - ownerVertex.correctVertexLabel(tgtLabel); +// ownerVertex.correctVertexLabel(tgtLabel); otherVertexLabel = srcLabel; } HugeVertex otherVertex = new HugeVertex(graph, otherVertexId, diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java index 7d1b0246fb..f53530c9b7 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java @@ -66,6 +66,8 @@ public class HugeVertex extends HugeElement implements Vertex, Cloneable { private Id id; private VertexLabel label; + // Implemented as LinkedHashMap, is it necessary? Eclipse Collection does + // not have replacement for LinkedHashMap now. private Set edges; public HugeVertex(final HugeGraph graph, Id id, VertexLabel label) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CollectionPathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CollectionPathsTraverser.java index da76202e5c..03e2b7f74e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CollectionPathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CollectionPathsTraverser.java @@ -19,7 +19,6 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -48,7 +47,7 @@ public Collection paths(Iterator sources, checkCapacity(capacity); checkLimit(limit); - List sourceList = new ArrayList<>(); + List sourceList = newIdList(); while (sources.hasNext()) { sourceList.add(((HugeVertex) sources.next()).id()); } @@ -56,7 +55,7 @@ public Collection paths(Iterator sources, E.checkState(sourceSize >= 1 && sourceSize <= MAX_VERTICES, "The number of source vertices must in [1, %s], " + "but got: %s", MAX_VERTICES, sourceList.size()); - List targetList = new ArrayList<>(); + List targetList = newIdList(); while (targets.hasNext()) { targetList.add(((HugeVertex) targets.next()).id()); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CountTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CountTraverser.java index ee27b40c44..af8ad2c830 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CountTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CountTraverser.java @@ -19,7 +19,6 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -40,7 +39,7 @@ public class CountTraverser extends HugeTraverser { private boolean containsTraversed = false; private long dedupSize = 1000000L; - private final Set dedupSet = new HashSet<>(); + private final Set dedupSet = newIdSet(); private final MutableLong count = new MutableLong(0L); public CountTraverser(HugeGraph graph) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CustomizePathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CustomizePathsTraverser.java index 4c1f73eb09..ad5bf5c80b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CustomizePathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CustomizePathsTraverser.java @@ -19,7 +19,6 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -74,7 +73,7 @@ public List customizedPaths(Iterator vertices, // Traversal vertices of previous level for (Map.Entry> entry : sources.entrySet()) { - List adjacency = new ArrayList<>(); + List adjacency = newList(); edges = this.edgesOfVertex(entry.getKey(), step.step()); while (edges.hasNext()) { HugeEdge edge = (HugeEdge) edges.next(); @@ -122,7 +121,7 @@ public List customizedPaths(Iterator vertices, if (stepNum != 0) { return ImmutableList.of(); } - List paths = new ArrayList<>(); + List paths = newList(); for (List nodes : newVertices.values()) { for (Node n : nodes) { if (sorted) { @@ -155,7 +154,7 @@ private static List sample(List nodes, long sample) { if (nodes.size() <= sample) { return nodes; } - List result = new ArrayList<>((int) sample); + List result = newList((int) sample); int size = nodes.size(); for (int random : CollectionUtil.randomSet(0, size, (int) sample)) { result.add(nodes.get(random)); @@ -173,7 +172,7 @@ public WeightNode(Id id, Node parent, double weight) { } public List weights() { - List weights = new ArrayList<>(); + List weights = newList(); WeightNode current = this; while (current.parent() != null) { weights.add(current.weight); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CustomizedCrosspointsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CustomizedCrosspointsTraverser.java index 4345646d55..253c7bb200 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CustomizedCrosspointsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CustomizedCrosspointsTraverser.java @@ -19,9 +19,7 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -60,14 +58,14 @@ public CrosspointsPaths crosspointsPaths(Iterator vertices, checkCapacity(capacity); checkLimit(limit); MultivaluedMap initialSources = newMultivalueMap(); - List verticesList = new ArrayList<>(); + List verticesList = newList(); while (vertices.hasNext()) { HugeVertex vertex = (HugeVertex) vertices.next(); verticesList.add(vertex); Node node = new Node(vertex.id(), null); initialSources.add(vertex.id(), node); } - List paths = new ArrayList<>(); + List paths = newList(); for (PathPattern pathPattern : pathPatterns) { MultivaluedMap sources = initialSources; @@ -81,7 +79,7 @@ public CrosspointsPaths crosspointsPaths(Iterator vertices, // Traversal vertices of previous level for (Map.Entry> entry : sources.entrySet()) { - List adjacency = new ArrayList<>(); + List adjacency = newList(); edges = this.edgesOfVertex(entry.getKey(), step.edgeStep); while (edges.hasNext()) { HugeEdge edge = (HugeEdge) edges.next(); @@ -151,11 +149,11 @@ private static CrosspointsPaths intersectionPaths(List sources, // Limit intersection number to limit crosspoints vertices in result int size = intersection.size(); if (limit != NO_LIMIT && size > limit) { - intersection = new ArrayList<>(intersection).subList(0, size - 1); + intersection = newList(intersection).subList(0, size - 1); } // Filter intersection paths - List results = new ArrayList<>(); + List results = newList(); for (Path path : paths) { List vertices = path.vertices(); int length = vertices.size(); @@ -163,7 +161,7 @@ private static CrosspointsPaths intersectionPaths(List sources, results.add(path); } } - return new CrosspointsPaths(new HashSet<>(intersection), results); + return new CrosspointsPaths(newSet(intersection), results); } public static class PathPattern { @@ -171,7 +169,7 @@ public static class PathPattern { private List steps; public PathPattern() { - this.steps = new ArrayList<>(); + this.steps = newList(); } public List steps() { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/FusiformSimilarityTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/FusiformSimilarityTraverser.java index 320833f1cd..03cd92d214 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/FusiformSimilarityTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/FusiformSimilarityTraverser.java @@ -19,8 +19,6 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -106,9 +104,9 @@ private Set fusiformSimilarityForVertex( // Get similar nodes and counts Iterator edges = this.edgesOfVertex(vertex.id(), direction, labelId, degree); - Map similars = new HashMap<>(); + Map similars = newMap(); MultivaluedMap intermediaries = new MultivaluedHashMap<>(); - Set neighbors = new HashSet<>(); + Set neighbors = newIdSet(); while (edges.hasNext()) { Id target = ((HugeEdge) edges.next()).id().otherVertexId(); if (neighbors.contains(target)) { @@ -120,7 +118,7 @@ private Set fusiformSimilarityForVertex( Directions backDir = direction.opposite(); Iterator backEdges = this.edgesOfVertex(target, backDir, labelId, degree); - Set currentSimilars = new HashSet<>(); + Set currentSimilars = newIdSet(); while (backEdges.hasNext()) { Id node = ((HugeEdge) backEdges.next()).id().otherVertexId(); if (currentSimilars.contains(node)) { @@ -149,7 +147,7 @@ private Set fusiformSimilarityForVertex( } // Match alpha double neighborNum = neighbors.size(); - Map matchedAlpha = new HashMap<>(); + Map matchedAlpha = newMap(); for (Map.Entry entry : similars.entrySet()) { double score = entry.getValue().intValue() / neighborNum; if (score >= alpha) { @@ -169,7 +167,7 @@ private Set fusiformSimilarityForVertex( } // Filter by groupCount by property if (groupProperty != null) { - Set values = new HashSet<>(); + Set values = newSet(); // Add groupProperty value of source vertex values.add(vertex.value(groupProperty)); for (Id id : topN.keySet()) { @@ -226,7 +224,7 @@ private boolean matchMinNeighborCount(HugeVertex vertex, neighborCount = IteratorUtils.count(edges); } else { edges = this.edgesOfVertex(vertex.id(), direction, labelId, degree); - Set neighbors = new HashSet<>(); + Set neighbors = newIdSet(); while (edges.hasNext()) { Id target = ((HugeEdge) edges.next()).id().otherVertexId(); neighbors.add(target); @@ -252,8 +250,8 @@ public static class Similar { public Similar(Id id, double score, List intermediaries) { this.id = id; this.score = score; - assert new HashSet<>(intermediaries).size() == - intermediaries.size() : "Invalid intermediaries"; + assert newSet(intermediaries).size() == intermediaries.size() : + "Invalid intermediaries"; this.intermediaries = intermediaries; } @@ -275,14 +273,20 @@ public Map toMap() { } } - public static class SimilarsMap extends HashMap> { + public static class SimilarsMap { private static final long serialVersionUID = -1906770930513268291L; + private Map> similars = newMap(); + + public void put(Id id, Set similars) { + this.similars.put(id, similars); + } + public Set vertices() { - Set vertices = new HashSet<>(); - vertices.addAll(this.keySet()); - for (Set similars : this.values()) { + Set vertices = newIdSet(); + vertices.addAll(this.similars.keySet()); + for (Set similars : this.similars.values()) { for (Similar similar : similars) { vertices.add(similar.id()); vertices.addAll(similar.intermediaries()); @@ -292,8 +296,8 @@ public Set vertices() { } public Map>> toMap() { - Map>> results = new HashMap<>(); - for (Map.Entry> entry : this.entrySet()) { + Map>> results = newMap(); + for (Map.Entry> entry : this.similars.entrySet()) { Id source = entry.getKey(); Set similars = entry.getValue(); Set> result = InsertionOrderUtil.newSet(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java index 3ec1362b8b..ef88cce4d6 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java @@ -19,10 +19,8 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -55,10 +53,12 @@ import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.traversal.optimize.TraversalUtil; import com.baidu.hugegraph.type.HugeType; +import com.baidu.hugegraph.type.define.CollectionImplType; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.util.CollectionUtil; import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.collection.CollectionFactory; import com.baidu.hugegraph.util.InsertionOrderUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -68,6 +68,8 @@ public class HugeTraverser { private HugeGraph graph; + private static CollectionFactory collectionFactory; + public static final String DEFAULT_CAPACITY = "10000000"; public static final String DEFAULT_ELEMENTS_LIMIT = "10000000"; public static final String DEFAULT_PATHS_LIMIT = "10"; @@ -87,6 +89,7 @@ public class HugeTraverser { public HugeTraverser(HugeGraph graph) { this.graph = graph; + collectionFactory = new CollectionFactory(this.collectionImplType()); } public HugeGraph graph() { @@ -97,6 +100,11 @@ protected int concurrentDepth() { return this.graph.option(CoreOptions.OLTP_CONCURRENT_DEPTH); } + private CollectionImplType collectionImplType() { + return CollectionImplType.valueOf(this.graph.option( + CoreOptions.OLTP_COLLECTION_IMPL_TYPE).toUpperCase()); + } + protected Set adjacentVertices(Id sourceV, Set vertices, Directions dir, Id label, Set excluded, long degree, @@ -105,7 +113,7 @@ protected Set adjacentVertices(Id sourceV, Set vertices, return ImmutableSet.of(); } - Set neighbors = newSet(); + Set neighbors = newIdSet(); for (Id source : vertices) { Iterator edges = this.edgesOfVertex(source, dir, label, degree); @@ -137,7 +145,7 @@ protected Iterator adjacentVertices(Id source, Directions dir, } protected Set adjacentVertices(Id source, EdgeStep step) { - Set neighbors = new HashSet<>(); + Set neighbors = newSet(); Iterator edges = this.edgesOfVertex(source, step); while (edges.hasNext()) { neighbors.add(((HugeEdge) edges.next()).id().otherVertexId()); @@ -413,7 +421,7 @@ public static Iterator skipSuperNodeIfNeeded(Iterator edges, if (skipDegree <= 0L) { return edges; } - List edgeList = new ArrayList<>(); + List edgeList = newList(); for (int i = 1; edges.hasNext(); i++) { Edge edge = edges.next(); if (i <= degree) { @@ -426,6 +434,14 @@ public static Iterator skipSuperNodeIfNeeded(Iterator edges, return edgeList.iterator(); } + protected static Set newIdSet() { + return collectionFactory.newIdSet(); + } + + protected static List newIdList() { + return collectionFactory.newIdList(); + } + protected static Set newSet() { return newSet(false); } @@ -434,12 +450,36 @@ protected static Set newSet(boolean concurrent) { if (concurrent) { return ConcurrentHashMap.newKeySet(); } else { - return new HashSet<>(); + return collectionFactory.newSet(); } } + protected static Set newSet(int initialCapacity) { + return collectionFactory.newSet(initialCapacity); + } + + protected static Set newSet(Collection collection) { + return collectionFactory.newSet(collection); + } + + protected static List newList() { + return collectionFactory.newList(); + } + + protected static List newList(int initialCapacity) { + return collectionFactory.newList(initialCapacity); + } + + protected static List newList(Collection collection) { + return collectionFactory.newList(collection); + } + protected static Map newMap() { - return new HashMap<>(); + return collectionFactory.newMap(); + } + + protected static Map newMap(int initialCapacity) { + return collectionFactory.newMap(initialCapacity); } protected static MultivaluedMap newMultivalueMap() { @@ -490,7 +530,7 @@ public Node parent() { } public List path() { - List ids = new ArrayList<>(); + List ids = newIdList(); Node current = this; do { ids.add(current.id); @@ -621,13 +661,39 @@ public boolean equals(Object other) { } } - public static class PathSet extends HashSet { + public static class PathSet { private static final long serialVersionUID = -8237531948776524872L; + private Set paths = newSet(); + + public boolean add(Path path) { + return this.paths.add(path); + } + + public boolean addAll(Collection collection) { + return this.paths.addAll(collection); + } + + public boolean isEmpty() { + return this.paths.isEmpty(); + } + + public int size() { + return this.paths.size(); + } + + public Set paths() { + return this.paths; + } + + public Iterator iterator() { + return this.paths.iterator(); + } + public Set vertices() { - Set vertices = new HashSet<>(); - for (Path path : this) { + Set vertices = newIdSet(); + for (Path path : this.paths) { vertices.addAll(path.vertices()); } return vertices; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/JaccardSimilarTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/JaccardSimilarTraverser.java index af9f321cb2..cc0571c6b4 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/JaccardSimilarTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/JaccardSimilarTraverser.java @@ -19,8 +19,6 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -151,7 +149,7 @@ public Map jaccardSimilarsConcurrent(Id source, EdgeStep step, public Map jaccardSimilarsSingle(Id source, EdgeStep step, long capacity) { long count = 0L; - Set accessed = new HashSet<>(); + Set accessed = newIdSet(); accessed.add(source); reachCapacity(++count, capacity); @@ -163,9 +161,9 @@ public Map jaccardSimilarsSingle(Id source, EdgeStep step, return ImmutableMap.of(); } - Map results = new HashMap<>(); + Map results = newMap(); Set layer2s; - Set layer2All = new HashSet<>(); + Set layer2All = newIdSet(); double jaccardSimilarity; for (Id neighbor : layer1s) { // Skip if accessed already diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java index ec8e108c94..e5b1c97872 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java @@ -55,10 +55,10 @@ public Set kout(Id sourceV, Directions dir, String label, Id labelId = this.getEdgeLabelId(label); - Set latest = newSet(); + Set latest = newIdSet(); latest.add(sourceV); - Set all = newSet(); + Set all = newIdSet(); all.add(sourceV); long remaining = capacity == NO_LIMIT ? diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/MultiNodeShortestPathTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/MultiNodeShortestPathTraverser.java index de047284c3..2db85dec20 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/MultiNodeShortestPathTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/MultiNodeShortestPathTraverser.java @@ -19,7 +19,6 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; @@ -51,7 +50,7 @@ public List multiNodeShortestPath(Iterator vertices, "The number of vertices of multiple node shortest path " + "must in [2, %s], but got: %s", MAX_VERTICES, vertexList.size()); - List> pairs = new ArrayList<>(); + List> pairs = newList(); cmn(vertexList, vertexCount, 2, 0, null, r -> { Id source = ((HugeVertex) r.get(0)).id(); Id target = ((HugeVertex) r.get(1)).id(); @@ -91,7 +90,7 @@ public List multiNodeShortestPathConcurrent(List> pairs, public List multiNodeShortestPathSingle(List> pairs, EdgeStep step, int maxDepth, long capacity) { - List results = new ArrayList<>(); + List results = newList(); ShortestPathTraverser traverser = new ShortestPathTraverser(this.graph()); for (Pair pair : pairs) { @@ -109,7 +108,7 @@ private static void cmn(List all, int m, int n, int current, assert m <= all.size(); assert current <= all.size(); if (result == null) { - result = new ArrayList<>(n); + result = newList(n); } if (n == 0) { // All n items are selected diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java index 339343e844..e6f21f4493 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/NeighborRankTraverser.java @@ -19,9 +19,6 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -64,13 +61,13 @@ public List> neighborRank(Id source, List steps) { boolean sameLayerTransfer = true; long access = 0; // Results: ranks of each layer - List ranks = new ArrayList<>(); + List ranks = newList(); ranks.add(Ranks.of(source, 1.0)); for (Step step : steps) { Ranks lastLayerRanks = ranks.get(ranks.size() - 1); - Map sameLayerIncrRanks = new HashMap<>(); - List adjacencies = new ArrayList<>(); + Map sameLayerIncrRanks = newMap(); + List adjacencies = newList(); MultivaluedMap newVertices = newMultivalueMap(); // Traversal vertices of previous level for (Map.Entry> entry : sources.entrySet()) { @@ -79,8 +76,8 @@ public List> neighborRank(Id source, List steps) { step.edgeStep); Adjacencies adjacenciesV = new Adjacencies(vertex); - Set sameLayerNodesV = new HashSet<>(); - Map> prevLayerNodesV = new HashMap<>(); + Set sameLayerNodesV = newIdSet(); + Map> prevLayerNodesV = newMap(); while (edges.hasNext()) { HugeEdge edge = (HugeEdge) edges.next(); Id target = edge.id().otherVertexId(); @@ -166,7 +163,8 @@ private boolean belongToPrevLayers(List ranks, Id target, for (int i = ranks.size() - 2; i > 0; i--) { Ranks prevLayerRanks = ranks.get(i); if (prevLayerRanks.containsKey(target)) { - Set nodes = prevLayerNodes.computeIfAbsent(i, HashSet::new); + Set nodes = prevLayerNodes.computeIfAbsent( + i, HugeTraverser::newSet); nodes.add(target); return true; } @@ -220,7 +218,7 @@ private Ranks contributeNewLayer(List adjacencies, private List> topRanks(List ranks, List steps) { assert ranks.size() > 0; - List> results = new ArrayList<>(ranks.size()); + List> results = newList(ranks.size()); // The first layer is root node so skip i=0 results.add(ranks.get(0)); for (int i = 1; i < ranks.size(); i++) { @@ -264,7 +262,7 @@ private static class Adjacencies { public Adjacencies(Id source) { this.source = source; - this.nodes = new ArrayList<>(); + this.nodes = newList(); this.degree = -1L; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java index 14bf14d445..0253fbdbf9 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java @@ -77,7 +77,7 @@ public PathSet paths(Id sourceV, Directions sourceDir, } traverser.backward(targetDir); } - paths.addAll(traverser.paths()); + paths.addAll(traverser.paths().paths()); return paths; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java index 5c87d61790..d656de559a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PersonalRankTraverser.java @@ -19,8 +19,6 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -57,21 +55,21 @@ public Map personalRank(Id source, String label, this.checkVertexExist(source, "source vertex"); E.checkArgumentNotNull(label, "The edge label can't be null"); - Map ranks = new HashMap<>(); + Map ranks = newMap(); ranks.put(source, 1.0); Id labelId = this.graph().edgeLabel(label).id(); Directions dir = this.getStartDirection(source, label); - Set outSeeds = new HashSet<>(); - Set inSeeds = new HashSet<>(); + Set outSeeds = newIdSet(); + Set inSeeds = newIdSet(); if (dir == Directions.OUT) { outSeeds.add(source); } else { inSeeds.add(source); } - Set rootAdjacencies = new HashSet<>(); + Set rootAdjacencies = newIdSet(); for (long i = 0; i < this.maxDepth; i++) { Map newRanks = this.calcNewRanks(outSeeds, inSeeds, labelId, ranks); @@ -93,10 +91,10 @@ public Map personalRank(Id source, String label, private Map calcNewRanks(Set outSeeds, Set inSeeds, Id label, Map ranks) { - Map newRanks = new HashMap<>(); + Map newRanks = newMap(); BiFunction, Directions, Set> neighborIncrRanks; neighborIncrRanks = (seeds, dir) -> { - Set tmpSeeds = new HashSet<>(); + Set tmpSeeds = newIdSet(); for (Id seed : seeds) { Double oldRank = ranks.get(seed); E.checkState(oldRank != null, "Expect rank of seed exists"); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java index 17b757c6bf..1f9f09023d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java @@ -19,9 +19,7 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -60,7 +58,7 @@ public Path shortestPath(Id sourceV, Id targetV, Directions dir, return new Path(ImmutableList.of(sourceV)); } - Map labelMap = new HashMap<>(labels.size()); + Map labelMap = newMap(labels.size()); for (String label : labels) { labelMap.put(this.getEdgeLabelId(label), label); } @@ -91,7 +89,7 @@ public Path shortestPath(Id sourceV, Id targetV, Directions dir, public Path shortestPath(Id sourceV, Id targetV, EdgeStep step, int depth, long capacity) { return this.shortestPath(sourceV, targetV, step.direction(), - new ArrayList<>(step.labels().values()), + newList(step.labels().values()), depth, step.degree(), step.skipDegree(), capacity); } @@ -115,7 +113,7 @@ public PathSet allShortestPaths(Id sourceV, Id targetV, Directions dir, return paths; } - Map labelMap = new HashMap<>(labels.size()); + Map labelMap = newMap(labels.size()); for (String label : labels) { labelMap.put(this.getEdgeLabelId(label), label); } @@ -131,7 +129,7 @@ public PathSet allShortestPaths(Id sourceV, Id targetV, Directions dir, if (!(paths = traverser.backward(true)).isEmpty() || --depth <= 0) { - for (Path path : paths) { + for (Path path : paths.paths()) { Collections.reverse(path.vertices()); } break; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/SingleSourceShortestPathTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/SingleSourceShortestPathTraverser.java index 217c687ff7..ce1f4afe24 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/SingleSourceShortestPathTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/SingleSourceShortestPathTraverser.java @@ -19,9 +19,6 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -223,7 +220,7 @@ private Iterator skipSuperNodeIfNeeded(Iterator edges) { if (this.skipDegree <= 0L) { return edges; } - List edgeList = new ArrayList<>(); + List edgeList = newList(); int count = 0; while (edges.hasNext()) { if (count < this.degree) { @@ -276,7 +273,7 @@ public static class WeightedPaths extends LinkedHashMap { private static final long serialVersionUID = -313873642177730993L; public Set vertices() { - Set vertices = new HashSet<>(); + Set vertices = newIdSet(); vertices.addAll(this.keySet()); for (NodeWithWeight nw : this.values()) { vertices.addAll(nw.node().path()); @@ -285,7 +282,7 @@ public Set vertices() { } public Map> toMap() { - Map> results = new HashMap<>(); + Map> results = newMap(); for (Map.Entry entry : this.entrySet()) { Id source = entry.getKey(); NodeWithWeight nw = entry.getValue(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/SubGraphTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/SubGraphTraverser.java index 7e7f02f75b..1e613a1c29 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/SubGraphTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/SubGraphTraverser.java @@ -19,7 +19,6 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -73,7 +72,7 @@ private PathSet subGraphPaths(Id sourceV, Directions dir, String label, sourceInRing); PathSet paths = new PathSet(); while (true) { - paths.addAll(traverser.forward(dir)); + paths.addAll(traverser.forward(dir).paths()); if (--depth <= 0 || traverser.reachLimit() || traverser.finished()) { break; @@ -104,7 +103,7 @@ private class Traverser { private final Id source; private MultivaluedMap sources = newMultivalueMap(); - private Set accessedVertices = newSet(); + private Set accessedVertices = newIdSet(); private final Id label; private int depth; @@ -162,7 +161,7 @@ public PathSet forward(Directions direction) { } int neighborCount = 0; - Set currentNeighbors = new HashSet<>(); + Set currentNeighbors = newIdSet(); while (edges.hasNext()) { neighborCount++; HugeEdge edge = (HugeEdge) edges.next(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/TemplatePathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/TemplatePathsTraverser.java index 3e4bdc5b00..3be665f2a1 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/TemplatePathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/TemplatePathsTraverser.java @@ -19,7 +19,6 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -50,7 +49,7 @@ public Set templatePaths(Iterator sources, checkCapacity(capacity); checkLimit(limit); - List sourceList = new ArrayList<>(); + List sourceList = newIdList(); while (sources.hasNext()) { sourceList.add(((HugeVertex) sources.next()).id()); } @@ -58,7 +57,7 @@ public Set templatePaths(Iterator sources, E.checkState(sourceSize >= 1 && sourceSize <= MAX_VERTICES, "The number of source vertices must in [1, %s], " + "but got: %s", MAX_VERTICES, sourceList.size()); - List targetList = new ArrayList<>(); + List targetList = newIdList(); while (targets.hasNext()) { targetList.add(((HugeVertex) targets.next()).id()); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/strategy/SingleTraverseStrategy.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/strategy/SingleTraverseStrategy.java index 817d188feb..be02f1c336 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/strategy/SingleTraverseStrategy.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/strategy/SingleTraverseStrategy.java @@ -55,7 +55,7 @@ public Map> newMultiValueMap() { @Override public Set newPathSet() { - return new HugeTraverser.PathSet(); + return new HugeTraverser.PathSet().paths(); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java new file mode 100644 index 0000000000..89ed435e71 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java @@ -0,0 +1,60 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.type.define; + +public enum CollectionImplType implements SerialEnum { + + JCF(1, "jcf"), + + EC(2, "ec"); + + private final byte code; + private final String name; + + static { + SerialEnum.register(CollectionImplType.class); + } + + CollectionImplType(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } + + public static CollectionImplType fromCode(byte code) { + switch (code) { + case 1: + return JCF; + case 2: + return EC; + default: + throw new AssertionError( + "Unsupported collection code: " + code); + } + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java new file mode 100644 index 0000000000..0fc93e45b7 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java @@ -0,0 +1,197 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.util.collection; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.eclipse.collections.impl.list.mutable.FastList; +import org.eclipse.collections.impl.map.mutable.UnifiedMap; +import org.eclipse.collections.impl.set.mutable.UnifiedSet; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.type.define.CollectionImplType; + +public class CollectionFactory { + + private CollectionImplType type; + + public CollectionFactory() { + this.type = CollectionImplType.EC; + } + + public CollectionFactory(CollectionImplType type) { + this.type = type; + } + + public List newList() { + return newList(this.type); + } + + public List newList(int initialCapacity) { + return newList(this.type, initialCapacity); + } + + public List newList(Collection collection) { + return newList(this.type, collection); + } + + public static List newList(CollectionImplType type) { + switch (type) { + case EC: + return new FastList<>(); + case JCF: + return new ArrayList<>(); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public static List newList(CollectionImplType type, + int initialCapacity) { + switch (type) { + case EC: + return new FastList<>(initialCapacity); + case JCF: + return new ArrayList<>(initialCapacity); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public static List newList(CollectionImplType type, + Collection collection) { + switch (type) { + case EC: + return new FastList<>(collection); + case JCF: + return new ArrayList<>(collection); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public Set newSet() { + return newSet(this.type); + } + + public Set newSet(int initialCapacity) { + return newSet(this.type, initialCapacity); + } + + public Set newSet(Collection collection) { + return newSet(this.type, collection); + } + + public static Set newSet(CollectionImplType type) { + switch (type) { + case EC: + return new UnifiedSet<>(); + case JCF: + return new HashSet<>(); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public static Set newSet(CollectionImplType type, + int initialCapacity) { + switch (type) { + case EC: + return new UnifiedSet<>(initialCapacity); + case JCF: + return new HashSet<>(initialCapacity); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public static Set newSet(CollectionImplType type, + Collection collection) { + switch (type) { + case EC: + return new UnifiedSet<>(collection); + case JCF: + return new HashSet<>(collection); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public Map newMap() { + return newMap(this.type); + } + + public Map newMap(int initialCapacity) { + return newMap(this.type, initialCapacity); + } + + public static Map newMap(CollectionImplType type) { + switch (type) { + case EC: + return new UnifiedMap<>(); + case JCF: + return new HashMap<>(); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public static Map newMap(CollectionImplType type, + int initialCapacity) { + switch (type) { + case EC: + return new UnifiedMap<>(initialCapacity); + case JCF: + return new HashMap<>(initialCapacity); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public Set newIdSet() { + return newIdSet(this.type); + } + + public static Set newIdSet(CollectionImplType type) { + return new IdSet(type); + } + + public List newIdList() { + return newIdList(this.type); + } + + public static List newIdList(CollectionImplType type) { + return new IdList(type); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdList.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdList.java new file mode 100644 index 0000000000..cb17055e09 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdList.java @@ -0,0 +1,133 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.util.collection; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.eclipse.collections.impl.list.mutable.FastList; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.id.IdGenerator; +import com.baidu.hugegraph.iterator.ExtendableIterator; +import com.baidu.hugegraph.iterator.MapperIterator; +import com.baidu.hugegraph.type.define.CollectionImplType; +import com.carrotsearch.hppc.LongArrayList; +import com.carrotsearch.hppc.cursors.LongCursor; + +public class IdList extends ArrayList { + + private LongArrayList numberIds; + private List nonNumberIds; + + public IdList(CollectionImplType type) { + this.numberIds = new LongArrayList(); + switch (type) { + case JCF: + this.nonNumberIds = new ArrayList<>(); + break; + case EC: + this.nonNumberIds = new FastList<>(); + break; + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + @Override + public int size() { + return this.numberIds.size() + this.nonNumberIds.size(); + } + + @Override + public boolean isEmpty() { + return this.numberIds.isEmpty() && this.nonNumberIds.isEmpty(); + } + + @Override + public boolean contains(Object o) { + if (o instanceof IdGenerator.LongId) { + return this.numberIds.contains(((IdGenerator.LongId) o).longValue()); + } else { + return this.nonNumberIds.contains(o); + } + } + + @Override + public Iterator iterator() { + ExtendableIterator iterator = new ExtendableIterator<>(); + iterator.extend(this.nonNumberIds.iterator()); + EcLongIterator iter = new EcLongIterator(this.numberIds.iterator()); + iterator.extend(new MapperIterator<>(iter, IdGenerator::of)); + return iterator; + } + + public boolean add(Id id) { + if (id instanceof IdGenerator.LongId) { + this.numberIds.add(((IdGenerator.LongId) id).longValue()); + return true; + } else { + return this.nonNumberIds.add(id); + } + } + + @Override + public boolean remove(Object o) { + super.remove(o); + if (o instanceof IdGenerator.LongId) { + long id = ((IdGenerator.LongId) o).longValue(); + return this.numberIds.removeFirst(id) >= 0; + } else { + return this.nonNumberIds.remove(o); + } + } + + @Override + public void clear() { + this.numberIds.clear(); + this.nonNumberIds.clear(); + } + + private static class EcLongIterator implements Iterator { + + private Iterator iterator; + + public EcLongIterator(Iterator iter) { + this.iterator = iter; + } + + @Override + public boolean hasNext() { + return this.iterator.hasNext(); + } + + @Override + public Long next() { + return this.iterator.next().value; + } + + @Override + public void remove() { + this.iterator.remove(); + } + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java new file mode 100644 index 0000000000..d42a9d2c33 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java @@ -0,0 +1,131 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.util.collection; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import org.eclipse.collections.api.iterator.MutableLongIterator; +import org.eclipse.collections.impl.set.mutable.UnifiedSet; +import org.eclipse.collections.impl.set.mutable.primitive.LongHashSet; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.id.IdGenerator; +import com.baidu.hugegraph.iterator.ExtendableIterator; +import com.baidu.hugegraph.iterator.MapperIterator; +import com.baidu.hugegraph.type.define.CollectionImplType; + +public class IdSet extends HashSet { + + private LongHashSet numberIds; + private Set nonNumberIds; + + public IdSet(CollectionImplType type) { + this.numberIds = new LongHashSet(); + switch (type) { + case JCF: + this.nonNumberIds = new HashSet<>(); + break; + case EC: + this.nonNumberIds = new UnifiedSet<>(); + break; + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + @Override + public int size() { + return this.numberIds.size() + this.nonNumberIds.size(); + } + + @Override + public boolean isEmpty() { + return this.numberIds.isEmpty() && this.nonNumberIds.isEmpty(); + } + + @Override + public boolean contains(Object o) { + if (o instanceof IdGenerator.LongId) { + return this.numberIds.contains(((IdGenerator.LongId) o).longValue()); + } else { + return this.nonNumberIds.contains(o); + } + } + + @Override + public Iterator iterator() { + ExtendableIterator iterator = new ExtendableIterator<>(); + iterator.extend(this.nonNumberIds.iterator()); + EcLongIterator iter = new EcLongIterator(this.numberIds.longIterator()); + iterator.extend(new MapperIterator<>(iter, IdGenerator::of)); + return iterator; + } + + @Override + public boolean add(Id id) { + if (id instanceof IdGenerator.LongId) { + return this.numberIds.add(((IdGenerator.LongId) id).longValue()); + } else { + return this.nonNumberIds.add(id); + } + } + + @Override + public boolean remove(Object o) { + if (o instanceof IdGenerator.LongId) { + return this.numberIds.remove(((IdGenerator.LongId) o).longValue()); + } else { + return this.nonNumberIds.remove(o); + } + } + + @Override + public void clear() { + this.numberIds.clear(); + this.nonNumberIds.clear(); + } + + private static class EcLongIterator implements Iterator { + + private MutableLongIterator iterator; + + public EcLongIterator(MutableLongIterator iter) { + this.iterator = iter; + } + + @Override + public boolean hasNext() { + return this.iterator.hasNext(); + } + + @Override + public Long next() { + return this.iterator.next(); + } + + @Override + public void remove() { + this.iterator.remove(); + } + } +} From ce8599d6bcdefa78d9c40acc4e8e508a21f670b0 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Wed, 24 Mar 2021 22:01:31 +0800 Subject: [PATCH 04/38] remove IdList implementation which is error Change-Id: I4d02e9f2fcb3e5197bbebf488fcc87d7524ce24d --- .../algorithm/CollectionPathsTraverser.java | 4 +- .../traversal/algorithm/HugeTraverser.java | 6 +- .../algorithm/TemplatePathsTraverser.java | 4 +- .../util/collection/CollectionFactory.java | 8 -- .../hugegraph/util/collection/IdList.java | 133 ------------------ 5 files changed, 5 insertions(+), 150 deletions(-) delete mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdList.java diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CollectionPathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CollectionPathsTraverser.java index 03e2b7f74e..2562697398 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CollectionPathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/CollectionPathsTraverser.java @@ -47,7 +47,7 @@ public Collection paths(Iterator sources, checkCapacity(capacity); checkLimit(limit); - List sourceList = newIdList(); + List sourceList = newList(); while (sources.hasNext()) { sourceList.add(((HugeVertex) sources.next()).id()); } @@ -55,7 +55,7 @@ public Collection paths(Iterator sources, E.checkState(sourceSize >= 1 && sourceSize <= MAX_VERTICES, "The number of source vertices must in [1, %s], " + "but got: %s", MAX_VERTICES, sourceList.size()); - List targetList = newIdList(); + List targetList = newList(); while (targets.hasNext()) { targetList.add(((HugeVertex) targets.next()).id()); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java index ef88cce4d6..54a368d03c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java @@ -438,10 +438,6 @@ protected static Set newIdSet() { return collectionFactory.newIdSet(); } - protected static List newIdList() { - return collectionFactory.newIdList(); - } - protected static Set newSet() { return newSet(false); } @@ -530,7 +526,7 @@ public Node parent() { } public List path() { - List ids = newIdList(); + List ids = newList(); Node current = this; do { ids.add(current.id); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/TemplatePathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/TemplatePathsTraverser.java index 3be665f2a1..2a3971cedd 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/TemplatePathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/TemplatePathsTraverser.java @@ -49,7 +49,7 @@ public Set templatePaths(Iterator sources, checkCapacity(capacity); checkLimit(limit); - List sourceList = newIdList(); + List sourceList = newList(); while (sources.hasNext()) { sourceList.add(((HugeVertex) sources.next()).id()); } @@ -57,7 +57,7 @@ public Set templatePaths(Iterator sources, E.checkState(sourceSize >= 1 && sourceSize <= MAX_VERTICES, "The number of source vertices must in [1, %s], " + "but got: %s", MAX_VERTICES, sourceList.size()); - List targetList = newIdList(); + List targetList = newList(); while (targets.hasNext()) { targetList.add(((HugeVertex) targets.next()).id()); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java index 0fc93e45b7..dc0039a6a3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java @@ -186,12 +186,4 @@ public Set newIdSet() { public static Set newIdSet(CollectionImplType type) { return new IdSet(type); } - - public List newIdList() { - return newIdList(this.type); - } - - public static List newIdList(CollectionImplType type) { - return new IdList(type); - } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdList.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdList.java deleted file mode 100644 index cb17055e09..0000000000 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdList.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2017 HugeGraph Authors - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.baidu.hugegraph.util.collection; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.eclipse.collections.impl.list.mutable.FastList; - -import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.backend.id.IdGenerator; -import com.baidu.hugegraph.iterator.ExtendableIterator; -import com.baidu.hugegraph.iterator.MapperIterator; -import com.baidu.hugegraph.type.define.CollectionImplType; -import com.carrotsearch.hppc.LongArrayList; -import com.carrotsearch.hppc.cursors.LongCursor; - -public class IdList extends ArrayList { - - private LongArrayList numberIds; - private List nonNumberIds; - - public IdList(CollectionImplType type) { - this.numberIds = new LongArrayList(); - switch (type) { - case JCF: - this.nonNumberIds = new ArrayList<>(); - break; - case EC: - this.nonNumberIds = new FastList<>(); - break; - default: - throw new AssertionError( - "Unsupported collection type: " + type); - } - } - - @Override - public int size() { - return this.numberIds.size() + this.nonNumberIds.size(); - } - - @Override - public boolean isEmpty() { - return this.numberIds.isEmpty() && this.nonNumberIds.isEmpty(); - } - - @Override - public boolean contains(Object o) { - if (o instanceof IdGenerator.LongId) { - return this.numberIds.contains(((IdGenerator.LongId) o).longValue()); - } else { - return this.nonNumberIds.contains(o); - } - } - - @Override - public Iterator iterator() { - ExtendableIterator iterator = new ExtendableIterator<>(); - iterator.extend(this.nonNumberIds.iterator()); - EcLongIterator iter = new EcLongIterator(this.numberIds.iterator()); - iterator.extend(new MapperIterator<>(iter, IdGenerator::of)); - return iterator; - } - - public boolean add(Id id) { - if (id instanceof IdGenerator.LongId) { - this.numberIds.add(((IdGenerator.LongId) id).longValue()); - return true; - } else { - return this.nonNumberIds.add(id); - } - } - - @Override - public boolean remove(Object o) { - super.remove(o); - if (o instanceof IdGenerator.LongId) { - long id = ((IdGenerator.LongId) o).longValue(); - return this.numberIds.removeFirst(id) >= 0; - } else { - return this.nonNumberIds.remove(o); - } - } - - @Override - public void clear() { - this.numberIds.clear(); - this.nonNumberIds.clear(); - } - - private static class EcLongIterator implements Iterator { - - private Iterator iterator; - - public EcLongIterator(Iterator iter) { - this.iterator = iter; - } - - @Override - public boolean hasNext() { - return this.iterator.hasNext(); - } - - @Override - public Long next() { - return this.iterator.next().value; - } - - @Override - public void remove() { - this.iterator.remove(); - } - } -} From 0247f099c6334d00fae783a0aef876d07d57153e Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Fri, 26 Mar 2021 11:08:01 +0800 Subject: [PATCH 05/38] add fastutil choice for collection Change-Id: I1283c6d39807903d68d6a450be37610243c8e9e9 --- hugegraph-core/pom.xml | 7 +++++++ .../baidu/hugegraph/config/CoreOptions.java | 2 +- .../type/define/CollectionImplType.java | 7 ++++++- .../util/collection/CollectionFactory.java | 20 +++++++++++++++++++ .../hugegraph/util/collection/IdSet.java | 5 +++++ 5 files changed, 39 insertions(+), 2 deletions(-) diff --git a/hugegraph-core/pom.xml b/hugegraph-core/pom.xml index 9853350b53..0b70dbea3c 100644 --- a/hugegraph-core/pom.xml +++ b/hugegraph-core/pom.xml @@ -167,6 +167,13 @@ eclipse-collections 10.4.0 + + + + it.unimi.dsi + fastutil + 8.1.0 + diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java index 6ca6976b15..60eede09a8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java @@ -629,7 +629,7 @@ public static synchronized CoreOptions instance() { "oltp.collection_impl_type", "The implementation type of collections " + "used in oltp algorithm.", - allowValues("jcf", "ec"), + allowValues("jcf", "ec", "fu"), "ec" ); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java index 89ed435e71..eb23855cd8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java @@ -21,9 +21,14 @@ public enum CollectionImplType implements SerialEnum { + // Java Collection Framework JCF(1, "jcf"), - EC(2, "ec"); + // Eclipse Collection + EC(2, "ec"), + + // FastUtil + FU(3, "fu"); private final byte code; private final String name; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java index dc0039a6a3..40380c1f92 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java @@ -34,6 +34,10 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.type.define.CollectionImplType; +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.objects.ObjectArrayList; +import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; + public class CollectionFactory { private CollectionImplType type; @@ -64,6 +68,8 @@ public static List newList(CollectionImplType type) { return new FastList<>(); case JCF: return new ArrayList<>(); + case FU: + return new ObjectArrayList<>(); default: throw new AssertionError( "Unsupported collection type: " + type); @@ -77,6 +83,8 @@ public static List newList(CollectionImplType type, return new FastList<>(initialCapacity); case JCF: return new ArrayList<>(initialCapacity); + case FU: + return new ObjectArrayList<>(initialCapacity); default: throw new AssertionError( "Unsupported collection type: " + type); @@ -90,6 +98,8 @@ public static List newList(CollectionImplType type, return new FastList<>(collection); case JCF: return new ArrayList<>(collection); + case FU: + return new ObjectArrayList<>(collection); default: throw new AssertionError( "Unsupported collection type: " + type); @@ -114,6 +124,8 @@ public static Set newSet(CollectionImplType type) { return new UnifiedSet<>(); case JCF: return new HashSet<>(); + case FU: + return new ObjectOpenHashSet<>(); default: throw new AssertionError( "Unsupported collection type: " + type); @@ -127,6 +139,8 @@ public static Set newSet(CollectionImplType type, return new UnifiedSet<>(initialCapacity); case JCF: return new HashSet<>(initialCapacity); + case FU: + return new ObjectOpenHashSet<>(initialCapacity); default: throw new AssertionError( "Unsupported collection type: " + type); @@ -140,6 +154,8 @@ public static Set newSet(CollectionImplType type, return new UnifiedSet<>(collection); case JCF: return new HashSet<>(collection); + case FU: + return new ObjectOpenHashSet<>(collection); default: throw new AssertionError( "Unsupported collection type: " + type); @@ -160,6 +176,8 @@ public static Map newMap(CollectionImplType type) { return new UnifiedMap<>(); case JCF: return new HashMap<>(); + case FU: + return new Object2ObjectOpenHashMap<>(); default: throw new AssertionError( "Unsupported collection type: " + type); @@ -173,6 +191,8 @@ public static Map newMap(CollectionImplType type, return new UnifiedMap<>(initialCapacity); case JCF: return new HashMap<>(initialCapacity); + case FU: + return new Object2ObjectOpenHashMap<>(initialCapacity); default: throw new AssertionError( "Unsupported collection type: " + type); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java index d42a9d2c33..472ecda8c0 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java @@ -33,6 +33,8 @@ import com.baidu.hugegraph.iterator.MapperIterator; import com.baidu.hugegraph.type.define.CollectionImplType; +import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; + public class IdSet extends HashSet { private LongHashSet numberIds; @@ -47,6 +49,9 @@ public IdSet(CollectionImplType type) { case EC: this.nonNumberIds = new UnifiedSet<>(); break; + case FU: + this.nonNumberIds = new ObjectOpenHashSet<>(); + break; default: throw new AssertionError( "Unsupported collection type: " + type); From c266850cc2ae10ab77a703eb743faf6ab150bf84 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Mon, 29 Mar 2021 21:06:11 +0800 Subject: [PATCH 06/38] change Set of HugeVertex to List Change-Id: I2f760d390fa52c667baddbfd5fef2e6cf440805b --- .../backend/serializer/BinarySerializer.java | 6 +-- .../backend/serializer/TableSerializer.java | 2 +- .../backend/serializer/TextSerializer.java | 2 +- .../hugegraph/backend/store/ram/RamTable.java | 2 +- .../baidu/hugegraph/structure/HugeEdge.java | 19 ++++++-- .../baidu/hugegraph/structure/HugeVertex.java | 43 ++++++++++++------- 6 files changed, 49 insertions(+), 25 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java index e585135252..0aaa6dde46 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java @@ -22,7 +22,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.commons.lang.NotImplementedException; @@ -267,7 +266,8 @@ protected void parseEdge(BackendColumn col, HugeVertex vertex, // Construct edge HugeEdge edge = HugeEdge.constructEdge(vertex, direction, edgeLabel, - sortValues, otherVertexId); + sortValues, otherVertexId, + false); // Parse edge-id + edge-properties buffer = BytesBuffer.wrap(col.value); @@ -459,7 +459,7 @@ public BackendEntry writeEdgeProperty(HugeEdgeProperty prop) { @Override public HugeEdge readEdge(HugeGraph graph, BackendEntry bytesEntry) { HugeVertex vertex = this.readVertex(graph, bytesEntry); - Set edges = vertex.getEdges(); + Collection edges = vertex.getEdges(); E.checkState(edges.size() == 1, "Expect one edge in vertex, but got %s", edges.size()); return edges.iterator().next(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java index f1659f0c84..94b57fcfad 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java @@ -179,7 +179,7 @@ protected HugeEdge parseEdge(TableBackendEntry.Row row, // Construct edge HugeEdge edge = HugeEdge.constructEdge(vertex, direction, edgeLabel, - sortValues, otherId); + sortValues, otherId, false); // Parse edge properties this.parseProperties(edge, row); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TextSerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TextSerializer.java index 18d896046e..9af6f8b1fc 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TextSerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TextSerializer.java @@ -225,7 +225,7 @@ private void parseEdge(String colName, String colValue, Id otherVertexId = readEntryId(colParts[3]); // Construct edge HugeEdge edge = HugeEdge.constructEdge(vertex, direction, edgeLabel, - sortValues, otherVertexId); + sortValues, otherVertexId, false); String[] valParts = colValue.split(VALUE_SPLITOR); // Parse edge expired time diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/RamTable.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/RamTable.java index 01614a007d..a87815b5d3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/RamTable.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/RamTable.java @@ -451,7 +451,7 @@ private HugeEdge fetch() { HugeEdge edge = HugeEdge.constructEdge(this.owner, direction, edgeLabel, sortValues, - otherVertexId); + otherVertexId, false); edge.propNotLoaded(); return edge; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java index 3503916e17..91987f8a4a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java @@ -493,6 +493,17 @@ public static HugeEdge constructEdge(HugeVertex ownerVertex, EdgeLabel edgeLabel, String sortValues, Id otherVertexId) { + return constructEdge(ownerVertex, isOutEdge, edgeLabel, + sortValues, otherVertexId, true); + + } + + public static HugeEdge constructEdge(HugeVertex ownerVertex, + boolean isOutEdge, + EdgeLabel edgeLabel, + String sortValues, + Id otherVertexId, + boolean needDedup) { HugeGraph graph = ownerVertex.graph(); VertexLabel srcLabel = graph.vertexLabelOrNone(edgeLabel.sourceLabel()); VertexLabel tgtLabel = graph.vertexLabelOrNone(edgeLabel.targetLabel()); @@ -517,11 +528,11 @@ public static HugeEdge constructEdge(HugeVertex ownerVertex, edge.assignId(); if (isOutEdge) { - ownerVertex.addOutEdge(edge); - otherVertex.addInEdge(edge.switchOwner()); + ownerVertex.addOutEdge(edge, needDedup); + otherVertex.addInEdge(edge.switchOwner(), needDedup); } else { - ownerVertex.addInEdge(edge); - otherVertex.addOutEdge(edge.switchOwner()); + ownerVertex.addInEdge(edge, needDedup); + otherVertex.addOutEdge(edge.switchOwner(), needDedup); } return edge; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java index f53530c9b7..82311de3f9 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java @@ -25,7 +25,6 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.Set; import org.apache.commons.collections.CollectionUtils; import org.apache.tinkerpop.gremlin.structure.Direction; @@ -35,6 +34,7 @@ import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.apache.tinkerpop.gremlin.structure.util.ElementHelper; import org.apache.tinkerpop.gremlin.structure.util.StringFactory; +import org.eclipse.collections.impl.list.mutable.FastList; import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.HugeGraph; @@ -57,18 +57,17 @@ import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.type.define.IdStrategy; import com.baidu.hugegraph.util.E; -import com.baidu.hugegraph.util.InsertionOrderUtil; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableList; public class HugeVertex extends HugeElement implements Vertex, Cloneable { - private static final Set EMPTY_SET = ImmutableSet.of(); + private static final List EMPTY_LIST = ImmutableList.of(); private Id id; private VertexLabel label; // Implemented as LinkedHashMap, is it necessary? Eclipse Collection does // not have replacement for LinkedHashMap now. - private Set edges; + private List edges; public HugeVertex(final HugeGraph graph, Id id, VertexLabel label) { super(graph); @@ -77,7 +76,7 @@ public HugeVertex(final HugeGraph graph, Id id, VertexLabel label) { this.label = label; this.id = id; - this.edges = EMPTY_SET; + this.edges = EMPTY_LIST; if (this.id != null) { if (label.idStrategy() == IdStrategy.CUSTOMIZE_UUID) { this.assignId(id); @@ -219,12 +218,12 @@ public boolean existsEdges() { return this.edges.size() > 0; } - public Set getEdges() { - return Collections.unmodifiableSet(this.edges); + public Collection getEdges() { + return Collections.unmodifiableList(this.edges); } public void resetEdges() { - this.edges = InsertionOrderUtil.newSet(); + this.edges = new FastList<>(); } public void removeEdge(HugeEdge edge) { @@ -232,8 +231,15 @@ public void removeEdge(HugeEdge edge) { } public void addEdge(HugeEdge edge) { - if (this.edges == EMPTY_SET) { - this.edges = InsertionOrderUtil.newSet(); + this.addEdge(edge, true); + } + + public void addEdge(HugeEdge edge, boolean needDedup) { + if (this.edges == EMPTY_LIST) { + this.edges = new FastList<>(); + } + if (needDedup && this.edges.contains(edge)) { + this.edges.remove(edge); } this.edges.add(edge); } @@ -321,32 +327,39 @@ public HugeEdge constructEdge(String label, HugeVertex vertex, return edge; } + public void addOutEdge(HugeEdge edge) { + this.addOutEdge(edge, true); + } + /** * Add edge with direction OUT * @param edge the out edge */ - public void addOutEdge(HugeEdge edge) { + public void addOutEdge(HugeEdge edge, boolean needDedup) { if (edge.ownerVertex() == null) { edge.sourceVertex(this); } E.checkState(edge.isDirection(Directions.OUT), "The owner vertex('%s') of OUT edge '%s' should be '%s'", edge.ownerVertex().id(), edge, this.id()); - this.addEdge(edge); + this.addEdge(edge, needDedup); } + public void addInEdge(HugeEdge edge) { + this.addInEdge(edge, true); + } /** * Add edge with direction IN * @param edge the in edge */ - public void addInEdge(HugeEdge edge) { + public void addInEdge(HugeEdge edge, boolean needDedup) { if (edge.ownerVertex() == null) { edge.targetVertex(this); } E.checkState(edge.isDirection(Directions.IN), "The owner vertex('%s') of IN edge '%s' should be '%s'", edge.ownerVertex().id(), edge, this.id()); - this.addEdge(edge); + this.addEdge(edge, needDedup); } public Iterator getEdges(Directions direction, String... edgeLabels) { From 0ebfd167c8df16c3b9aa5d5c24367b956bbd2589 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Mon, 29 Mar 2021 21:06:51 +0800 Subject: [PATCH 07/38] use accessed set to reduce use of Node.contains() for shortest path traverser Change-Id: I320032f0ccc34f15645bfd19063eeebab8457f5c --- .../algorithm/ShortestPathTraverser.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java index 1f9f09023d..a46828c035 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java @@ -23,6 +23,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; @@ -144,6 +145,7 @@ private class Traverser { // TODO: change Map to Set to reduce memory cost private Map sources = newMap(); private Map targets = newMap(); + private Set accessed = newIdSet(); private final Directions direction; private final Map labels; @@ -152,6 +154,8 @@ private class Traverser { private final long capacity; private long size; + private boolean foundPath = false; + public Traverser(Id sourceV, Id targetV, Directions dir, Map labels, long degree, long skipDegree, long capacity) { @@ -170,6 +174,7 @@ public Traverser(Id sourceV, Id targetV, Directions dir, */ public PathSet forward(boolean all) { PathSet paths = new PathSet(); + this.accessed.addAll(this.sources.keySet()); Map newVertices = newMap(); long degree = this.skipDegree > 0L ? this.skipDegree : this.degree; // Traversal vertices of previous level @@ -189,6 +194,7 @@ public PathSet forward(boolean all) { } paths.add(new Path( v.joinPath(this.targets.get(target)))); + this.foundPath = true; if (!all) { return paths; } @@ -200,9 +206,8 @@ public PathSet forward(boolean all) { * 1. not in sources and newVertices yet * 2. path of node doesn't have loop */ - if (!newVertices.containsKey(target) && - !this.sources.containsKey(target) && - !v.contains(target)) { + if (!this.foundPath && !newVertices.containsKey(target) && + !this.accessed.contains(target)) { newVertices.put(target, new Node(target, v)); } } @@ -220,6 +225,7 @@ public PathSet forward(boolean all) { */ public PathSet backward(boolean all) { PathSet paths = new PathSet(); + this.accessed.addAll(this.targets.keySet()); Map newVertices = newMap(); long degree = this.skipDegree > 0L ? this.skipDegree : this.degree; Directions opposite = this.direction.opposite(); @@ -240,6 +246,7 @@ public PathSet backward(boolean all) { } paths.add(new Path( v.joinPath(this.sources.get(target)))); + this.foundPath = true; if (!all) { return paths; } @@ -251,9 +258,8 @@ public PathSet backward(boolean all) { * 1. not in targets and newVertices yet * 2. path of node doesn't have loop */ - if (!newVertices.containsKey(target) && - !this.targets.containsKey(target) && - !v.contains(target)) { + if (!this.foundPath && !newVertices.containsKey(target) && + !this.accessed.contains(target)) { newVertices.put(target, new Node(target, v)); } } From 2830773d0009c7f117509be8a5f47e262e67fe85 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 30 Mar 2021 16:00:31 +0800 Subject: [PATCH 08/38] use LongObjectMap and ObjectLongMap to improve shortest path bad result, ObjectLongMap is terrible Change-Id: Ic018c48fd9b78eb1cddd62fb9fb842728bc079b8 --- .../traversal/algorithm/IdMapping.java | 52 +++++++ .../algorithm/ShortestPathTraverser.java | 142 +++++++++++++----- 2 files changed, 154 insertions(+), 40 deletions(-) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java new file mode 100644 index 0000000000..cec5eabf50 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java @@ -0,0 +1,52 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm; + +import org.eclipse.collections.impl.map.mutable.primitive.LongObjectHashMap; +import org.eclipse.collections.impl.map.mutable.primitive.ObjectLongHashMap; + +import com.baidu.hugegraph.backend.id.Id; + +public class IdMapping { + + private LongObjectHashMap long2IdMap; + private ObjectLongHashMap id2LongMap; + private long nextLong; + + public IdMapping() { + this.long2IdMap = new LongObjectHashMap(); + this.id2LongMap = new ObjectLongHashMap(); + this.nextLong = 0L; + } + + public long getLong(Id id) { + if (this.id2LongMap.containsKey(id)) { + return this.id2LongMap.get(id); + } + this.nextLong++; + this.id2LongMap.put(id, this.nextLong); + this.long2IdMap.put(this.nextLong, id); + return this.nextLong; + } + + public Id getId(long code) { + return (Id) this.long2IdMap.get(code); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java index a46828c035..4ae89f72d5 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java @@ -19,14 +19,18 @@ package com.baidu.hugegraph.traversal.algorithm; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.Stack; import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; +import org.eclipse.collections.api.iterator.LongIterator; +import org.eclipse.collections.impl.map.mutable.primitive.LongLongHashMap; +import org.eclipse.collections.impl.set.mutable.primitive.LongHashSet; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.id.Id; @@ -76,10 +80,6 @@ public Path shortestPath(Id sourceV, Id targetV, Directions dir, if (!(paths = traverser.backward(false)).isEmpty() || --depth <= 0) { - if (!paths.isEmpty()) { - Path path = paths.iterator().next(); - Collections.reverse(path.vertices()); - } break; } checkCapacity(traverser.capacity, traverser.size, "shortest path"); @@ -130,9 +130,6 @@ public PathSet allShortestPaths(Id sourceV, Id targetV, Directions dir, if (!(paths = traverser.backward(true)).isEmpty() || --depth <= 0) { - for (Path path : paths.paths()) { - Collections.reverse(path.vertices()); - } break; } checkCapacity(traverser.capacity, traverser.size, "shortest path"); @@ -143,9 +140,13 @@ public PathSet allShortestPaths(Id sourceV, Id targetV, Directions dir, private class Traverser { // TODO: change Map to Set to reduce memory cost - private Map sources = newMap(); - private Map targets = newMap(); - private Set accessed = newIdSet(); + private IdMapping idMapping; + + private Stack sourceLayers; + private Stack targetLayers; + + + private LongHashSet accessed = new LongHashSet(); private final Directions direction; private final Map labels; @@ -159,8 +160,21 @@ private class Traverser { public Traverser(Id sourceV, Id targetV, Directions dir, Map labels, long degree, long skipDegree, long capacity) { - this.sources.put(sourceV, new Node(sourceV)); - this.targets.put(targetV, new Node(targetV)); + this.idMapping = new IdMapping(); + long sourceCode = this.code(sourceV); + long targetCode = this.code(targetV); + LongLongHashMap firstSourceLayer = new LongLongHashMap(); + LongLongHashMap firstTargetLayer = new LongLongHashMap(); + firstSourceLayer.put(sourceCode, 0L); + firstTargetLayer.put(targetCode, 0L); + this.sourceLayers = new Stack<>(); + this.targetLayers = new Stack<>(); + this.sourceLayers.push(firstSourceLayer); + this.targetLayers.push(firstTargetLayer); + + this.accessed.add(sourceCode); + this.accessed.add(targetCode); + this.direction = dir; this.labels = labels; this.degree = degree; @@ -174,26 +188,34 @@ public Traverser(Id sourceV, Id targetV, Directions dir, */ public PathSet forward(boolean all) { PathSet paths = new PathSet(); - this.accessed.addAll(this.sources.keySet()); - Map newVertices = newMap(); + LongLongHashMap newLayer = new LongLongHashMap(); long degree = this.skipDegree > 0L ? this.skipDegree : this.degree; - // Traversal vertices of previous level - for (Node v : this.sources.values()) { - Iterator edges = edgesOfVertex(v.id(), this.direction, + + assert !this.sourceLayers.isEmpty(); + assert !this.targetLayers.isEmpty(); + LongLongHashMap sourceTopLayer = this.sourceLayers.peek(); + LongLongHashMap targetTopLayer = this.targetLayers.peek(); + + LongIterator iterator = sourceTopLayer.keySet().longIterator(); + while (iterator.hasNext()) { + long sourceCode = iterator.next(); + Iterator edges = edgesOfVertex(this.id(sourceCode), + this.direction, this.labels, degree); edges = skipSuperNodeIfNeeded(edges, this.degree, this.skipDegree); while (edges.hasNext()) { HugeEdge edge = (HugeEdge) edges.next(); Id target = edge.id().otherVertexId(); + long targetCode = this.code(target); // If cross point exists, shortest path found, concat them - if (this.targets.containsKey(target)) { + if (targetTopLayer.containsKey(targetCode)) { if (this.superNode(target, this.direction)) { continue; } - paths.add(new Path( - v.joinPath(this.targets.get(target)))); + + paths.add(this.getPath(sourceCode, targetCode)); this.foundPath = true; if (!all) { return paths; @@ -206,16 +228,16 @@ public PathSet forward(boolean all) { * 1. not in sources and newVertices yet * 2. path of node doesn't have loop */ - if (!this.foundPath && !newVertices.containsKey(target) && - !this.accessed.contains(target)) { - newVertices.put(target, new Node(target, v)); + if (!this.foundPath && !newLayer.containsKey(targetCode) && + !this.accessed.contains(this.code(target))) { + newLayer.put(targetCode, sourceCode); } } } // Re-init sources - this.sources = newVertices; - this.size += newVertices.size(); + this.sourceLayers.push(newLayer); + this.size += newLayer.size(); return paths; } @@ -225,27 +247,35 @@ public PathSet forward(boolean all) { */ public PathSet backward(boolean all) { PathSet paths = new PathSet(); - this.accessed.addAll(this.targets.keySet()); - Map newVertices = newMap(); + LongLongHashMap newLayer = new LongLongHashMap(); long degree = this.skipDegree > 0L ? this.skipDegree : this.degree; + + assert !this.sourceLayers.isEmpty(); + assert !this.targetLayers.isEmpty(); + LongLongHashMap sourceTopLayer = this.targetLayers.peek(); + LongLongHashMap targetTopLayer = this.sourceLayers.peek(); + Directions opposite = this.direction.opposite(); - // Traversal vertices of previous level - for (Node v : this.targets.values()) { - Iterator edges = edgesOfVertex(v.id(), opposite, - this.labels, degree); + + LongIterator iterator = sourceTopLayer.keySet().longIterator(); + while (iterator.hasNext()) { + long sourceCode = iterator.next(); + Iterator edges = edgesOfVertex(this.id(sourceCode), + opposite, this.labels, + degree); edges = skipSuperNodeIfNeeded(edges, this.degree, this.skipDegree); while (edges.hasNext()) { HugeEdge edge = (HugeEdge) edges.next(); Id target = edge.id().otherVertexId(); + long targetCode = this.code(target); // If cross point exists, shortest path found, concat them - if (this.sources.containsKey(target)) { + if (targetTopLayer.containsKey(targetCode)) { if (this.superNode(target, opposite)) { continue; } - paths.add(new Path( - v.joinPath(this.sources.get(target)))); + paths.add(this.getPath(targetCode, sourceCode)); this.foundPath = true; if (!all) { return paths; @@ -258,16 +288,16 @@ public PathSet backward(boolean all) { * 1. not in targets and newVertices yet * 2. path of node doesn't have loop */ - if (!this.foundPath && !newVertices.containsKey(target) && - !this.accessed.contains(target)) { - newVertices.put(target, new Node(target, v)); + if (!this.foundPath && !newLayer.containsKey(targetCode) && + !this.accessed.contains(targetCode)) { + newLayer.put(targetCode, sourceCode); } } } // Re-init targets - this.targets = newVertices; - this.size += newVertices.size(); + this.targetLayers.push(newLayer); + this.size += newLayer.size(); return paths; } @@ -280,5 +310,37 @@ private boolean superNode(Id vertex, Directions direction) { this.labels, this.skipDegree); return IteratorUtils.count(edges) >= this.skipDegree; } + + private Path getPath(long source, long target) { + int sourceLayerSize = this.sourceLayers.size(); + int targetLayerSize = this.targetLayers.size(); + + List ids = new ArrayList<>(sourceLayerSize + targetLayerSize); + + ids.add(this.id(source)); + long value = source; + for (int i = sourceLayerSize - 1; i > 0 ; i--) { + LongLongHashMap layer = this.sourceLayers.elementAt(i); + value = layer.get(value); + ids.add(this.id(value)); + } + Collections.reverse(ids); + ids.add(this.id(target)); + value = target; + for (int i = this.targetLayers.size() - 1; i > 0 ; i--) { + LongLongHashMap layer = this.targetLayers.elementAt(i); + value = layer.get(value); + ids.add(this.id(value)); + } + return new Path(ids); + } + + private Id id(long code) { + return this.idMapping.getId(code); + } + + private long code(Id id) { + return this.idMapping.getLong(id); + } } } From e0654a17795602a2e759ed689d7183c9de37d2f7 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 30 Mar 2021 19:50:03 +0800 Subject: [PATCH 09/38] use IntObjectMap and IntIntMap to improve shortest path Change-Id: I960167c1c87d1ebcebe29a5a7073d6251239d29d --- .../traversal/algorithm/IdMapping.java | 39 +++++++------ .../algorithm/ShortestPathTraverser.java | 58 +++++++++---------- 2 files changed, 51 insertions(+), 46 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java index cec5eabf50..58f23afee6 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java @@ -19,34 +19,39 @@ package com.baidu.hugegraph.traversal.algorithm; -import org.eclipse.collections.impl.map.mutable.primitive.LongObjectHashMap; -import org.eclipse.collections.impl.map.mutable.primitive.ObjectLongHashMap; +import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; +import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.backend.id.Id; public class IdMapping { - private LongObjectHashMap long2IdMap; - private ObjectLongHashMap id2LongMap; - private long nextLong; + private static final int MAGIC = 1 << 16; + private IntObjectHashMap int2IdMap; public IdMapping() { - this.long2IdMap = new LongObjectHashMap(); - this.id2LongMap = new ObjectLongHashMap(); - this.nextLong = 0L; + this.int2IdMap = new IntObjectHashMap(); } - public long getLong(Id id) { - if (this.id2LongMap.containsKey(id)) { - return this.id2LongMap.get(id); + public int getCode(Id id) { + int key = id.hashCode(); + for (int i = 1; i > 0; i <<= 1) { + for (int j = 0; i >= MAGIC && j < 10; j++) { + Id existed = (Id) this.int2IdMap.get(key); + if (existed == null) { + this.int2IdMap.put(key, id); + return key; + } + if (existed.equals(id)) { + return key; + } + key = key + i + j; + } } - this.nextLong++; - this.id2LongMap.put(id, this.nextLong); - this.long2IdMap.put(this.nextLong, id); - return this.nextLong; + throw new HugeException("Failed to get code for id: %s", id); } - public Id getId(long code) { - return (Id) this.long2IdMap.get(code); + public Id getId(int code) { + return (Id) this.int2IdMap.get(code); } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java index 4ae89f72d5..8fc89b5436 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java @@ -28,8 +28,8 @@ import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; -import org.eclipse.collections.api.iterator.LongIterator; -import org.eclipse.collections.impl.map.mutable.primitive.LongLongHashMap; +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; import org.eclipse.collections.impl.set.mutable.primitive.LongHashSet; import com.baidu.hugegraph.HugeGraph; @@ -142,8 +142,8 @@ private class Traverser { // TODO: change Map to Set to reduce memory cost private IdMapping idMapping; - private Stack sourceLayers; - private Stack targetLayers; + private Stack sourceLayers; + private Stack targetLayers; private LongHashSet accessed = new LongHashSet(); @@ -161,12 +161,12 @@ public Traverser(Id sourceV, Id targetV, Directions dir, Map labels, long degree, long skipDegree, long capacity) { this.idMapping = new IdMapping(); - long sourceCode = this.code(sourceV); - long targetCode = this.code(targetV); - LongLongHashMap firstSourceLayer = new LongLongHashMap(); - LongLongHashMap firstTargetLayer = new LongLongHashMap(); - firstSourceLayer.put(sourceCode, 0L); - firstTargetLayer.put(targetCode, 0L); + int sourceCode = this.code(sourceV); + int targetCode = this.code(targetV); + IntIntHashMap firstSourceLayer = new IntIntHashMap(); + IntIntHashMap firstTargetLayer = new IntIntHashMap(); + firstSourceLayer.put(sourceCode, 0); + firstTargetLayer.put(targetCode, 0); this.sourceLayers = new Stack<>(); this.targetLayers = new Stack<>(); this.sourceLayers.push(firstSourceLayer); @@ -188,17 +188,17 @@ public Traverser(Id sourceV, Id targetV, Directions dir, */ public PathSet forward(boolean all) { PathSet paths = new PathSet(); - LongLongHashMap newLayer = new LongLongHashMap(); + IntIntHashMap newLayer = new IntIntHashMap(); long degree = this.skipDegree > 0L ? this.skipDegree : this.degree; assert !this.sourceLayers.isEmpty(); assert !this.targetLayers.isEmpty(); - LongLongHashMap sourceTopLayer = this.sourceLayers.peek(); - LongLongHashMap targetTopLayer = this.targetLayers.peek(); + IntIntHashMap sourceTopLayer = this.sourceLayers.peek(); + IntIntHashMap targetTopLayer = this.targetLayers.peek(); - LongIterator iterator = sourceTopLayer.keySet().longIterator(); + IntIterator iterator = sourceTopLayer.keySet().intIterator(); while (iterator.hasNext()) { - long sourceCode = iterator.next(); + int sourceCode = iterator.next(); Iterator edges = edgesOfVertex(this.id(sourceCode), this.direction, this.labels, degree); @@ -207,7 +207,7 @@ public PathSet forward(boolean all) { while (edges.hasNext()) { HugeEdge edge = (HugeEdge) edges.next(); Id target = edge.id().otherVertexId(); - long targetCode = this.code(target); + int targetCode = this.code(target); // If cross point exists, shortest path found, concat them if (targetTopLayer.containsKey(targetCode)) { @@ -247,19 +247,19 @@ public PathSet forward(boolean all) { */ public PathSet backward(boolean all) { PathSet paths = new PathSet(); - LongLongHashMap newLayer = new LongLongHashMap(); + IntIntHashMap newLayer = new IntIntHashMap(); long degree = this.skipDegree > 0L ? this.skipDegree : this.degree; assert !this.sourceLayers.isEmpty(); assert !this.targetLayers.isEmpty(); - LongLongHashMap sourceTopLayer = this.targetLayers.peek(); - LongLongHashMap targetTopLayer = this.sourceLayers.peek(); + IntIntHashMap sourceTopLayer = this.targetLayers.peek(); + IntIntHashMap targetTopLayer = this.sourceLayers.peek(); Directions opposite = this.direction.opposite(); - LongIterator iterator = sourceTopLayer.keySet().longIterator(); + IntIterator iterator = sourceTopLayer.keySet().intIterator(); while (iterator.hasNext()) { - long sourceCode = iterator.next(); + int sourceCode = iterator.next(); Iterator edges = edgesOfVertex(this.id(sourceCode), opposite, this.labels, degree); @@ -268,7 +268,7 @@ public PathSet backward(boolean all) { while (edges.hasNext()) { HugeEdge edge = (HugeEdge) edges.next(); Id target = edge.id().otherVertexId(); - long targetCode = this.code(target); + int targetCode = this.code(target); // If cross point exists, shortest path found, concat them if (targetTopLayer.containsKey(targetCode)) { @@ -311,16 +311,16 @@ private boolean superNode(Id vertex, Directions direction) { return IteratorUtils.count(edges) >= this.skipDegree; } - private Path getPath(long source, long target) { + private Path getPath(int source, int target) { int sourceLayerSize = this.sourceLayers.size(); int targetLayerSize = this.targetLayers.size(); List ids = new ArrayList<>(sourceLayerSize + targetLayerSize); ids.add(this.id(source)); - long value = source; + int value = source; for (int i = sourceLayerSize - 1; i > 0 ; i--) { - LongLongHashMap layer = this.sourceLayers.elementAt(i); + IntIntHashMap layer = this.sourceLayers.elementAt(i); value = layer.get(value); ids.add(this.id(value)); } @@ -328,19 +328,19 @@ private Path getPath(long source, long target) { ids.add(this.id(target)); value = target; for (int i = this.targetLayers.size() - 1; i > 0 ; i--) { - LongLongHashMap layer = this.targetLayers.elementAt(i); + IntIntHashMap layer = this.targetLayers.elementAt(i); value = layer.get(value); ids.add(this.id(value)); } return new Path(ids); } - private Id id(long code) { + private Id id(int code) { return this.idMapping.getId(code); } - private long code(Id id) { - return this.idMapping.getLong(id); + private int code(Id id) { + return this.idMapping.getCode(id); } } } From 539c35927f6c7d6c90535b6fdc287ea69a777f60 Mon Sep 17 00:00:00 2001 From: zhoney Date: Wed, 31 Mar 2021 22:57:03 +0800 Subject: [PATCH 10/38] add perf(watch) for shortest path --- .../baidu/hugegraph/api/traversers/ShortestPathAPI.java | 8 +------- .../main/java/com/baidu/hugegraph/StandardHugeGraph.java | 5 +++++ .../hugegraph/backend/cache/CachedGraphTransaction.java | 2 ++ .../hugegraph/backend/cache/CachedSchemaTransaction.java | 2 ++ .../main/java/com/baidu/hugegraph/backend/id/EdgeId.java | 2 ++ .../com/baidu/hugegraph/backend/query/ConditionQuery.java | 2 ++ .../com/baidu/hugegraph/backend/store/ram/RamTable.java | 4 ++++ .../com/baidu/hugegraph/backend/tx/GraphTransaction.java | 2 ++ .../main/java/com/baidu/hugegraph/structure/HugeEdge.java | 5 +++++ .../java/com/baidu/hugegraph/structure/HugeVertex.java | 4 ++++ .../hugegraph/traversal/algorithm/HugeTraverser.java | 5 ++++- .../baidu/hugegraph/traversal/algorithm/IdMapping.java | 3 +++ .../traversal/algorithm/ShortestPathTraverser.java | 6 ++++++ .../java/com/baidu/hugegraph/dist/HugeGraphServer.java | 7 +++++++ 14 files changed, 49 insertions(+), 8 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java index 189a770e64..98f1c7ea41 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java @@ -25,12 +25,7 @@ import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE; import javax.inject.Singleton; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import org.slf4j.Logger; @@ -76,7 +71,6 @@ public String get(@Context GraphManager manager, "max degree '{}', skipped maxDegree '{}' and capacity '{}'", graph, source, target, direction, edgeLabel, depth, maxDegree, skipDegree, capacity); - Id sourceId = VertexAPI.checkAndParseVertexId(source); Id targetId = VertexAPI.checkAndParseVertexId(target); Directions dir = Directions.convert(EdgeAPI.parseDirection(direction)); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java index d09b553035..1a3e87547c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java @@ -71,6 +71,7 @@ import com.baidu.hugegraph.event.EventListener; import com.baidu.hugegraph.exception.NotAllowException; import com.baidu.hugegraph.io.HugeGraphIoRegistry; +import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.rpc.RpcServiceConfig4Client; import com.baidu.hugegraph.rpc.RpcServiceConfig4Server; import com.baidu.hugegraph.schema.EdgeLabel; @@ -454,6 +455,7 @@ private BackendStore loadSystemStore() { return this.storeProvider.loadSystemStore(name); } + @Watched private SchemaTransaction schemaTransaction() { this.checkGraphNotClosed(); /* @@ -473,6 +475,7 @@ private SysTransaction systemTransaction() { return this.tx.systemTransaction(); } + @Watched private GraphTransaction graphTransaction() { this.checkGraphNotClosed(); /* @@ -657,6 +660,7 @@ public Iterator edges(Object... objects) { } @Override + @Watched public Iterator edges(Query query) { return this.graphTransaction().queryEdges(query); } @@ -728,6 +732,7 @@ public Collection vertexLabels() { } @Override + @Watched public VertexLabel vertexLabelOrNone(Id id) { VertexLabel vl = this.schemaTransaction().getVertexLabel(id); if (vl == null) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java index a24d54df3d..6ef6684c17 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java @@ -44,6 +44,7 @@ import com.baidu.hugegraph.exception.NotSupportException; import com.baidu.hugegraph.iterator.ExtendableIterator; import com.baidu.hugegraph.iterator.ListIterator; +import com.baidu.hugegraph.perf.PerfUtil; import com.baidu.hugegraph.schema.IndexLabel; import com.baidu.hugegraph.structure.HugeEdge; import com.baidu.hugegraph.structure.HugeVertex; @@ -264,6 +265,7 @@ private Iterator queryVerticesByIds(IdQuery query) { } @Override + @PerfUtil.Watched protected final Iterator queryEdgesFromBackend(Query query) { RamTable ramtable = this.params().ramtable(); if (ramtable != null && ramtable.matched(query)) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java index 7adc1bf77a..f94d52d78d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java @@ -33,6 +33,7 @@ import com.baidu.hugegraph.config.CoreOptions; import com.baidu.hugegraph.event.EventHub; import com.baidu.hugegraph.event.EventListener; +import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.schema.SchemaElement; import com.baidu.hugegraph.type.HugeType; import com.baidu.hugegraph.util.E; @@ -348,6 +349,7 @@ public void updateIfNeeded(V schema) { } } + @Watched public V get(HugeType type, Id id) { assert id.number(); long longId = id.asLong(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java index 0673888d3c..1832702150 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java @@ -20,6 +20,7 @@ package com.baidu.hugegraph.backend.id; import com.baidu.hugegraph.exception.NotFoundException; +import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeVertex; import com.baidu.hugegraph.type.HugeType; import com.baidu.hugegraph.type.define.Directions; @@ -76,6 +77,7 @@ public EdgeId(Id ownerVertexId, Directions direction, Id edgeLabelId, this.cache = null; } + @Watched public EdgeId switchDirection() { Directions direction = this.direction.opposite(); return new EdgeId(this.otherVertexId, direction, this.edgeLabelId, diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java index 80e2bf9c18..2ff7857bd0 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java @@ -35,6 +35,7 @@ import com.baidu.hugegraph.backend.id.SplicingIdGenerator; import com.baidu.hugegraph.backend.query.Condition.Relation; import com.baidu.hugegraph.backend.query.Condition.RelationType; +import com.baidu.hugegraph.perf.PerfUtil; import com.baidu.hugegraph.structure.HugeElement; import com.baidu.hugegraph.type.HugeType; import com.baidu.hugegraph.type.define.HugeKeys; @@ -141,6 +142,7 @@ public List relations() { return relations; } + @PerfUtil.Watched public T condition(Object key) { List values = new ArrayList<>(); for (Condition c : this.conditions) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/RamTable.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/RamTable.java index a87815b5d3..a9aaaebfcc 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/RamTable.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/RamTable.java @@ -50,6 +50,7 @@ import com.baidu.hugegraph.backend.query.Condition; import com.baidu.hugegraph.backend.query.ConditionQuery; import com.baidu.hugegraph.backend.query.Query; +import com.baidu.hugegraph.perf.PerfUtil; import com.baidu.hugegraph.schema.EdgeLabel; import com.baidu.hugegraph.schema.VertexLabel; import com.baidu.hugegraph.structure.HugeEdge; @@ -254,6 +255,7 @@ public long edgesSize() { return this.edges.size() - 1L; } + @PerfUtil.Watched public boolean matched(Query query) { if (this.edgesSize() == 0L || this.loading) { return false; @@ -293,6 +295,7 @@ public boolean matched(Query query) { return matchedConds == cq.conditions().size(); } + @PerfUtil.Watched public Iterator query(Query query) { assert this.matched(query); assert this.edgesSize() > 0; @@ -311,6 +314,7 @@ public Iterator query(Query query) { return this.query(owner.asLong(), dir, (int) label.asLong()); } + @PerfUtil.Watched public Iterator query(long owner, Directions dir, int label) { if (this.loading) { // don't query when loading diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java index a2af8b0d3d..702bb1bb59 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java @@ -896,6 +896,7 @@ public Iterator queryEdges() { return this.queryEdges(q); } + @Watched public Iterator queryEdges(Query query) { E.checkArgument(this.removedEdges.isEmpty() || query.noLimit(), "It's not allowed to query with limit when " + @@ -1131,6 +1132,7 @@ public void removeEdgeProperty(HugeEdgeProperty prop) { * @param edgeLabels edge labels of queried edges * @return constructed condition query */ + @Watched public static ConditionQuery constructEdgesQuery(Id sourceVertex, Directions direction, Id... edgeLabels) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java index 91987f8a4a..5fbb7dd346 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java @@ -58,6 +58,7 @@ public class HugeEdge extends HugeElement implements Edge, Cloneable { private HugeVertex targetVertex; private boolean isOutEdge; + @Watched public HugeEdge(HugeVertex owner, Id id, EdgeLabel label, HugeVertex other) { this(owner.graph(), id, label); @@ -65,6 +66,7 @@ public HugeEdge(HugeVertex owner, Id id, EdgeLabel label, this.vertices(owner, other); } + @Watched public HugeEdge(final HugeGraph graph, Id id, EdgeLabel label) { super(graph); @@ -370,6 +372,7 @@ public void vertices(boolean outEdge, HugeVertex owner, HugeVertex other) { } } + @Watched public HugeEdge switchOwner() { HugeEdge edge = this.clone(); edge.isOutEdge = !edge.isOutEdge; @@ -488,6 +491,7 @@ public static final EdgeId getIdValue(Object idValue, return EdgeId.parse(id.asString(), returnNullIfError); } + @Watched public static HugeEdge constructEdge(HugeVertex ownerVertex, boolean isOutEdge, EdgeLabel edgeLabel, @@ -498,6 +502,7 @@ public static HugeEdge constructEdge(HugeVertex ownerVertex, } + @Watched public static HugeEdge constructEdge(HugeVertex ownerVertex, boolean isOutEdge, EdgeLabel edgeLabel, diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java index 82311de3f9..b946f46bbd 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java @@ -47,6 +47,7 @@ import com.baidu.hugegraph.backend.serializer.BytesBuffer; import com.baidu.hugegraph.backend.tx.GraphTransaction; import com.baidu.hugegraph.config.CoreOptions; +import com.baidu.hugegraph.perf.PerfUtil; import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.schema.EdgeLabel; import com.baidu.hugegraph.schema.PropertyKey; @@ -69,6 +70,7 @@ public class HugeVertex extends HugeElement implements Vertex, Cloneable { // not have replacement for LinkedHashMap now. private List edges; + @Watched public HugeVertex(final HugeGraph graph, Id id, VertexLabel label) { super(graph); @@ -335,6 +337,7 @@ public void addOutEdge(HugeEdge edge) { * Add edge with direction OUT * @param edge the out edge */ + @Watched public void addOutEdge(HugeEdge edge, boolean needDedup) { if (edge.ownerVertex() == null) { edge.sourceVertex(this); @@ -352,6 +355,7 @@ public void addInEdge(HugeEdge edge) { * Add edge with direction IN * @param edge the in edge */ + @Watched public void addInEdge(HugeEdge edge, boolean needDedup) { if (edge.ownerVertex() == null) { edge.targetVertex(this); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java index 54a368d03c..9da09710c3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java @@ -48,6 +48,7 @@ import com.baidu.hugegraph.iterator.FilterIterator; import com.baidu.hugegraph.iterator.LimitIterator; import com.baidu.hugegraph.iterator.MapperIterator; +import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.schema.SchemaLabel; import com.baidu.hugegraph.structure.HugeEdge; import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; @@ -58,8 +59,8 @@ import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.util.CollectionUtil; import com.baidu.hugegraph.util.E; -import com.baidu.hugegraph.util.collection.CollectionFactory; import com.baidu.hugegraph.util.InsertionOrderUtil; +import com.baidu.hugegraph.util.collection.CollectionFactory; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -177,6 +178,7 @@ protected Set adjacentVertices(Id start, Set vertices, return neighbors; } + @Watched protected Iterator edgesOfVertex(Id source, Directions dir, Id label, long limit) { Id[] labels = {}; @@ -191,6 +193,7 @@ protected Iterator edgesOfVertex(Id source, Directions dir, return this.graph.edges(query); } + @Watched protected Iterator edgesOfVertex(Id source, Directions dir, Map labels, long limit) { if (labels == null || labels.isEmpty()) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java index 58f23afee6..63d96dfc10 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java @@ -23,6 +23,7 @@ import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.perf.PerfUtil; public class IdMapping { @@ -33,6 +34,7 @@ public IdMapping() { this.int2IdMap = new IntObjectHashMap(); } + @PerfUtil.Watched public int getCode(Id id) { int key = id.hashCode(); for (int i = 1; i > 0; i <<= 1) { @@ -51,6 +53,7 @@ public int getCode(Id id) { throw new HugeException("Failed to get code for id: %s", id); } + @PerfUtil.Watched public Id getId(int code) { return (Id) this.int2IdMap.get(code); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java index 8fc89b5436..77600fc4e6 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java @@ -34,6 +34,7 @@ import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeEdge; import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.type.define.Directions; @@ -46,6 +47,7 @@ public ShortestPathTraverser(HugeGraph graph) { super(graph); } + @Watched public Path shortestPath(Id sourceV, Id targetV, Directions dir, List labels, int depth, long degree, long skipDegree, long capacity) { @@ -186,6 +188,7 @@ public Traverser(Id sourceV, Id targetV, Directions dir, /** * Search forward from source */ + @Watched public PathSet forward(boolean all) { PathSet paths = new PathSet(); IntIntHashMap newLayer = new IntIntHashMap(); @@ -245,6 +248,7 @@ public PathSet forward(boolean all) { /** * Search backward from target */ + @Watched public PathSet backward(boolean all) { PathSet paths = new PathSet(); IntIntHashMap newLayer = new IntIntHashMap(); @@ -335,10 +339,12 @@ private Path getPath(int source, int target) { return new Path(ids); } + @Watched private Id id(int code) { return this.idMapping.getId(code); } + @Watched private int code(Id id) { return this.idMapping.getCode(id); } diff --git a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java index 7b0def5ab6..b8c940b756 100644 --- a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java +++ b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java @@ -19,6 +19,7 @@ package com.baidu.hugegraph.dist; +import com.baidu.hugegraph.perf.PerfUtil; import org.apache.tinkerpop.gremlin.server.GremlinServer; import org.slf4j.Logger; @@ -95,6 +96,12 @@ public static void main(String[] args) throws Exception { throw new HugeException(msg); } + //PerfUtil.instance().profilePackage("com.baidu.hugegraph."); + //PerfUtil.instance().profileClass("com.baidu.hugegraph.traversal" + + // "algorithm.ShortestPathTraverser$Traverser"); + //PerfUtil.instance().profileClass("com.baidu.hugegraph.backend.cache." + + // "CachedSchemaTransaction$SchemaCaches"); + HugeRestServer.register(); HugeGraphServer server = new HugeGraphServer(args[0], args[1]); From 48cb96f3ead72efbabf926fa4c7002d8ac2d4a52 Mon Sep 17 00:00:00 2001 From: zhoney Date: Mon, 5 Apr 2021 15:20:51 +0800 Subject: [PATCH 11/38] improve paths api by remove redundant sourcesAll and targetsAll --- .../traversal/algorithm/PathsTraverser.java | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java index 0253fbdbf9..3ec706e104 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java @@ -85,21 +85,19 @@ private class Traverser { private MultivaluedMap sources = newMultivalueMap(); private MultivaluedMap targets = newMultivalueMap(); - private MultivaluedMap sourcesAll = newMultivalueMap(); - private MultivaluedMap targetsAll = newMultivalueMap(); private final Id label; private final long degree; private final long capacity; private final long limit; private PathSet paths; + private int accessed; public Traverser(Id sourceV, Id targetV, Id label, long degree, long capacity, long limit) { this.sources.add(sourceV, new Node(sourceV)); this.targets.add(targetV, new Node(targetV)); - this.sourcesAll.putAll(this.sources); - this.targetsAll.putAll(this.targets); + this.accessed = 2; this.label = label; this.degree = degree; this.capacity = capacity; @@ -129,8 +127,8 @@ public void forward(Directions direction) { } // If cross point exists, path found, concat them - if (this.targetsAll.containsKey(target)) { - for (Node node : this.targetsAll.get(target)) { + if (this.targets.containsKey(target)) { + for (Node node : this.targets.get(target)) { List path = n.joinPath(node); if (!path.isEmpty()) { this.paths.add(new Path(target, path)); @@ -148,10 +146,7 @@ public void forward(Directions direction) { } // Re-init sources this.sources = newVertices; - // Record all passed vertices - for (Map.Entry> entry : newVertices.entrySet()) { - this.sourcesAll.addAll(entry.getKey(), entry.getValue()); - } + this.accessed += this.sources.size(); } /** @@ -176,8 +171,8 @@ public void backward(Directions direction) { } // If cross point exists, path found, concat them - if (this.sourcesAll.containsKey(target)) { - for (Node node : this.sourcesAll.get(target)) { + if (this.sources.containsKey(target)) { + for (Node node : this.sources.get(target)) { List path = n.joinPath(node); if (!path.isEmpty()) { Path newPath = new Path(target, path); @@ -198,22 +193,15 @@ public void backward(Directions direction) { // Re-init targets this.targets = newVertices; - // Record all passed vertices - for (Map.Entry> entry : newVertices.entrySet()) { - this.targetsAll.addAll(entry.getKey(), entry.getValue()); - } + this.accessed = this.targets.size(); } public PathSet paths() { return this.paths; } - private int accessedNodes() { - return this.sourcesAll.size() + this.targetsAll.size(); - } - private boolean reachLimit() { - checkCapacity(this.capacity, this.accessedNodes(), "paths"); + checkCapacity(this.capacity, this.accessed, "paths"); if (this.limit == NO_LIMIT || this.paths.size() < this.limit) { return false; } From 5b1dfb0bf2e5e5d3f3fb95334081d22f423366ce Mon Sep 17 00:00:00 2001 From: zhoney Date: Wed, 7 Apr 2021 12:49:42 +0800 Subject: [PATCH 12/38] improve paths api performance by using IntObjectHashMap --- .../traversal/algorithm/HugeTraverser.java | 15 ++ .../traversal/algorithm/PathsTraverser.java | 232 +++++++++++++----- .../algorithm/ShortestPathTraverser.java | 16 +- 3 files changed, 191 insertions(+), 72 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java index 9da09710c3..58d8ef4f60 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java @@ -70,6 +70,7 @@ public class HugeTraverser { private HugeGraph graph; private static CollectionFactory collectionFactory; + protected IdMapping idMapping; public static final String DEFAULT_CAPACITY = "10000000"; public static final String DEFAULT_ELEMENTS_LIMIT = "10000000"; @@ -505,6 +506,16 @@ protected static List joinPath(Node prev, Node back, boolean ring) { return path; } + @Watched + protected Id id(int code) { + return this.idMapping.getId(code); + } + + @Watched + protected int code(Id id) { + return this.idMapping.getCode(id); + } + public static class Node { private Id id; @@ -611,6 +622,10 @@ public Id crosspoint() { return this.crosspoint; } + public void addToEnd(Id id) { + this.vertices.add(id); + } + public List vertices() { return this.vertices; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java index 3ec706e104..dbece4c1ef 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java @@ -19,13 +19,16 @@ package com.baidu.hugegraph.traversal.algorithm; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.Map; - -import javax.ws.rs.core.MultivaluedMap; +import java.util.Stack; +import org.apache.commons.collections.CollectionUtils; import org.apache.tinkerpop.gremlin.structure.Edge; +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; +import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.id.Id; @@ -33,11 +36,13 @@ import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; public class PathsTraverser extends HugeTraverser { public PathsTraverser(HugeGraph graph) { super(graph); + this.idMapping = new IdMapping(); } public PathSet paths(Id sourceV, Directions sourceDir, @@ -83,8 +88,8 @@ public PathSet paths(Id sourceV, Directions sourceDir, private class Traverser { - private MultivaluedMap sources = newMultivalueMap(); - private MultivaluedMap targets = newMultivalueMap(); + private Stack> sourceLayers; + private Stack> targetLayers; private final Id label; private final long degree; @@ -95,105 +100,202 @@ private class Traverser { public Traverser(Id sourceV, Id targetV, Id label, long degree, long capacity, long limit) { - this.sources.add(sourceV, new Node(sourceV)); - this.targets.add(targetV, new Node(targetV)); this.accessed = 2; this.label = label; this.degree = degree; this.capacity = capacity; this.limit = limit; this.paths = new PathSet(); + + int sourceCode = this.code(sourceV); + int targetCode = this.code(targetV); + IntObjectHashMap firstSourceLayer = + new IntObjectHashMap<>(); + IntObjectHashMap firstTargetLayer = + new IntObjectHashMap<>(); + firstSourceLayer.put(sourceCode, new IntHashSet()); + firstTargetLayer.put(targetCode, new IntHashSet()); + this.sourceLayers = new Stack<>(); + this.targetLayers = new Stack<>(); + this.sourceLayers.push(firstSourceLayer); + this.targetLayers.push(firstTargetLayer); } /** * Search forward from source */ public void forward(Directions direction) { - MultivaluedMap newVertices = newMultivalueMap(); - Iterator edges; // Traversal vertices of previous level - for (Map.Entry> entry : this.sources.entrySet()) { - Id vid = entry.getKey(); + assert !this.sourceLayers.isEmpty(); + IntObjectHashMap sourceTopLayer = + this.sourceLayers.peek(); + IntObjectHashMap newSourceLayer = + new IntObjectHashMap<>(); + Iterator edges; + IntIterator keys = sourceTopLayer.keySet().intIterator(); + while (keys.hasNext()) { + int id = keys.next(); + Id vid = this.id(id); edges = edgesOfVertex(vid, direction, this.label, this.degree); while (edges.hasNext()) { HugeEdge edge = (HugeEdge) edges.next(); - Id target = edge.id().otherVertexId(); - - for (Node n : entry.getValue()) { - // If have loop, skip target - if (n.contains(target)) { - continue; - } + Id targetId = edge.id().otherVertexId(); + int target = this.code(targetId); - // If cross point exists, path found, concat them - if (this.targets.containsKey(target)) { - for (Node node : this.targets.get(target)) { - List path = n.joinPath(node); - if (!path.isEmpty()) { - this.paths.add(new Path(target, path)); - if (this.reachLimit()) { - return; - } - } + // If cross point exists, path found, concat them + if (this.targetLayers.peek().containsKey(target)) { + List paths = this.getPath(id, target, false); + for (Path path : paths) { + this.paths.add(path); + if (this.reachLimit()) { + return; } } - - // Add node to next start-nodes - newVertices.add(target, new Node(target, n)); } + this.add(newSourceLayer, target, id); } + } - // Re-init sources - this.sources = newVertices; - this.accessed += this.sources.size(); + this.accessed += newSourceLayer.size(); + this.sourceLayers.push(newSourceLayer); } /** * Search backward from target */ public void backward(Directions direction) { - MultivaluedMap newVertices = newMultivalueMap(); + assert !this.targetLayers.isEmpty(); + IntObjectHashMap targetTopLayer = + this.targetLayers.peek(); + IntObjectHashMap newTargetLayer = + new IntObjectHashMap<>(); Iterator edges; - // Traversal vertices of previous level - for (Map.Entry> entry : this.targets.entrySet()) { - Id vid = entry.getKey(); + IntIterator keys = targetTopLayer.keySet().intIterator(); + while (keys.hasNext()) { + int id = keys.next(); + Id vid = this.id(id); edges = edgesOfVertex(vid, direction, this.label, this.degree); while (edges.hasNext()) { HugeEdge edge = (HugeEdge) edges.next(); - Id target = edge.id().otherVertexId(); + Id targetId = edge.id().otherVertexId(); + int target = this.code(targetId); - for (Node n : entry.getValue()) { - // If have loop, skip target - if (n.contains(target)) { - continue; + // If cross point exists, path found, concat them + if (this.sourceLayers.peek().containsKey(target)) { + List paths = this.getPath(target, id, false); + for (Path path : paths) { + this.paths.add(path); + if (this.reachLimit()) { + return; + } } + } - // If cross point exists, path found, concat them - if (this.sources.containsKey(target)) { - for (Node node : this.sources.get(target)) { - List path = n.joinPath(node); - if (!path.isEmpty()) { - Path newPath = new Path(target, path); - newPath.reverse(); - this.paths.add(newPath); - if (this.reachLimit()) { - return; - } - } - } + this.add(newTargetLayer, target, id); + } + } + + // Re-init targets + this.accessed += newTargetLayer.size(); + this.targetLayers.push(newTargetLayer); + } + + private boolean hasLoop(Stack> all, + int current, int target) { + + if (current == target) { + return true; + } + int layers = all.size(); + IntHashSet keys = IntHashSet.newSetWith(current); + for (int i = layers - 1; i > 0 ; i--) { + IntObjectHashMap currentLayer = + this.sourceLayers.elementAt(i); + IntIterator iterator = keys.intIterator(); + IntHashSet parents = null; + while (iterator.hasNext()) { + int key = iterator.next(); + parents = currentLayer.get(key); + if (!parents.isEmpty() && parents.contains(target)) { + return true; + } + } + keys = parents; + } + return false; + } + + private List getPath(int source, int target, boolean ring) { + List results = new ArrayList<>(); + List sources = this.getSourcePath(source); + List targets = this.getTargetPath(target); + for (Path tpath : targets) { + tpath.reverse(); + for (Path spath : sources) { + if (!ring) { + // Avoid loop in path + if (CollectionUtils.containsAny(spath.vertices(), + tpath.vertices())) { + continue; } + } + List ids = new ArrayList<>(spath.vertices()); + ids.addAll(tpath.vertices()); + results.add(new Path(ids)); + } + } + return results; + } + + private List getPath(Stack> all, + int id, int layerIndex) { + if (layerIndex == 0) { + Id sid = this.id(id); + return ImmutableList.of(new Path(Lists.newArrayList(sid))); + } - // Add node to next start-nodes - newVertices.add(target, new Node(target, n)); + Id sid = this.id(id); + List results = new ArrayList<>(); + IntObjectHashMap layer = all.elementAt(layerIndex); + IntHashSet parents = layer.get(id); + IntIterator iterator = parents.intIterator(); + while (iterator.hasNext()) { + int parent = iterator.next(); + List paths = this.getPath(all, parent, layerIndex - 1); + for (Iterator iter = paths.iterator(); iter.hasNext();) { + Path path = iter.next(); + if (path.vertices().contains(sid)) { + iter.remove(); + continue; } + path.addToEnd(sid); } + + results.addAll(paths); } + return results; + } - // Re-init targets - this.targets = newVertices; - this.accessed = this.targets.size(); + private List getSourcePath(int source) { + return this.getPath(this.sourceLayers, source, + this.sourceLayers.size() - 1); + } + + private List getTargetPath(int target) { + return this.getPath(this.targetLayers, target, + this.targetLayers.size() - 1); + } + + private void add(IntObjectHashMap layer, + int current, int parent) { + + if (layer.containsKey(current)) { + layer.get(current).add(parent); + } else { + layer.put(current, IntHashSet.newSetWith(parent)); + } } public PathSet paths() { @@ -207,5 +309,13 @@ private boolean reachLimit() { } return true; } + + private int code(Id id) { + return PathsTraverser.this.code(id); + } + + private Id id(int code) { + return PathsTraverser.this.id(code); + } } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java index 77600fc4e6..da5ae25888 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java @@ -45,6 +45,7 @@ public class ShortestPathTraverser extends HugeTraverser { public ShortestPathTraverser(HugeGraph graph) { super(graph); + this.idMapping = new IdMapping(); } @Watched @@ -141,13 +142,9 @@ public PathSet allShortestPaths(Id sourceV, Id targetV, Directions dir, private class Traverser { - // TODO: change Map to Set to reduce memory cost - private IdMapping idMapping; - private Stack sourceLayers; private Stack targetLayers; - private LongHashSet accessed = new LongHashSet(); private final Directions direction; @@ -162,7 +159,6 @@ private class Traverser { public Traverser(Id sourceV, Id targetV, Directions dir, Map labels, long degree, long skipDegree, long capacity) { - this.idMapping = new IdMapping(); int sourceCode = this.code(sourceV); int targetCode = this.code(targetV); IntIntHashMap firstSourceLayer = new IntIntHashMap(); @@ -339,14 +335,12 @@ private Path getPath(int source, int target) { return new Path(ids); } - @Watched - private Id id(int code) { - return this.idMapping.getId(code); + private int code(Id id) { + return ShortestPathTraverser.this.code(id); } - @Watched - private int code(Id id) { - return this.idMapping.getCode(id); + private Id id(int code) { + return ShortestPathTraverser.this.id(code); } } } From 22639d94e01c31640dab9da04790a6d43c422638 Mon Sep 17 00:00:00 2001 From: zhoney Date: Wed, 7 Apr 2021 21:07:28 +0800 Subject: [PATCH 13/38] improve and perf paths api --- .../api/traversers/CrosspointsAPI.java | 2 +- .../hugegraph/api/traversers/PathsAPI.java | 10 ++++ .../backend/cache/CachedBackendStore.java | 6 -- .../backend/cache/CachedGraphTransaction.java | 4 +- .../baidu/hugegraph/backend/id/EdgeId.java | 5 -- .../com/baidu/hugegraph/backend/id/Id.java | 2 - .../hugegraph/backend/id/IdGenerator.java | 20 ------- .../backend/query/ConditionQuery.java | 4 +- .../serializer/BinaryBackendEntry.java | 5 -- .../backend/serializer/BinarySerializer.java | 3 +- .../backend/serializer/TableSerializer.java | 2 +- .../backend/serializer/TextSerializer.java | 2 +- .../hugegraph/backend/store/ram/RamTable.java | 10 ++-- .../baidu/hugegraph/structure/HugeEdge.java | 24 ++------ .../hugegraph/structure/HugeElement.java | 27 ++++++--- .../baidu/hugegraph/structure/HugeVertex.java | 47 +++++++-------- .../traversal/algorithm/HugeTraverser.java | 58 +++++++++++++++++-- .../traversal/algorithm/PathsTraverser.java | 13 ++++- .../algorithm/ShortestPathTraverser.java | 10 ++-- .../algorithm/SubGraphTraverser.java | 2 +- .../strategy/SingleTraverseStrategy.java | 2 +- .../util/collection/CollectionFactory.java | 16 +++++ .../hugegraph/util/collection/IdSet.java | 20 +++---- .../collection/ObjectIntMapping.java} | 20 +++---- .../baidu/hugegraph/dist/HugeGraphServer.java | 12 ++-- .../baidu/hugegraph/example/ExampleUtil.java | 4 +- 26 files changed, 185 insertions(+), 145 deletions(-) rename hugegraph-core/src/main/java/com/baidu/hugegraph/{traversal/algorithm/IdMapping.java => util/collection/ObjectIntMapping.java} (82%) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CrosspointsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CrosspointsAPI.java index 59f054d18a..8efbab6917 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CrosspointsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CrosspointsAPI.java @@ -85,6 +85,6 @@ public String get(@Context GraphManager manager, dir, edgeLabel, depth, maxDegree, capacity, limit); return manager.serializer(g).writePaths("crosspoints", - paths.paths(),true); + paths.paths(), true); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java index e98e1384ec..8093c9bdd5 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java @@ -48,6 +48,7 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.query.QueryResults; import com.baidu.hugegraph.core.GraphManager; +import com.baidu.hugegraph.perf.PerfUtil; import com.baidu.hugegraph.server.RestServer; import com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser; import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; @@ -87,6 +88,10 @@ public String get(@Context GraphManager manager, graph, source, target, direction, edgeLabel, depth, maxDegree, capacity, limit); + PerfUtil.instance().clear(); + PerfUtil.instance().start("paths-get"); + try { + Id sourceId = VertexAPI.checkAndParseVertexId(source); Id targetId = VertexAPI.checkAndParseVertexId(target); Directions dir = Directions.convert(EdgeAPI.parseDirection(direction)); @@ -99,6 +104,11 @@ public String get(@Context GraphManager manager, limit); return manager.serializer(g).writePaths("paths", paths.paths(), false); + + } finally { + PerfUtil.instance().end("paths-get"); + LOG.info("option = {}", PerfUtil.instance().toECharts()); + } } @POST diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedBackendStore.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedBackendStore.java index e27f4899f6..b2daec42fb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedBackendStore.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedBackendStore.java @@ -225,12 +225,6 @@ public long asLong() { return 0L; } - @Override - public int asInt() { - // TODO: improve - return 0; - } - @Override public byte[] asBytes() { return StringEncoding.encode(this.query); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java index 6ef6684c17..37d7bf8cf6 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java @@ -44,7 +44,7 @@ import com.baidu.hugegraph.exception.NotSupportException; import com.baidu.hugegraph.iterator.ExtendableIterator; import com.baidu.hugegraph.iterator.ListIterator; -import com.baidu.hugegraph.perf.PerfUtil; +import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.schema.IndexLabel; import com.baidu.hugegraph.structure.HugeEdge; import com.baidu.hugegraph.structure.HugeVertex; @@ -265,7 +265,7 @@ private Iterator queryVerticesByIds(IdQuery query) { } @Override - @PerfUtil.Watched + @Watched protected final Iterator queryEdgesFromBackend(Query query) { RamTable ramtable = this.params().ramtable(); if (ramtable != null && ramtable.matched(query)) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java index 1832702150..60a5aa7194 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java @@ -157,11 +157,6 @@ public long asLong() { throw new UnsupportedOperationException(); } - @Override - public int asInt() { - throw new UnsupportedOperationException(); - } - @Override public byte[] asBytes() { return StringEncoding.encode(this.asString()); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/Id.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/Id.java index 689fd5b74d..78afad4823 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/Id.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/Id.java @@ -31,8 +31,6 @@ public interface Id extends Comparable { public long asLong(); - public int asInt(); - public byte[] asBytes(); public int length(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java index 224de5da8e..1858fb4c0b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java @@ -160,11 +160,6 @@ public long asLong() { return Long.parseLong(this.id); } - @Override - public int asInt() { - return Integer.parseInt(this.id); - } - @Override public byte[] asBytes() { return StringEncoding.encode(this.id); @@ -238,11 +233,6 @@ public long asLong() { return this.id; } - @Override - public int asInt() { - return (int) this.id; - } - @Override public byte[] asBytes() { return NumericUtil.longToBytes(this.id); @@ -338,11 +328,6 @@ public long asLong() { throw new UnsupportedOperationException(); } - @Override - public int asInt() { - throw new UnsupportedOperationException(); - } - @Override public byte[] asBytes() { BytesBuffer buffer = BytesBuffer.allocate(16); @@ -425,11 +410,6 @@ public long asLong() { throw new UnsupportedOperationException(); } - @Override - public int asInt() { - throw new UnsupportedOperationException(); - } - @Override public byte[] asBytes() { throw new UnsupportedOperationException(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java index 2ff7857bd0..d083b8f674 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java @@ -35,7 +35,7 @@ import com.baidu.hugegraph.backend.id.SplicingIdGenerator; import com.baidu.hugegraph.backend.query.Condition.Relation; import com.baidu.hugegraph.backend.query.Condition.RelationType; -import com.baidu.hugegraph.perf.PerfUtil; +import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeElement; import com.baidu.hugegraph.type.HugeType; import com.baidu.hugegraph.type.define.HugeKeys; @@ -142,7 +142,7 @@ public List relations() { return relations; } - @PerfUtil.Watched + @Watched public T condition(Object key) { List values = new ArrayList<>(); for (Condition c : this.conditions) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryBackendEntry.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryBackendEntry.java index 3bfbbbca8a..6b77141d7a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryBackendEntry.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryBackendEntry.java @@ -208,11 +208,6 @@ public long asLong() { throw new UnsupportedOperationException(); } - @Override - public int asInt() { - throw new UnsupportedOperationException(); - } - @Override public int compareTo(Id other) { return Bytes.compare(this.bytes, other.asBytes()); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java index 0aaa6dde46..9e3441f67e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java @@ -266,8 +266,7 @@ protected void parseEdge(BackendColumn col, HugeVertex vertex, // Construct edge HugeEdge edge = HugeEdge.constructEdge(vertex, direction, edgeLabel, - sortValues, otherVertexId, - false); + sortValues, otherVertexId); // Parse edge-id + edge-properties buffer = BytesBuffer.wrap(col.value); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java index 94b57fcfad..f1659f0c84 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java @@ -179,7 +179,7 @@ protected HugeEdge parseEdge(TableBackendEntry.Row row, // Construct edge HugeEdge edge = HugeEdge.constructEdge(vertex, direction, edgeLabel, - sortValues, otherId, false); + sortValues, otherId); // Parse edge properties this.parseProperties(edge, row); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TextSerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TextSerializer.java index 9af6f8b1fc..18d896046e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TextSerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TextSerializer.java @@ -225,7 +225,7 @@ private void parseEdge(String colName, String colValue, Id otherVertexId = readEntryId(colParts[3]); // Construct edge HugeEdge edge = HugeEdge.constructEdge(vertex, direction, edgeLabel, - sortValues, otherVertexId, false); + sortValues, otherVertexId); String[] valParts = colValue.split(VALUE_SPLITOR); // Parse edge expired time diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/RamTable.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/RamTable.java index a9aaaebfcc..728b995578 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/RamTable.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/ram/RamTable.java @@ -50,7 +50,7 @@ import com.baidu.hugegraph.backend.query.Condition; import com.baidu.hugegraph.backend.query.ConditionQuery; import com.baidu.hugegraph.backend.query.Query; -import com.baidu.hugegraph.perf.PerfUtil; +import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.schema.EdgeLabel; import com.baidu.hugegraph.schema.VertexLabel; import com.baidu.hugegraph.structure.HugeEdge; @@ -255,7 +255,7 @@ public long edgesSize() { return this.edges.size() - 1L; } - @PerfUtil.Watched + @Watched public boolean matched(Query query) { if (this.edgesSize() == 0L || this.loading) { return false; @@ -295,7 +295,7 @@ public boolean matched(Query query) { return matchedConds == cq.conditions().size(); } - @PerfUtil.Watched + @Watched public Iterator query(Query query) { assert this.matched(query); assert this.edgesSize() > 0; @@ -314,7 +314,7 @@ public Iterator query(Query query) { return this.query(owner.asLong(), dir, (int) label.asLong()); } - @PerfUtil.Watched + @Watched public Iterator query(long owner, Directions dir, int label) { if (this.loading) { // don't query when loading @@ -455,7 +455,7 @@ private HugeEdge fetch() { HugeEdge edge = HugeEdge.constructEdge(this.owner, direction, edgeLabel, sortValues, - otherVertexId, false); + otherVertexId); edge.propNotLoaded(); return edge; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java index 5fbb7dd346..28cfe3142a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java @@ -497,28 +497,16 @@ public static HugeEdge constructEdge(HugeVertex ownerVertex, EdgeLabel edgeLabel, String sortValues, Id otherVertexId) { - return constructEdge(ownerVertex, isOutEdge, edgeLabel, - sortValues, otherVertexId, true); - - } - - @Watched - public static HugeEdge constructEdge(HugeVertex ownerVertex, - boolean isOutEdge, - EdgeLabel edgeLabel, - String sortValues, - Id otherVertexId, - boolean needDedup) { HugeGraph graph = ownerVertex.graph(); VertexLabel srcLabel = graph.vertexLabelOrNone(edgeLabel.sourceLabel()); VertexLabel tgtLabel = graph.vertexLabelOrNone(edgeLabel.targetLabel()); VertexLabel otherVertexLabel; if (isOutEdge) { -// ownerVertex.correctVertexLabel(srcLabel); + ownerVertex.correctVertexLabel(srcLabel); otherVertexLabel = tgtLabel; } else { -// ownerVertex.correctVertexLabel(tgtLabel); + ownerVertex.correctVertexLabel(tgtLabel); otherVertexLabel = srcLabel; } HugeVertex otherVertex = new HugeVertex(graph, otherVertexId, @@ -533,11 +521,11 @@ public static HugeEdge constructEdge(HugeVertex ownerVertex, edge.assignId(); if (isOutEdge) { - ownerVertex.addOutEdge(edge, needDedup); - otherVertex.addInEdge(edge.switchOwner(), needDedup); + ownerVertex.addOutEdge(edge); + otherVertex.addInEdge(edge.switchOwner()); } else { - ownerVertex.addInEdge(edge, needDedup); - otherVertex.addOutEdge(edge.switchOwner(), needDedup); + ownerVertex.addInEdge(edge); + otherVertex.addOutEdge(edge.switchOwner()); } return edge; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java index 6188576a06..1d3c672ff8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java @@ -54,6 +54,7 @@ import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.util.CollectionUtil; import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.collection.CollectionFactory; public abstract class HugeElement implements Element, GraphType, Idfiable { @@ -100,7 +101,7 @@ protected void updateToDefaultValueIfNone() { this.defaultValueUpdated = true; // Set default value if needed for (Id pkeyId : this.schemaLabel().properties()) { - if (this.properties.containsKey(pkeyId.asInt())) { + if (this.properties.containsKey(intFromId(pkeyId))) { continue; } PropertyKey pkey = this.graph().propertyKey(pkeyId); @@ -237,12 +238,12 @@ public Map> getAggregateProperties() { @SuppressWarnings("unchecked") public HugeProperty getProperty(Id key) { - return (HugeProperty) this.properties.get(key.asInt()); + return (HugeProperty) this.properties.get(intFromId(key)); } @SuppressWarnings("unchecked") public V getPropertyValue(Id key) { - HugeProperty prop = this.properties.get(key.asInt()); + HugeProperty prop = this.properties.get(intFromId(key)); if (prop == null) { return null; } @@ -250,7 +251,7 @@ public V getPropertyValue(Id key) { } public boolean hasProperty(Id key) { - return this.properties.containsKey(key.asInt()); + return this.properties.containsKey(intFromId(key)); } public boolean hasProperties() { @@ -279,14 +280,15 @@ public HugeProperty setProperty(HugeProperty prop) { this.properties = new IntObjectHashMap<>(); } PropertyKey pkey = prop.propertyKey(); - E.checkArgument(this.properties.containsKey(pkey.id().asInt()) || + + E.checkArgument(this.properties.containsKey(intFromId(pkey.id())) || this.properties.size() < MAX_PROPERTIES, "Exceeded the maximum number of properties"); - return this.properties.put(pkey.id().asInt(), prop); + return this.properties.put(intFromId(pkey.id()), prop); } public HugeProperty removeProperty(Id key) { - return this.properties.remove(key.asInt()); + return this.properties.remove(intFromId(key)); } public HugeProperty addProperty(PropertyKey pkey, V value) { @@ -363,7 +365,7 @@ private HugeProperty addProperty(PropertyKey pkey, V value, } public void resetProperties() { - this.properties = new IntObjectHashMap<>(); + this.properties = CollectionFactory.newIntObjectMap(); this.propLoaded = false; } @@ -371,7 +373,8 @@ protected void copyProperties(HugeElement element) { if (element.properties == EMPTY_MAP) { this.properties = EMPTY_MAP; } else { - this.properties = new IntObjectHashMap<>(element.properties); + this.properties = CollectionFactory.newIntObjectMap( + element.properties); } this.propLoaded = true; } @@ -503,6 +506,12 @@ public static final Object getLabelValue(Object... keyValues) { return labelValue; } + public static int intFromId(Id id) { + E.checkArgument(id instanceof IdGenerator.LongId, + "Can't get number from %s(%s)", id, id.getClass()); + return ((IdGenerator.LongId) id).intValue(); + } + public static final class ElementKeys { private Object label = null; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java index b946f46bbd..2d5877b01f 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java @@ -47,17 +47,18 @@ import com.baidu.hugegraph.backend.serializer.BytesBuffer; import com.baidu.hugegraph.backend.tx.GraphTransaction; import com.baidu.hugegraph.config.CoreOptions; -import com.baidu.hugegraph.perf.PerfUtil; import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.schema.EdgeLabel; import com.baidu.hugegraph.schema.PropertyKey; import com.baidu.hugegraph.schema.VertexLabel; import com.baidu.hugegraph.type.HugeType; import com.baidu.hugegraph.type.define.Cardinality; +import com.baidu.hugegraph.type.define.CollectionImplType; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.type.define.IdStrategy; import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.collection.CollectionFactory; import com.google.common.collect.ImmutableList; public class HugeVertex extends HugeElement implements Vertex, Cloneable { @@ -66,9 +67,7 @@ public class HugeVertex extends HugeElement implements Vertex, Cloneable { private Id id; private VertexLabel label; - // Implemented as LinkedHashMap, is it necessary? Eclipse Collection does - // not have replacement for LinkedHashMap now. - private List edges; + protected Collection edges; @Watched public HugeVertex(final HugeGraph graph, Id id, VertexLabel label) { @@ -221,7 +220,7 @@ public boolean existsEdges() { } public Collection getEdges() { - return Collections.unmodifiableList(this.edges); + return Collections.unmodifiableCollection(this.edges); } public void resetEdges() { @@ -233,16 +232,9 @@ public void removeEdge(HugeEdge edge) { } public void addEdge(HugeEdge edge) { - this.addEdge(edge, true); - } - - public void addEdge(HugeEdge edge, boolean needDedup) { if (this.edges == EMPTY_LIST) { this.edges = new FastList<>(); } - if (needDedup && this.edges.contains(edge)) { - this.edges.remove(edge); - } this.edges.add(edge); } @@ -329,41 +321,34 @@ public HugeEdge constructEdge(String label, HugeVertex vertex, return edge; } - public void addOutEdge(HugeEdge edge) { - this.addOutEdge(edge, true); - } - /** * Add edge with direction OUT * @param edge the out edge */ @Watched - public void addOutEdge(HugeEdge edge, boolean needDedup) { + public void addOutEdge(HugeEdge edge) { if (edge.ownerVertex() == null) { edge.sourceVertex(this); } E.checkState(edge.isDirection(Directions.OUT), "The owner vertex('%s') of OUT edge '%s' should be '%s'", edge.ownerVertex().id(), edge, this.id()); - this.addEdge(edge, needDedup); + this.addEdge(edge); } - public void addInEdge(HugeEdge edge) { - this.addInEdge(edge, true); - } /** * Add edge with direction IN * @param edge the in edge */ @Watched - public void addInEdge(HugeEdge edge, boolean needDedup) { + public void addInEdge(HugeEdge edge) { if (edge.ownerVertex() == null) { edge.targetVertex(this); } E.checkState(edge.isDirection(Directions.IN), "The owner vertex('%s') of IN edge '%s' should be '%s'", edge.ownerVertex().id(), edge, this.id()); - this.addEdge(edge, needDedup); + this.addEdge(edge); } public Iterator getEdges(Directions direction, String... edgeLabels) { @@ -665,10 +650,26 @@ private static final class HugeVertex4Insert extends HugeVertex { public HugeVertex4Insert(final GraphTransaction tx, Id id, VertexLabel label) { super(tx.graph(), id, label); + this.edges = CollectionFactory.newSet(CollectionImplType.EC); this.tx = tx; this.fresh(true); } + public void resetEdges() { + this.edges = CollectionFactory.newSet(CollectionImplType.EC);; + } + + public void removeEdge(HugeEdge edge) { + this.edges.remove(edge); + } + + public void addEdge(HugeEdge edge) { + if (this.edges == EMPTY_LIST) { + this.edges = CollectionFactory.newSet(CollectionImplType.EC); + } + this.edges.add(edge); + } + @Override public void committed() { super.committed(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java index 58d8ef4f60..cab5099a1d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java @@ -61,6 +61,7 @@ import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.InsertionOrderUtil; import com.baidu.hugegraph.util.collection.CollectionFactory; +import com.baidu.hugegraph.util.collection.ObjectIntMapping; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -70,7 +71,7 @@ public class HugeTraverser { private HugeGraph graph; private static CollectionFactory collectionFactory; - protected IdMapping idMapping; + protected ObjectIntMapping idMapping; public static final String DEFAULT_CAPACITY = "10000000"; public static final String DEFAULT_ELEMENTS_LIMIT = "10000000"; @@ -508,12 +509,12 @@ protected static List joinPath(Node prev, Node back, boolean ring) { @Watched protected Id id(int code) { - return this.idMapping.getId(code); + return (Id) this.idMapping.code2Object(code); } @Watched protected int code(Id id) { - return this.idMapping.getCode(id); + return this.idMapping.object2Code(id); } public static class Node { @@ -675,7 +676,7 @@ public boolean equals(Object other) { } } - public static class PathSet { + public static class PathSet implements Set { private static final long serialVersionUID = -8237531948776524872L; @@ -685,14 +686,49 @@ public boolean add(Path path) { return this.paths.add(path); } - public boolean addAll(Collection collection) { - return this.paths.addAll(collection); + @Override + public boolean remove(Object o) { + return this.paths.remove(o); + } + + @Override + public boolean containsAll(Collection c) { + return this.paths.containsAll(c); + } + + @Override + public boolean addAll(Collection c) { + return this.paths.addAll(c); + } + + @Override + public boolean retainAll(Collection c) { + return this.paths.retainAll(c); + } + + @Override + public boolean removeAll(Collection c) { + return this.paths.removeAll(c); + } + + @Override + public void clear() { + this.paths.clear(); + } + + public boolean addAll(PathSet paths) { + return this.paths.addAll(paths.paths); } public boolean isEmpty() { return this.paths.isEmpty(); } + @Override + public boolean contains(Object o) { + return this.paths.contains(o); + } + public int size() { return this.paths.size(); } @@ -705,6 +741,16 @@ public Iterator iterator() { return this.paths.iterator(); } + @Override + public Object[] toArray() { + return this.paths.toArray(); + } + + @Override + public T[] toArray(T[] a) { + return this.paths.toArray(a); + } + public Set vertices() { Set vertices = newIdSet(); for (Path path : this.paths) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java index dbece4c1ef..de935cf5d6 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java @@ -32,9 +32,11 @@ import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeEdge; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.collection.ObjectIntMapping; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -42,9 +44,10 @@ public class PathsTraverser extends HugeTraverser { public PathsTraverser(HugeGraph graph) { super(graph); - this.idMapping = new IdMapping(); + this.idMapping = new ObjectIntMapping(); } + @Watched public PathSet paths(Id sourceV, Directions sourceDir, Id targetV, Directions targetDir, String label, int depth, long degree, long capacity, long limit) { @@ -82,7 +85,7 @@ public PathSet paths(Id sourceV, Directions sourceDir, } traverser.backward(targetDir); } - paths.addAll(traverser.paths().paths()); + paths.addAll(traverser.paths()); return paths; } @@ -124,6 +127,7 @@ public Traverser(Id sourceV, Id targetV, Id label, /** * Search forward from source */ + @Watched public void forward(Directions direction) { // Traversal vertices of previous level assert !this.sourceLayers.isEmpty(); @@ -164,6 +168,7 @@ public void forward(Directions direction) { /** * Search backward from target */ + @Watched public void backward(Directions direction) { assert !this.targetLayers.isEmpty(); IntObjectHashMap targetTopLayer = @@ -227,6 +232,7 @@ private boolean hasLoop(Stack> all, return false; } + @Watched private List getPath(int source, int target, boolean ring) { List results = new ArrayList<>(); List sources = this.getSourcePath(source); @@ -278,16 +284,19 @@ private List getPath(Stack> all, return results; } + @Watched private List getSourcePath(int source) { return this.getPath(this.sourceLayers, source, this.sourceLayers.size() - 1); } + @Watched private List getTargetPath(int target) { return this.getPath(this.targetLayers, target, this.targetLayers.size() - 1); } + @Watched private void add(IntObjectHashMap layer, int current, int parent) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java index da5ae25888..efc0e7b010 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java @@ -39,13 +39,14 @@ import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.collection.ObjectIntMapping; import com.google.common.collect.ImmutableList; public class ShortestPathTraverser extends HugeTraverser { public ShortestPathTraverser(HugeGraph graph) { super(graph); - this.idMapping = new IdMapping(); + this.idMapping = new ObjectIntMapping(); } @Watched @@ -142,10 +143,10 @@ public PathSet allShortestPaths(Id sourceV, Id targetV, Directions dir, private class Traverser { - private Stack sourceLayers; - private Stack targetLayers; + private final Stack sourceLayers; + private final Stack targetLayers; - private LongHashSet accessed = new LongHashSet(); + private final LongHashSet accessed; private final Directions direction; private final Map labels; @@ -170,6 +171,7 @@ public Traverser(Id sourceV, Id targetV, Directions dir, this.sourceLayers.push(firstSourceLayer); this.targetLayers.push(firstTargetLayer); + this.accessed = new LongHashSet(); this.accessed.add(sourceCode); this.accessed.add(targetCode); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/SubGraphTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/SubGraphTraverser.java index 1e613a1c29..87c8032a52 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/SubGraphTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/SubGraphTraverser.java @@ -72,7 +72,7 @@ private PathSet subGraphPaths(Id sourceV, Directions dir, String label, sourceInRing); PathSet paths = new PathSet(); while (true) { - paths.addAll(traverser.forward(dir).paths()); + paths.addAll(traverser.forward(dir)); if (--depth <= 0 || traverser.reachLimit() || traverser.finished()) { break; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/strategy/SingleTraverseStrategy.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/strategy/SingleTraverseStrategy.java index be02f1c336..817d188feb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/strategy/SingleTraverseStrategy.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/strategy/SingleTraverseStrategy.java @@ -55,7 +55,7 @@ public Map> newMultiValueMap() { @Override public Set newPathSet() { - return new HugeTraverser.PathSet().paths(); + return new HugeTraverser.PathSet(); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java index 40380c1f92..f0f89707b1 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java @@ -27,8 +27,11 @@ import java.util.Map; import java.util.Set; +import org.eclipse.collections.api.map.primitive.IntObjectMap; +import org.eclipse.collections.api.map.primitive.MutableIntObjectMap; import org.eclipse.collections.impl.list.mutable.FastList; import org.eclipse.collections.impl.map.mutable.UnifiedMap; +import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; import org.eclipse.collections.impl.set.mutable.UnifiedSet; import com.baidu.hugegraph.backend.id.Id; @@ -199,6 +202,19 @@ public static Map newMap(CollectionImplType type, } } + public static MutableIntObjectMap newIntObjectMap() { + return new IntObjectHashMap<>(); + } + + public static MutableIntObjectMap newIntObjectMap(int initialCapacity) { + return new IntObjectHashMap<>(initialCapacity); + } + + public static MutableIntObjectMap newIntObjectMap( + IntObjectMap map) { + return new IntObjectHashMap<>(map); + } + public Set newIdSet() { return newIdSet(this.type); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java index 472ecda8c0..d1c38f429a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java @@ -19,6 +19,7 @@ package com.baidu.hugegraph.util.collection; +import java.util.AbstractSet; import java.util.HashSet; import java.util.Iterator; import java.util.Set; @@ -30,12 +31,11 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.id.IdGenerator; import com.baidu.hugegraph.iterator.ExtendableIterator; -import com.baidu.hugegraph.iterator.MapperIterator; import com.baidu.hugegraph.type.define.CollectionImplType; import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; -public class IdSet extends HashSet { +public class IdSet extends AbstractSet { private LongHashSet numberIds; private Set nonNumberIds; @@ -79,11 +79,9 @@ public boolean contains(Object o) { @Override public Iterator iterator() { - ExtendableIterator iterator = new ExtendableIterator<>(); - iterator.extend(this.nonNumberIds.iterator()); - EcLongIterator iter = new EcLongIterator(this.numberIds.longIterator()); - iterator.extend(new MapperIterator<>(iter, IdGenerator::of)); - return iterator; + return new ExtendableIterator<>( + this.nonNumberIds.iterator(), + new EcLongIterator(this.numberIds.longIterator())); } @Override @@ -110,9 +108,9 @@ public void clear() { this.nonNumberIds.clear(); } - private static class EcLongIterator implements Iterator { + private static class EcLongIterator implements Iterator { - private MutableLongIterator iterator; + private final MutableLongIterator iterator; public EcLongIterator(MutableLongIterator iter) { this.iterator = iter; @@ -124,8 +122,8 @@ public boolean hasNext() { } @Override - public Long next() { - return this.iterator.next(); + public Id next() { + return IdGenerator.of(this.iterator.next()); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java similarity index 82% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java index 63d96dfc10..8b92cd6924 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/IdMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java @@ -17,25 +17,25 @@ * under the License. */ -package com.baidu.hugegraph.traversal.algorithm; +package com.baidu.hugegraph.util.collection; import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.perf.PerfUtil; +import com.baidu.hugegraph.perf.PerfUtil.Watched; -public class IdMapping { +public final class ObjectIntMapping { private static final int MAGIC = 1 << 16; - private IntObjectHashMap int2IdMap; + private final IntObjectHashMap int2IdMap; - public IdMapping() { + public ObjectIntMapping() { this.int2IdMap = new IntObjectHashMap(); } - @PerfUtil.Watched - public int getCode(Id id) { + @Watched + public int object2Code(Object id) { int key = id.hashCode(); for (int i = 1; i > 0; i <<= 1) { for (int j = 0; i >= MAGIC && j < 10; j++) { @@ -53,8 +53,8 @@ public int getCode(Id id) { throw new HugeException("Failed to get code for id: %s", id); } - @PerfUtil.Watched - public Id getId(int code) { - return (Id) this.int2IdMap.get(code); + @Watched + public Object code2Object(int code) { + return this.int2IdMap.get(code); } } diff --git a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java index b8c940b756..0bd89f5f62 100644 --- a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java +++ b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java @@ -89,18 +89,18 @@ public void stop() { } } - public static void main(String[] args) throws Exception { + public static void main(String[] args) throws Throwable { if (args.length != 2) { String msg = "HugeGraphServer can only accept two config files"; LOG.error(msg); throw new HugeException(msg); } - //PerfUtil.instance().profilePackage("com.baidu.hugegraph."); - //PerfUtil.instance().profileClass("com.baidu.hugegraph.traversal" + - // "algorithm.ShortestPathTraverser$Traverser"); - //PerfUtil.instance().profileClass("com.baidu.hugegraph.backend.cache." + - // "CachedSchemaTransaction$SchemaCaches"); + //PerfUtil.instance().profilePackage("com.baidu.hugegraph"); + PerfUtil.instance().profilePackage("com.baidu.hugegraph.traversal.algorithm"); + PerfUtil.instance().profilePackage("com.baidu.hugegraph.util.collection"); + PerfUtil.instance().profileClass("com.baidu.hugegraph.structure.HugeEdge"); + PerfUtil.useLocalTimer(true); HugeRestServer.register(); diff --git a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/ExampleUtil.java b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/ExampleUtil.java index 3c63d1a2dd..35177d017e 100644 --- a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/ExampleUtil.java +++ b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/ExampleUtil.java @@ -87,8 +87,8 @@ public static HugeGraph loadGraph(boolean needClear, boolean needProfile) { public static void profile() { try { PerfUtil.instance().profilePackage("com.baidu.hugegraph"); - } catch (Exception e) { - throw new RuntimeException(e); + } catch (Throwable t) { + throw new RuntimeException(t); } } From eaa53122ecae2241519c436d130557176a6f0c80 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Fri, 9 Apr 2021 15:55:42 +0800 Subject: [PATCH 14/38] extract shortestpath and paths record to hold traversed paths Change-Id: I32098ca82c238105d1c3984a4ebf5d5c3d268fac --- .../hugegraph/api/traversers/PathsAPI.java | 10 - .../baidu/hugegraph/structure/HugeEdge.java | 1 - .../baidu/hugegraph/structure/HugeVertex.java | 12 +- .../traversal/algorithm/PathsTraverser.java | 220 +++--------------- .../algorithm/ShortestPathTraverser.java | 190 ++++----------- .../algorithm/records/PathsRecord.java | 214 +++++++++++++++++ .../traversal/algorithm/records/Record.java | 41 ++++ .../algorithm/records/ShortestPathRecord.java | 190 +++++++++++++++ .../util/collection/ObjectIntMapping.java | 10 +- .../baidu/hugegraph/dist/HugeGraphServer.java | 7 - 10 files changed, 541 insertions(+), 354 deletions(-) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecord.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java index 8093c9bdd5..e98e1384ec 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java @@ -48,7 +48,6 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.query.QueryResults; import com.baidu.hugegraph.core.GraphManager; -import com.baidu.hugegraph.perf.PerfUtil; import com.baidu.hugegraph.server.RestServer; import com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser; import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; @@ -88,10 +87,6 @@ public String get(@Context GraphManager manager, graph, source, target, direction, edgeLabel, depth, maxDegree, capacity, limit); - PerfUtil.instance().clear(); - PerfUtil.instance().start("paths-get"); - try { - Id sourceId = VertexAPI.checkAndParseVertexId(source); Id targetId = VertexAPI.checkAndParseVertexId(target); Directions dir = Directions.convert(EdgeAPI.parseDirection(direction)); @@ -104,11 +99,6 @@ public String get(@Context GraphManager manager, limit); return manager.serializer(g).writePaths("paths", paths.paths(), false); - - } finally { - PerfUtil.instance().end("paths-get"); - LOG.info("option = {}", PerfUtil.instance().toECharts()); - } } @POST diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java index 28cfe3142a..25034d3164 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java @@ -58,7 +58,6 @@ public class HugeEdge extends HugeElement implements Edge, Cloneable { private HugeVertex targetVertex; private boolean isOutEdge; - @Watched public HugeEdge(HugeVertex owner, Id id, EdgeLabel label, HugeVertex other) { this(owner.graph(), id, label); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java index 2d5877b01f..f22c1c4a96 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java @@ -25,6 +25,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Set; import org.apache.commons.collections.CollectionUtils; import org.apache.tinkerpop.gremlin.structure.Direction; @@ -69,7 +70,6 @@ public class HugeVertex extends HugeElement implements Vertex, Cloneable { private VertexLabel label; protected Collection edges; - @Watched public HugeVertex(final HugeGraph graph, Id id, VertexLabel label) { super(graph); @@ -650,13 +650,13 @@ private static final class HugeVertex4Insert extends HugeVertex { public HugeVertex4Insert(final GraphTransaction tx, Id id, VertexLabel label) { super(tx.graph(), id, label); - this.edges = CollectionFactory.newSet(CollectionImplType.EC); + this.edges = newSet(); this.tx = tx; this.fresh(true); } public void resetEdges() { - this.edges = CollectionFactory.newSet(CollectionImplType.EC);; + this.edges = newSet(); } public void removeEdge(HugeEdge edge) { @@ -665,7 +665,7 @@ public void removeEdge(HugeEdge edge) { public void addEdge(HugeEdge edge) { if (this.edges == EMPTY_LIST) { - this.edges = CollectionFactory.newSet(CollectionImplType.EC); + this.edges = newSet(); } this.edges.add(edge); } @@ -684,5 +684,9 @@ protected GraphTransaction tx() { } return null; } + + private static Set newSet() { + return CollectionFactory.newSet(CollectionImplType.EC); + } } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java index de935cf5d6..d2db9efcd8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java @@ -19,32 +19,23 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; -import java.util.Stack; -import org.apache.commons.collections.CollectionUtils; import org.apache.tinkerpop.gremlin.structure.Edge; -import org.eclipse.collections.api.iterator.IntIterator; -import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; -import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeEdge; +import com.baidu.hugegraph.traversal.algorithm.records.PathsRecord; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; -import com.baidu.hugegraph.util.collection.ObjectIntMapping; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; public class PathsTraverser extends HugeTraverser { public PathsTraverser(HugeGraph graph) { super(graph); - this.idMapping = new ObjectIntMapping(); } @Watched @@ -91,37 +82,25 @@ public PathSet paths(Id sourceV, Directions sourceDir, private class Traverser { - private Stack> sourceLayers; - private Stack> targetLayers; + private final PathsRecord record; private final Id label; private final long degree; private final long capacity; private final long limit; + private PathSet paths; - private int accessed; public Traverser(Id sourceV, Id targetV, Id label, long degree, long capacity, long limit) { - this.accessed = 2; + this.record = new PathsRecord(sourceV, targetV); + this.label = label; this.degree = degree; this.capacity = capacity; this.limit = limit; - this.paths = new PathSet(); - int sourceCode = this.code(sourceV); - int targetCode = this.code(targetV); - IntObjectHashMap firstSourceLayer = - new IntObjectHashMap<>(); - IntObjectHashMap firstTargetLayer = - new IntObjectHashMap<>(); - firstSourceLayer.put(sourceCode, new IntHashSet()); - firstTargetLayer.put(targetCode, new IntHashSet()); - this.sourceLayers = new Stack<>(); - this.targetLayers = new Stack<>(); - this.sourceLayers.push(firstSourceLayer); - this.targetLayers.push(firstTargetLayer); + this.paths = new PathSet(); } /** @@ -129,40 +108,29 @@ public Traverser(Id sourceV, Id targetV, Id label, */ @Watched public void forward(Directions direction) { - // Traversal vertices of previous level - assert !this.sourceLayers.isEmpty(); - IntObjectHashMap sourceTopLayer = - this.sourceLayers.peek(); - IntObjectHashMap newSourceLayer = - new IntObjectHashMap<>(); Iterator edges; - IntIterator keys = sourceTopLayer.keySet().intIterator(); - while (keys.hasNext()) { - int id = keys.next(); - Id vid = this.id(id); + + this.record.startOneLayer(true); + while (this.record.hasNext()) { + Id vid = this.record.next(); + edges = edgesOfVertex(vid, direction, this.label, this.degree); while (edges.hasNext()) { HugeEdge edge = (HugeEdge) edges.next(); - Id targetId = edge.id().otherVertexId(); - int target = this.code(targetId); - - // If cross point exists, path found, concat them - if (this.targetLayers.peek().containsKey(target)) { - List paths = this.getPath(id, target, false); - for (Path path : paths) { - this.paths.add(path); - if (this.reachLimit()) { - return; - } + Id target = edge.id().otherVertexId(); + + PathSet results = this.record.findPath(target, null, + true, false); + for (Path path : results) { + this.paths.add(path); + if (this.reachLimit()) { + return; } } - this.add(newSourceLayer, target, id); } - } - this.accessed += newSourceLayer.size(); - this.sourceLayers.push(newSourceLayer); + this.record.finishOneLayer(); } /** @@ -170,141 +138,29 @@ public void forward(Directions direction) { */ @Watched public void backward(Directions direction) { - assert !this.targetLayers.isEmpty(); - IntObjectHashMap targetTopLayer = - this.targetLayers.peek(); - IntObjectHashMap newTargetLayer = - new IntObjectHashMap<>(); Iterator edges; - IntIterator keys = targetTopLayer.keySet().intIterator(); - while (keys.hasNext()) { - int id = keys.next(); - Id vid = this.id(id); + + this.record.startOneLayer(false); + while (this.record.hasNext()) { + Id vid = this.record.next(); edges = edgesOfVertex(vid, direction, this.label, this.degree); while (edges.hasNext()) { HugeEdge edge = (HugeEdge) edges.next(); - Id targetId = edge.id().otherVertexId(); - int target = this.code(targetId); - - // If cross point exists, path found, concat them - if (this.sourceLayers.peek().containsKey(target)) { - List paths = this.getPath(target, id, false); - for (Path path : paths) { - this.paths.add(path); - if (this.reachLimit()) { - return; - } - } - } - - this.add(newTargetLayer, target, id); - } - } - - // Re-init targets - this.accessed += newTargetLayer.size(); - this.targetLayers.push(newTargetLayer); - } - - private boolean hasLoop(Stack> all, - int current, int target) { - - if (current == target) { - return true; - } - int layers = all.size(); - IntHashSet keys = IntHashSet.newSetWith(current); - for (int i = layers - 1; i > 0 ; i--) { - IntObjectHashMap currentLayer = - this.sourceLayers.elementAt(i); - IntIterator iterator = keys.intIterator(); - IntHashSet parents = null; - while (iterator.hasNext()) { - int key = iterator.next(); - parents = currentLayer.get(key); - if (!parents.isEmpty() && parents.contains(target)) { - return true; - } - } - keys = parents; - } - return false; - } - - @Watched - private List getPath(int source, int target, boolean ring) { - List results = new ArrayList<>(); - List sources = this.getSourcePath(source); - List targets = this.getTargetPath(target); - for (Path tpath : targets) { - tpath.reverse(); - for (Path spath : sources) { - if (!ring) { - // Avoid loop in path - if (CollectionUtils.containsAny(spath.vertices(), - tpath.vertices())) { - continue; + Id target = edge.id().otherVertexId(); + + PathSet results = this.record.findPath(target, null, + true, false); + for (Path path : results) { + this.paths.add(path); + if (this.reachLimit()) { + return; } } - List ids = new ArrayList<>(spath.vertices()); - ids.addAll(tpath.vertices()); - results.add(new Path(ids)); - } - } - return results; - } - - private List getPath(Stack> all, - int id, int layerIndex) { - if (layerIndex == 0) { - Id sid = this.id(id); - return ImmutableList.of(new Path(Lists.newArrayList(sid))); - } - - Id sid = this.id(id); - List results = new ArrayList<>(); - IntObjectHashMap layer = all.elementAt(layerIndex); - IntHashSet parents = layer.get(id); - IntIterator iterator = parents.intIterator(); - while (iterator.hasNext()) { - int parent = iterator.next(); - List paths = this.getPath(all, parent, layerIndex - 1); - for (Iterator iter = paths.iterator(); iter.hasNext();) { - Path path = iter.next(); - if (path.vertices().contains(sid)) { - iter.remove(); - continue; - } - path.addToEnd(sid); } - - results.addAll(paths); } - return results; - } - - @Watched - private List getSourcePath(int source) { - return this.getPath(this.sourceLayers, source, - this.sourceLayers.size() - 1); - } - - @Watched - private List getTargetPath(int target) { - return this.getPath(this.targetLayers, target, - this.targetLayers.size() - 1); - } - - @Watched - private void add(IntObjectHashMap layer, - int current, int parent) { - if (layer.containsKey(current)) { - layer.get(current).add(parent); - } else { - layer.put(current, IntHashSet.newSetWith(parent)); - } + this.record.finishOneLayer(); } public PathSet paths() { @@ -312,19 +168,11 @@ public PathSet paths() { } private boolean reachLimit() { - checkCapacity(this.capacity, this.accessed, "paths"); + checkCapacity(this.capacity, this.record.accessed(), "paths"); if (this.limit == NO_LIMIT || this.paths.size() < this.limit) { return false; } return true; } - - private int code(Id id) { - return PathsTraverser.this.code(id); - } - - private Id id(int code) { - return PathsTraverser.this.id(code); - } } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java index efc0e7b010..b2a6a3e2f4 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java @@ -19,23 +19,18 @@ package com.baidu.hugegraph.traversal.algorithm; -import java.util.ArrayList; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Stack; import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; -import org.eclipse.collections.api.iterator.IntIterator; -import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; -import org.eclipse.collections.impl.set.mutable.primitive.LongHashSet; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeEdge; +import com.baidu.hugegraph.traversal.algorithm.records.ShortestPathRecord; import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; @@ -80,13 +75,15 @@ public Path shortestPath(Id sourceV, Id targetV, Directions dir, --depth <= 0) { break; } - checkCapacity(traverser.capacity, traverser.size, "shortest path"); + checkCapacity(traverser.capacity, traverser.accessed(), + "shortest path"); if (!(paths = traverser.backward(false)).isEmpty() || --depth <= 0) { break; } - checkCapacity(traverser.capacity, traverser.size, "shortest path"); + checkCapacity(traverser.capacity, traverser.accessed(), + "shortest path"); } return paths.isEmpty() ? Path.EMPTY_PATH : paths.iterator().next(); } @@ -130,57 +127,37 @@ public PathSet allShortestPaths(Id sourceV, Id targetV, Directions dir, --depth <= 0) { break; } - checkCapacity(traverser.capacity, traverser.size, "shortest path"); + checkCapacity(traverser.capacity, traverser.accessed(), + "shortest path"); if (!(paths = traverser.backward(true)).isEmpty() || --depth <= 0) { break; } - checkCapacity(traverser.capacity, traverser.size, "shortest path"); + checkCapacity(traverser.capacity, traverser.accessed(), + "shortest path"); } return paths; } private class Traverser { - private final Stack sourceLayers; - private final Stack targetLayers; - - private final LongHashSet accessed; - + private ShortestPathRecord record; private final Directions direction; private final Map labels; private final long degree; private final long skipDegree; private final long capacity; - private long size; - - private boolean foundPath = false; public Traverser(Id sourceV, Id targetV, Directions dir, Map labels, long degree, long skipDegree, long capacity) { - int sourceCode = this.code(sourceV); - int targetCode = this.code(targetV); - IntIntHashMap firstSourceLayer = new IntIntHashMap(); - IntIntHashMap firstTargetLayer = new IntIntHashMap(); - firstSourceLayer.put(sourceCode, 0); - firstTargetLayer.put(targetCode, 0); - this.sourceLayers = new Stack<>(); - this.targetLayers = new Stack<>(); - this.sourceLayers.push(firstSourceLayer); - this.targetLayers.push(firstTargetLayer); - - this.accessed = new LongHashSet(); - this.accessed.add(sourceCode); - this.accessed.add(targetCode); - + this.record = new ShortestPathRecord(sourceV, targetV); this.direction = dir; this.labels = labels; this.degree = degree; this.skipDegree = skipDegree; this.capacity = capacity; - this.size = 0L; } /** @@ -188,59 +165,38 @@ public Traverser(Id sourceV, Id targetV, Directions dir, */ @Watched public PathSet forward(boolean all) { - PathSet paths = new PathSet(); - IntIntHashMap newLayer = new IntIntHashMap(); + PathSet results = new PathSet(); long degree = this.skipDegree > 0L ? this.skipDegree : this.degree; - assert !this.sourceLayers.isEmpty(); - assert !this.targetLayers.isEmpty(); - IntIntHashMap sourceTopLayer = this.sourceLayers.peek(); - IntIntHashMap targetTopLayer = this.targetLayers.peek(); + this.record.startOneLayer(true); + while (this.record.hasNext()) { + Id source = this.record.next(); - IntIterator iterator = sourceTopLayer.keySet().intIterator(); - while (iterator.hasNext()) { - int sourceCode = iterator.next(); - Iterator edges = edgesOfVertex(this.id(sourceCode), - this.direction, + Iterator edges = edgesOfVertex(source, this.direction, this.labels, degree); edges = skipSuperNodeIfNeeded(edges, this.degree, this.skipDegree); while (edges.hasNext()) { HugeEdge edge = (HugeEdge) edges.next(); Id target = edge.id().otherVertexId(); - int targetCode = this.code(target); - - // If cross point exists, shortest path found, concat them - if (targetTopLayer.containsKey(targetCode)) { - if (this.superNode(target, this.direction)) { - continue; - } - - paths.add(this.getPath(sourceCode, targetCode)); - this.foundPath = true; - if (!all) { - return paths; - } - } - /* - * Not found shortest path yet, node is added to - * newVertices if: - * 1. not in sources and newVertices yet - * 2. path of node doesn't have loop - */ - if (!this.foundPath && !newLayer.containsKey(targetCode) && - !this.accessed.contains(this.code(target))) { - newLayer.put(targetCode, sourceCode); + PathSet paths = this.record.findPath(target, + t -> this.superNode(t, this.direction), + all, false); + + if (paths.isEmpty()) { + continue; + } + results.addAll(paths); + if (!all) { + return paths; } } } - // Re-init sources - this.sourceLayers.push(newLayer); - this.size += newLayer.size(); + this.record.finishOneLayer(); - return paths; + return results; } /** @@ -248,60 +204,40 @@ public PathSet forward(boolean all) { */ @Watched public PathSet backward(boolean all) { - PathSet paths = new PathSet(); - IntIntHashMap newLayer = new IntIntHashMap(); + PathSet results = new PathSet(); long degree = this.skipDegree > 0L ? this.skipDegree : this.degree; - - assert !this.sourceLayers.isEmpty(); - assert !this.targetLayers.isEmpty(); - IntIntHashMap sourceTopLayer = this.targetLayers.peek(); - IntIntHashMap targetTopLayer = this.sourceLayers.peek(); - Directions opposite = this.direction.opposite(); - IntIterator iterator = sourceTopLayer.keySet().intIterator(); - while (iterator.hasNext()) { - int sourceCode = iterator.next(); - Iterator edges = edgesOfVertex(this.id(sourceCode), - opposite, this.labels, - degree); + this.record.startOneLayer(false); + while (this.record.hasNext()) { + Id source = this.record.next(); + + Iterator edges = edgesOfVertex(source, opposite, + this.labels, degree); edges = skipSuperNodeIfNeeded(edges, this.degree, this.skipDegree); while (edges.hasNext()) { HugeEdge edge = (HugeEdge) edges.next(); Id target = edge.id().otherVertexId(); - int targetCode = this.code(target); - - // If cross point exists, shortest path found, concat them - if (targetTopLayer.containsKey(targetCode)) { - if (this.superNode(target, opposite)) { - continue; - } - paths.add(this.getPath(targetCode, sourceCode)); - this.foundPath = true; - if (!all) { - return paths; - } - } - /* - * Not found shortest path yet, node is added to - * newVertices if: - * 1. not in targets and newVertices yet - * 2. path of node doesn't have loop - */ - if (!this.foundPath && !newLayer.containsKey(targetCode) && - !this.accessed.contains(targetCode)) { - newLayer.put(targetCode, sourceCode); + PathSet paths = this.record.findPath(target, + t -> this.superNode(t, opposite), + all, false); + + if (paths.isEmpty()) { + continue; + } + results.addAll(paths); + if (!all) { + return results; } } } // Re-init targets - this.targetLayers.push(newLayer); - this.size += newLayer.size(); + this.record.finishOneLayer(); - return paths; + return results; } private boolean superNode(Id vertex, Directions direction) { @@ -313,36 +249,8 @@ private boolean superNode(Id vertex, Directions direction) { return IteratorUtils.count(edges) >= this.skipDegree; } - private Path getPath(int source, int target) { - int sourceLayerSize = this.sourceLayers.size(); - int targetLayerSize = this.targetLayers.size(); - - List ids = new ArrayList<>(sourceLayerSize + targetLayerSize); - - ids.add(this.id(source)); - int value = source; - for (int i = sourceLayerSize - 1; i > 0 ; i--) { - IntIntHashMap layer = this.sourceLayers.elementAt(i); - value = layer.get(value); - ids.add(this.id(value)); - } - Collections.reverse(ids); - ids.add(this.id(target)); - value = target; - for (int i = this.targetLayers.size() - 1; i > 0 ; i--) { - IntIntHashMap layer = this.targetLayers.elementAt(i); - value = layer.get(value); - ids.add(this.id(value)); - } - return new Path(ids); - } - - private int code(Id id) { - return ShortestPathTraverser.this.code(id); - } - - private Id id(int code) { - return ShortestPathTraverser.this.id(code); + private long accessed() { + return this.record.accessed(); } } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecord.java new file mode 100644 index 0000000000..3cdfe889f8 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecord.java @@ -0,0 +1,214 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Stack; +import java.util.function.Function; + +import org.apache.commons.collections.CollectionUtils; +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; +import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.perf.PerfUtil.Watched; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; +import com.baidu.hugegraph.util.collection.ObjectIntMapping; +import com.google.common.collect.Lists; + +public class PathsRecord implements Record { + + private ObjectIntMapping idMapping; + + private final Stack> sourceLayers; + private final Stack> targetLayers; + + private IntObjectHashMap currentLayer; + private IntIterator iterator; + private int current; + private boolean forward; + + private int accessed; + + public PathsRecord(Id sourceV, Id targetV) { + + this.idMapping = new ObjectIntMapping(); + + int sourceCode = this.code(sourceV); + int targetCode = this.code(targetV); + IntObjectHashMap firstSourceLayer = new IntObjectHashMap<>(); + IntObjectHashMap firstTargetLayer = new IntObjectHashMap<>(); + firstSourceLayer.put(sourceCode, new IntHashSet()); + firstTargetLayer.put(targetCode, new IntHashSet()); + this.sourceLayers = new Stack<>(); + this.targetLayers = new Stack<>(); + this.sourceLayers.push(firstSourceLayer); + this.targetLayers.push(firstTargetLayer); + + this.accessed = 2; + } + + public void startOneLayer(boolean forward) { + this.forward = forward; + this.currentLayer = new IntObjectHashMap<>(); + this.iterator = this.forward ? + this.sourceLayers.peek().keySet().intIterator() : + this.targetLayers.peek().keySet().intIterator(); + } + + public void finishOneLayer() { + if (this.forward) { + this.sourceLayers.push(this.currentLayer); + } else { + this.targetLayers.push(this.currentLayer); + } + this.accessed += this.currentLayer.size(); + } + + @Watched + public boolean hasNext() { + return this.iterator.hasNext(); + } + + @Watched + public Id next() { + this.current = this.iterator.next(); + return this.id(current); + } + + @Watched + public PathSet findPath(Id target, Function filter, + boolean all, boolean ring) { + assert all; + PathSet results = new PathSet(); + int targetCode = this.code(target); + // If cross point exists, path found, concat them + if (this.contains(targetCode)) { + results = this.forward ? + this.getPath(this.current, targetCode, ring) : + this.getPath(targetCode, this.current, ring); + } + this.add(targetCode, this.current); + return results; + } + + public long accessed() { + return this.accessed; + } + + private boolean contains(int node) { + return this.forward ? this.targetContains(node) : + this.sourceContains(node); + } + + private boolean sourceContains(int node) { + return this.sourceLayers.peek().containsKey(node); + } + + private boolean targetContains(int node) { + return this.targetLayers.peek().containsKey(node); + } + + @Watched + private PathSet getPath(int source, int target, boolean ring) { + PathSet results = new PathSet(); + PathSet sources = this.getSourcePath(source); + PathSet targets = this.getTargetPath(target); + for (Path tpath : targets) { + tpath.reverse(); + for (Path spath : sources) { + if (!ring) { + // Avoid loop in path + if (CollectionUtils.containsAny(spath.vertices(), + tpath.vertices())) { + continue; + } + } + List ids = new ArrayList<>(spath.vertices()); + ids.addAll(tpath.vertices()); + results.add(new Path(ids)); + } + } + return results; + } + + private PathSet getPath(Stack> all, + int id, int layerIndex) { + PathSet results = new PathSet(); + if (layerIndex == 0) { + Id sid = this.id(id); + results.add(new Path(Lists.newArrayList(sid))); + return results; + } + + Id sid = this.id(id); + IntObjectHashMap layer = all.elementAt(layerIndex); + IntHashSet parents = layer.get(id); + IntIterator iterator = parents.intIterator(); + while (iterator.hasNext()) { + int parent = iterator.next(); + PathSet paths = this.getPath(all, parent, layerIndex - 1); + for (Iterator iter = paths.iterator(); iter.hasNext();) { + Path path = iter.next(); + if (path.vertices().contains(sid)) { + iter.remove(); + continue; + } + path.addToEnd(sid); + } + + results.addAll(paths); + } + return results; + } + + private PathSet getSourcePath(int source) { + return this.getPath(this.sourceLayers, source, + this.sourceLayers.size() - 1); + } + + private PathSet getTargetPath(int target) { + return this.getPath(this.targetLayers, target, + this.targetLayers.size() - 1); + } + + @Watched + private void add(int current, int parent) { + if (this.currentLayer.containsKey(current)) { + this.currentLayer.get(current).add(parent); + } else { + this.currentLayer.put(current, IntHashSet.newSetWith(parent)); + } + } + + @Watched + private int code(Id id) { + return this.idMapping.object2Code(id); + } + + @Watched + private Id id(int code) { + return (Id) this.idMapping.code2Object(code); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java new file mode 100644 index 0000000000..f26edb738f --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java @@ -0,0 +1,41 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import java.util.function.Function; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; + +public interface Record { + + public void startOneLayer(boolean forward); + + public void finishOneLayer(); + + public boolean hasNext(); + + public Id next(); + + public PathSet findPath(Id target, Function filter, + boolean all, boolean ring); + + public long accessed(); +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java new file mode 100644 index 0000000000..16c129f943 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java @@ -0,0 +1,190 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Stack; +import java.util.function.Function; + +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; +import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; +import com.baidu.hugegraph.util.collection.ObjectIntMapping; + +public class ShortestPathRecord implements Record { + + private ObjectIntMapping idMapping; + + private final Stack sourceLayers; + private final Stack targetLayers; + + private IntIntHashMap currentLayer; + private IntIterator iterator; + private int current; + private boolean forward; + + private final IntHashSet accessed; + private long size; + private boolean foundPath; + + public ShortestPathRecord(Id sourceV, Id targetV) { + + this.idMapping = new ObjectIntMapping(); + + int sourceCode = this.code(sourceV); + int targetCode = this.code(targetV); + IntIntHashMap firstSourceLayer = new IntIntHashMap(); + IntIntHashMap firstTargetLayer = new IntIntHashMap(); + firstSourceLayer.put(sourceCode, 0); + firstTargetLayer.put(targetCode, 0); + this.sourceLayers = new Stack<>(); + this.targetLayers = new Stack<>(); + this.sourceLayers.push(firstSourceLayer); + this.targetLayers.push(firstTargetLayer); + + this.accessed = new IntHashSet(); + this.accessed.add(sourceCode); + this.accessed.add(targetCode); + + this.size = 2L; + } + + public void startOneLayer(boolean forward) { + this.forward = forward; + this.currentLayer = new IntIntHashMap(); + this.iterator = this.forward ? + this.sourceLayers.peek().keySet().intIterator() : + this.targetLayers.peek().keySet().intIterator(); + } + + public void finishOneLayer() { + if (this.forward) { + this.sourceLayers.push(this.currentLayer); + } else { + this.targetLayers.push(this.currentLayer); + } + this.size += this.currentLayer.size(); + } + + public boolean hasNext() { + return this.iterator.hasNext(); + } + + public Id next() { + this.current = this.iterator.next(); + return this.id(current); + } + + public PathSet findPath(Id target, Function filter, + boolean all, boolean ring) { + assert !ring; + PathSet paths = new PathSet(); + int targetCode = this.code(target); + // If cross point exists, shortest path found, concat them + if (this.contains(targetCode)) { + if (filter.apply(target)) { + return paths; + } + + Path path = this.forward ? this.getPath(this.current, targetCode) : + this.getPath(targetCode, this.current); + this.foundPath = true; + if (!all) { + paths.add(path); + return paths; + } + } + + /* + * Not found shortest path yet, node is added to + * newVertices if: + * 1. not in sources and newVertices yet + * 2. path of node doesn't have loop + */ + if (!this.foundPath && this.isNew(targetCode)) { + this.addOneStep(this.current, targetCode); + } + return paths; + } + + public long accessed() { + return this.size; + } + + private boolean contains(int node) { + return this.forward ? this.targetContains(node) : + this.sourceContains(node); + } + + private boolean sourceContains(int node) { + return this.sourceLayers.peek().containsKey(node); + } + + private boolean targetContains(int node) { + return this.targetLayers.peek().containsKey(node); + } + + private boolean isNew(int node) { + return !this.currentLayer.containsKey(node) && + !this.accessed.contains(node); + } + + private void addOneStep(int source, int target) { + this.currentLayer.put(target, source); + } + + private Path getPath(int source, int target) { + int sourceLayerSize = this.sourceLayers.size(); + int targetLayerSize = this.targetLayers.size(); + + List ids = new ArrayList<>(sourceLayerSize + targetLayerSize); + + ids.add(this.id(source)); + int value = source; + for (int i = sourceLayerSize - 1; i > 0 ; i--) { + IntIntHashMap layer = this.sourceLayers.elementAt(i); + value = layer.get(value); + ids.add(this.id(value)); + } + Collections.reverse(ids); + ids.add(this.id(target)); + value = target; + for (int i = this.targetLayers.size() - 1; i > 0 ; i--) { + IntIntHashMap layer = this.targetLayers.elementAt(i); + value = layer.get(value); + ids.add(this.id(value)); + } + return new Path(ids); + } + + private int code(Id id) { + return this.idMapping.object2Code(id); + } + + private Id id(int code) { + return (Id) this.idMapping.code2Object(code); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java index 8b92cd6924..f6a081c0a9 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java @@ -35,22 +35,22 @@ public ObjectIntMapping() { } @Watched - public int object2Code(Object id) { - int key = id.hashCode(); + public int object2Code(Object object) { + int key = object.hashCode(); for (int i = 1; i > 0; i <<= 1) { for (int j = 0; i >= MAGIC && j < 10; j++) { Id existed = (Id) this.int2IdMap.get(key); if (existed == null) { - this.int2IdMap.put(key, id); + this.int2IdMap.put(key, object); return key; } - if (existed.equals(id)) { + if (existed.equals(object)) { return key; } key = key + i + j; } } - throw new HugeException("Failed to get code for id: %s", id); + throw new HugeException("Failed to get code for id: %s", object); } @Watched diff --git a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java index 0bd89f5f62..2dfdccea54 100644 --- a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java +++ b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java @@ -19,7 +19,6 @@ package com.baidu.hugegraph.dist; -import com.baidu.hugegraph.perf.PerfUtil; import org.apache.tinkerpop.gremlin.server.GremlinServer; import org.slf4j.Logger; @@ -96,12 +95,6 @@ public static void main(String[] args) throws Throwable { throw new HugeException(msg); } - //PerfUtil.instance().profilePackage("com.baidu.hugegraph"); - PerfUtil.instance().profilePackage("com.baidu.hugegraph.traversal.algorithm"); - PerfUtil.instance().profilePackage("com.baidu.hugegraph.util.collection"); - PerfUtil.instance().profileClass("com.baidu.hugegraph.structure.HugeEdge"); - PerfUtil.useLocalTimer(true); - HugeRestServer.register(); HugeGraphServer server = new HugeGraphServer(args[0], args[1]); From e2c75fc9ec5fa7fe992338ddf0f5af151b2d048d Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Fri, 9 Apr 2021 17:30:56 +0800 Subject: [PATCH 15/38] improve Change-Id: I85bb9cb0835421d12b37827d870d968788f805f6 --- .../hugegraph/api/traversers/AllShortestPathsAPI.java | 4 ++-- .../com/baidu/hugegraph/api/traversers/PathsAPI.java | 10 +++++----- .../hugegraph/api/traversers/PersonalRankAPI.java | 10 +++++----- .../hugegraph/api/traversers/ShortestPathAPI.java | 11 ++++++++--- .../baidu/hugegraph/api/traversers/TraverserAPI.java | 4 ++-- .../java/com/baidu/hugegraph/structure/HugeEdge.java | 1 - 6 files changed, 22 insertions(+), 18 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java index d26aef7048..b6de8187f1 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java @@ -19,11 +19,11 @@ package com.baidu.hugegraph.api.traversers; -import java.util.List; - import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE; +import java.util.List; + import javax.inject.Singleton; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java index e98e1384ec..9a488c304a 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java @@ -19,15 +19,15 @@ package com.baidu.hugegraph.api.traversers; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT; + import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.Set; -import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY; -import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE; -import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT; - import javax.inject.Singleton; import javax.ws.rs.Consumes; import javax.ws.rs.DefaultValue; @@ -50,9 +50,9 @@ import com.baidu.hugegraph.core.GraphManager; import com.baidu.hugegraph.server.RestServer; import com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser; -import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser; import com.baidu.hugegraph.traversal.algorithm.PathsTraverser; +import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.Log; diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java index 563d08ca5a..5ec50e382a 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java @@ -19,6 +19,11 @@ package com.baidu.hugegraph.api.traversers; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_LIMIT; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEPTH; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.NO_LIMIT; + import java.util.Map; import javax.inject.Singleton; @@ -43,11 +48,6 @@ import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.annotation.JsonProperty; -import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE; -import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_LIMIT; -import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEPTH; -import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.NO_LIMIT; - @Path("graphs/{graph}/traversers/personalrank") @Singleton public class PersonalRankAPI extends API { diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java index 98f1c7ea41..87e2f7cd60 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java @@ -19,13 +19,18 @@ package com.baidu.hugegraph.api.traversers; -import java.util.List; - import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE; +import java.util.List; + import javax.inject.Singleton; -import javax.ws.rs.*; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import org.slf4j.Logger; diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/TraverserAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/TraverserAPI.java index cab71015af..62c4cc46ac 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/TraverserAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/TraverserAPI.java @@ -19,6 +19,8 @@ package com.baidu.hugegraph.api.traversers; +import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE; + import java.util.List; import java.util.Map; @@ -29,8 +31,6 @@ import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonProperty; -import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE; - public class TraverserAPI extends API { protected static EdgeStep step(HugeGraph graph, Step step) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java index 25034d3164..f579fc9b13 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java @@ -65,7 +65,6 @@ public HugeEdge(HugeVertex owner, Id id, EdgeLabel label, this.vertices(owner, other); } - @Watched public HugeEdge(final HugeGraph graph, Id id, EdgeLabel label) { super(graph); From 564620d4a1bb76f79acf0ae4bbe50655b8e72113 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Fri, 9 Apr 2021 19:32:41 +0800 Subject: [PATCH 16/38] adapt edgecoretest Change-Id: Ife2c1aa0462c17679fcfcbe5660df1f499c13702 --- .../src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java index b51f49344b..d0aef334aa 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java @@ -21,7 +21,6 @@ import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; @@ -77,6 +76,7 @@ import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.util.Events; +import com.baidu.hugegraph.util.collection.CollectionFactory; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -3090,7 +3090,8 @@ public void testQueryAdjacentVerticesOfEdges() { // Fill edge properties Assert.assertEquals(2, edge.getProperties().size()); Whitebox.setInternalState(edge, "propLoaded", false); - Whitebox.setInternalState(edge, "properties", new HashMap<>()); + Whitebox.setInternalState(edge, "properties", + CollectionFactory.newIntObjectMap()); Assert.assertEquals(0, edge.getProperties().size()); Assert.assertEquals(2, edge.getFilledProperties().size()); Assert.assertEquals(2, edge.getProperties().size()); From 23a06741eeed05a9d7b14dd8fc0db6d733f4e544 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Sun, 11 Apr 2021 12:06:41 +0800 Subject: [PATCH 17/38] implement Int2IntsMap for array paths record Change-Id: I6418971e2d0a3d9153f4280541529c3bf2a00be8 --- .../traversal/algorithm/PathsTraverser.java | 5 +- .../algorithm/records/ArrayPathsRecord.java | 362 ++++++++++++++++++ .../hugegraph/unit/util/JsonUtilTest.java | 26 +- 3 files changed, 381 insertions(+), 12 deletions(-) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecord.java diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java index d2db9efcd8..7d42ac9732 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java @@ -27,6 +27,7 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeEdge; +import com.baidu.hugegraph.traversal.algorithm.records.ArrayPathsRecord; import com.baidu.hugegraph.traversal.algorithm.records.PathsRecord; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; @@ -82,7 +83,7 @@ public PathSet paths(Id sourceV, Directions sourceDir, private class Traverser { - private final PathsRecord record; + private final ArrayPathsRecord record; private final Id label; private final long degree; @@ -93,7 +94,7 @@ private class Traverser { public Traverser(Id sourceV, Id targetV, Id label, long degree, long capacity, long limit) { - this.record = new PathsRecord(sourceV, targetV); + this.record = new ArrayPathsRecord(sourceV, targetV); this.label = label; this.degree = degree; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecord.java new file mode 100644 index 0000000000..69a175ad7b --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecord.java @@ -0,0 +1,362 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.Stack; +import java.util.function.Function; + +import org.apache.commons.collections.CollectionUtils; +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.perf.PerfUtil.Watched; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; +import com.baidu.hugegraph.util.collection.ObjectIntMapping; +import com.google.common.collect.Lists; + +public class ArrayPathsRecord implements Record { + + private ObjectIntMapping idMapping; + + private final Stack sourceLayers; + private final Stack targetLayers; + + private Int2IntsMap currentLayer; + private IntIterator iterator; + private int current; + private boolean forward; + + private int accessed; + + public ArrayPathsRecord(Id sourceV, Id targetV) { + + this.idMapping = new ObjectIntMapping(); + + int sourceCode = this.code(sourceV); + int targetCode = this.code(targetV); + Int2IntsMap firstSourceLayer = new Int2IntsMap(); + Int2IntsMap firstTargetLayer = new Int2IntsMap(); + firstSourceLayer.add(sourceCode, 0); + firstTargetLayer.add(targetCode, 0); + this.sourceLayers = new Stack<>(); + this.targetLayers = new Stack<>(); + this.sourceLayers.push(firstSourceLayer); + this.targetLayers.push(firstTargetLayer); + + this.accessed = 2; + } + + public void startOneLayer(boolean forward) { + this.forward = forward; + this.currentLayer = new Int2IntsMap(); + this.iterator = this.forward ? + this.sourceLayers.peek().keyIterator() : + this.targetLayers.peek().keyIterator(); + } + + public void finishOneLayer() { + if (this.forward) { + this.sourceLayers.push(this.currentLayer); + } else { + this.targetLayers.push(this.currentLayer); + } + this.accessed += this.currentLayer.size(); + } + + @Watched + public boolean hasNext() { + return this.iterator.hasNext(); + } + + @Watched + public Id next() { + this.current = this.iterator.next(); + return this.id(current); + } + + @Watched + public PathSet findPath(Id target, Function filter, + boolean all, boolean ring) { + assert all; + PathSet results = new PathSet(); + int targetCode = this.code(target); + // If cross point exists, path found, concat them + if (this.contains(targetCode)) { + results = this.forward ? + this.getPath(this.current, targetCode, ring) : + this.getPath(targetCode, this.current, ring); + } + this.add(targetCode, this.current); + return results; + } + + public long accessed() { + return this.accessed; + } + + private boolean contains(int node) { + return this.forward ? this.targetContains(node) : + this.sourceContains(node); + } + + private boolean sourceContains(int node) { + return this.sourceLayers.peek().containsKey(node); + } + + private boolean targetContains(int node) { + return this.targetLayers.peek().containsKey(node); + } + + @Watched + private PathSet getPath(int source, int target, boolean ring) { + PathSet results = new PathSet(); + PathSet sources = this.getSourcePath(source); + PathSet targets = this.getTargetPath(target); + for (Path tpath : targets) { + tpath.reverse(); + for (Path spath : sources) { + if (!ring) { + // Avoid loop in path + if (CollectionUtils.containsAny(spath.vertices(), + tpath.vertices())) { + continue; + } + } + List ids = new ArrayList<>(spath.vertices()); + ids.addAll(tpath.vertices()); + results.add(new Path(ids)); + } + } + return results; + } + + private PathSet getPath(Stack all, int id, int layerIndex) { + PathSet results = new PathSet(); + if (layerIndex == 0) { + Id sid = this.id(id); + results.add(new Path(Lists.newArrayList(sid))); + return results; + } + + Id sid = this.id(id); + Int2IntsMap layer = all.elementAt(layerIndex); + int[] parents = layer.get(id); + for (int parent : parents) { + PathSet paths = this.getPath(all, parent, layerIndex - 1); + for (Iterator iter = paths.iterator(); iter.hasNext();) { + Path path = iter.next(); + if (path.vertices().contains(sid)) { + iter.remove(); + continue; + } + path.addToEnd(sid); + } + + results.addAll(paths); + } + return results; + } + + private PathSet getSourcePath(int source) { + return this.getPath(this.sourceLayers, source, + this.sourceLayers.size() - 1); + } + + private PathSet getTargetPath(int target) { + return this.getPath(this.targetLayers, target, + this.targetLayers.size() - 1); + } + + @Watched + private void add(int current, int parent) { + this.currentLayer.add(current, parent); + } + + @Watched + private int code(Id id) { + return this.idMapping.object2Code(id); + } + + @Watched + private Id id(int code) { + return (Id) this.idMapping.code2Object(code); + } + + public static class Int2IntsMap { + + private static final int INIT_CAPACITY = 16; + private static final int CHUNK_SIZE = 10; + + private IntIntHashMap offsetMap; + private int[] intsTable; + + private int nextBlock; + + public Int2IntsMap() { + this.offsetMap = new IntIntHashMap(INIT_CAPACITY); + this.intsTable = new int[INIT_CAPACITY * CHUNK_SIZE]; + this.nextBlock = 0; + } + + public void add(int key, int value) { + if (this.offsetMap.containsKey(key)) { + int positionIndex = this.offsetMap.get(key); + int position = this.intsTable[positionIndex]; + if (this.endOfChunk(position)) { + this.ensureCapacity(); + + this.intsTable[position] = this.nextBlock; + + this.intsTable[this.nextBlock] = value; + this.intsTable[positionIndex] = this.nextBlock + 1; + + // Update next block + this.nextBlock += CHUNK_SIZE; + + } else { + intsTable[position] = value; + this.intsTable[positionIndex]++; + } + this.intsTable[positionIndex + 1]++; + } else { + // New key, allocate 1st chunk and init + this.ensureCapacity(); + + // Allocate 1st chunk + this.offsetMap.put(key, this.nextBlock); + + // Init first chunk + this.intsTable[this.nextBlock] = this.nextBlock + 3; + this.intsTable[this.nextBlock + 1] = 1; + this.intsTable[this.nextBlock + 2] = value; + + // Update next block + this.nextBlock += CHUNK_SIZE; + } + } + + public boolean containsKey(int key) { + return this.offsetMap.containsKey(key); + } + + public int[] get(int key) { + int firstChunk = this.offsetMap.get(key); + int size = this.intsTable[firstChunk + 1]; + int[] values = new int[size]; + for (int i = 0, position = firstChunk + 2; i < size; i++) { + if (!this.endOfChunk(position)) { + values[i] = this.intsTable[position++]; + } else { + position = this.intsTable[position]; + i--; + } + } + return values; + } + + public IntIterator keyIterator() { + return this.offsetMap.keySet().intIterator(); + } + + public int size() { + return this.offsetMap.size(); + } + + private boolean endOfChunk(int position) { + return (position + 1) % CHUNK_SIZE == 0; + } + + private void ensureCapacity() { + if (this.nextBlock >= this.intsTable.length) { + this.expansion(); + } + } + + private void expansion() { + int currentSize = this.intsTable.length; + int[] newTable = new int[2 * currentSize]; + System.arraycopy(this.intsTable, 0, newTable, 0, currentSize); + this.intsTable = newTable; + } + } + + public static void main(String[] args) { + int[] array = new int[5]; + for (int i = 0; i < array.length; i++) { + System.out.println(array[i]); + } + array[3]++; + System.out.println(array[3]); + + Int2IntsMap map = new Int2IntsMap(); + + Random random = new Random(); + int i = 100; + int j = 200; + int k = 250; + int l = 255; + int m = 270; + + for (int n = 0; n < 1000; n++) { + switch (random.nextInt() % 5) { + case 0: + if (i < 200) { + map.add(1, i++); + } + break; + case 1: + if (j < 250) { + map.add(2, j++); + } + break; + case 2: + if (k < 255) { + map.add(3, k++); + } + break; + case 3: + if (l < 270) { + map.add(4, l++); + } + break; + case 4: + if (m < 300) { + map.add(5, m++); + } + break; + } + } + + for (int ii = 1; ii <= 5; ii++) { + int[] result = map.get(ii); + for (int jj : result) { + System.out.print(jj); + System.out.print(","); + } + System.out.println(); + } + } +} diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/JsonUtilTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/JsonUtilTest.java index eb2521e072..a42d55082b 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/JsonUtilTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/JsonUtilTest.java @@ -21,10 +21,11 @@ import java.util.Arrays; import java.util.List; -import java.util.Map; import java.util.UUID; import org.apache.tinkerpop.shaded.jackson.core.type.TypeReference; +import org.eclipse.collections.api.map.primitive.MutableIntObjectMap; +import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; import org.junit.Test; import org.mockito.Mockito; @@ -54,7 +55,6 @@ import com.baidu.hugegraph.unit.FakeObjects; import com.baidu.hugegraph.util.JsonUtil; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; public class JsonUtilTest extends BaseUnitTest { @@ -234,10 +234,14 @@ public void testSerializeVertexWithNumberId() { Id id = IdGenerator.of(123456L); HugeVertex vertex = new HugeVertex(fakeObject.graph(), id, vl); - Map> properties = ImmutableMap.of( - name.id(), new HugeVertexProperty<>(vertex, name, "marko"), - age.id(), new HugeVertexProperty<>(vertex, age, 29), - city.id(), new HugeVertexProperty<>(vertex, city, "Beijing") + MutableIntObjectMap> properties = + IntObjectHashMap.newWithKeysValues( + (int) name.id().asLong(), + new HugeVertexProperty<>(vertex, name, "marko"), + (int) age.id().asLong(), + new HugeVertexProperty<>(vertex, age, 29), + (int) city.id().asLong(), + new HugeVertexProperty<>(vertex, city, "Beijing") ); Whitebox.setInternalState(vertex, "properties", properties); @@ -282,10 +286,12 @@ public void testSerializeEdge() { Whitebox.setInternalState(edge, "sourceVertex", source); Whitebox.setInternalState(edge, "targetVertex", target); - Map> properties = ImmutableMap.of( - date.id(), new HugeEdgeProperty<>(edge, date, - Utils.date("2019-03-12")), - weight.id(), new HugeEdgeProperty<>(edge, weight, 0.8) + MutableIntObjectMap> properties = + IntObjectHashMap.newWithKeysValues( + (int) date.id().asLong(), + new HugeEdgeProperty<>(edge, date, Utils.date("2019-03-12")), + (int) weight.id().asLong(), + new HugeEdgeProperty<>(edge, weight, 0.8) ); Whitebox.setInternalState(edge, "properties", properties); From e4dac482bfec73b7fe96d25f20caa5babd11f00e Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Mon, 12 Apr 2021 21:22:43 +0800 Subject: [PATCH 18/38] improve Change-Id: I6c13f504986183804f4c7ae45a5f4e0e6099125b --- .../hugegraph/structure/HugeElement.java | 2 +- .../baidu/hugegraph/structure/HugeVertex.java | 4 +- .../traversal/algorithm/PathsTraverser.java | 15 +- .../algorithm/ShortestPathTraverser.java | 8 +- .../algorithm/records/ArrayPathsRecord.java | 362 ------------------ .../algorithm/records/ArrayPathsRecords.java | 204 ++++++++++ .../algorithm/records/Int2IntsMap.java | 175 +++++++++ .../{PathsRecord.java => PathsRecords.java} | 42 +- .../records/{Record.java => Records.java} | 6 +- .../algorithm/records/ShortestPathRecord.java | 6 +- .../util/collection/ObjectIntMapping.java | 2 +- .../hugegraph/unit/core/Int2IntsMapTest.java | 133 +++++++ 12 files changed, 554 insertions(+), 405 deletions(-) delete mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecord.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecords.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Int2IntsMap.java rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/{PathsRecord.java => PathsRecords.java} (84%) rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/{Record.java => Records.java} (93%) create mode 100644 hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java index 1d3c672ff8..cbd03d37f3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java @@ -277,7 +277,7 @@ public int sizeOfSubProperties() { @Watched(prefix = "element") public HugeProperty setProperty(HugeProperty prop) { if (this.properties == EMPTY_MAP) { - this.properties = new IntObjectHashMap<>(); + this.properties = CollectionFactory.newIntObjectMap(); } PropertyKey pkey = prop.propertyKey(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java index f22c1c4a96..60692f41d1 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java @@ -224,7 +224,7 @@ public Collection getEdges() { } public void resetEdges() { - this.edges = new FastList<>(); + this.edges = CollectionFactory.newList(CollectionImplType.EC); } public void removeEdge(HugeEdge edge) { @@ -233,7 +233,7 @@ public void removeEdge(HugeEdge edge) { public void addEdge(HugeEdge edge) { if (this.edges == EMPTY_LIST) { - this.edges = new FastList<>(); + this.edges = CollectionFactory.newList(CollectionImplType.EC); } this.edges.add(edge); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java index 7d42ac9732..5ec93d8807 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java @@ -27,8 +27,7 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeEdge; -import com.baidu.hugegraph.traversal.algorithm.records.ArrayPathsRecord; -import com.baidu.hugegraph.traversal.algorithm.records.PathsRecord; +import com.baidu.hugegraph.traversal.algorithm.records.ArrayPathsRecords; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; import com.google.common.collect.ImmutableList; @@ -83,7 +82,7 @@ public PathSet paths(Id sourceV, Directions sourceDir, private class Traverser { - private final ArrayPathsRecord record; + private final ArrayPathsRecords record; private final Id label; private final long degree; @@ -94,7 +93,7 @@ private class Traverser { public Traverser(Id sourceV, Id targetV, Id label, long degree, long capacity, long limit) { - this.record = new ArrayPathsRecord(sourceV, targetV); + this.record = new ArrayPathsRecords(sourceV, targetV); this.label = label; this.degree = degree; @@ -112,8 +111,8 @@ public void forward(Directions direction) { Iterator edges; this.record.startOneLayer(true); - while (this.record.hasNext()) { - Id vid = this.record.next(); + while (this.record.hasNextKey()) { + Id vid = this.record.nextKey(); edges = edgesOfVertex(vid, direction, this.label, this.degree); @@ -142,8 +141,8 @@ public void backward(Directions direction) { Iterator edges; this.record.startOneLayer(false); - while (this.record.hasNext()) { - Id vid = this.record.next(); + while (this.record.hasNextKey()) { + Id vid = this.record.nextKey(); edges = edgesOfVertex(vid, direction, this.label, this.degree); while (edges.hasNext()) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java index b2a6a3e2f4..2e8b511bcb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java @@ -169,8 +169,8 @@ public PathSet forward(boolean all) { long degree = this.skipDegree > 0L ? this.skipDegree : this.degree; this.record.startOneLayer(true); - while (this.record.hasNext()) { - Id source = this.record.next(); + while (this.record.hasNextKey()) { + Id source = this.record.nextKey(); Iterator edges = edgesOfVertex(source, this.direction, this.labels, degree); @@ -209,8 +209,8 @@ public PathSet backward(boolean all) { Directions opposite = this.direction.opposite(); this.record.startOneLayer(false); - while (this.record.hasNext()) { - Id source = this.record.next(); + while (this.record.hasNextKey()) { + Id source = this.record.nextKey(); Iterator edges = edgesOfVertex(source, opposite, this.labels, degree); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecord.java deleted file mode 100644 index 69a175ad7b..0000000000 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecord.java +++ /dev/null @@ -1,362 +0,0 @@ -/* - * Copyright 2017 HugeGraph Authors - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.baidu.hugegraph.traversal.algorithm.records; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Random; -import java.util.Stack; -import java.util.function.Function; - -import org.apache.commons.collections.CollectionUtils; -import org.eclipse.collections.api.iterator.IntIterator; -import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; - -import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.perf.PerfUtil.Watched; -import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; -import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; -import com.baidu.hugegraph.util.collection.ObjectIntMapping; -import com.google.common.collect.Lists; - -public class ArrayPathsRecord implements Record { - - private ObjectIntMapping idMapping; - - private final Stack sourceLayers; - private final Stack targetLayers; - - private Int2IntsMap currentLayer; - private IntIterator iterator; - private int current; - private boolean forward; - - private int accessed; - - public ArrayPathsRecord(Id sourceV, Id targetV) { - - this.idMapping = new ObjectIntMapping(); - - int sourceCode = this.code(sourceV); - int targetCode = this.code(targetV); - Int2IntsMap firstSourceLayer = new Int2IntsMap(); - Int2IntsMap firstTargetLayer = new Int2IntsMap(); - firstSourceLayer.add(sourceCode, 0); - firstTargetLayer.add(targetCode, 0); - this.sourceLayers = new Stack<>(); - this.targetLayers = new Stack<>(); - this.sourceLayers.push(firstSourceLayer); - this.targetLayers.push(firstTargetLayer); - - this.accessed = 2; - } - - public void startOneLayer(boolean forward) { - this.forward = forward; - this.currentLayer = new Int2IntsMap(); - this.iterator = this.forward ? - this.sourceLayers.peek().keyIterator() : - this.targetLayers.peek().keyIterator(); - } - - public void finishOneLayer() { - if (this.forward) { - this.sourceLayers.push(this.currentLayer); - } else { - this.targetLayers.push(this.currentLayer); - } - this.accessed += this.currentLayer.size(); - } - - @Watched - public boolean hasNext() { - return this.iterator.hasNext(); - } - - @Watched - public Id next() { - this.current = this.iterator.next(); - return this.id(current); - } - - @Watched - public PathSet findPath(Id target, Function filter, - boolean all, boolean ring) { - assert all; - PathSet results = new PathSet(); - int targetCode = this.code(target); - // If cross point exists, path found, concat them - if (this.contains(targetCode)) { - results = this.forward ? - this.getPath(this.current, targetCode, ring) : - this.getPath(targetCode, this.current, ring); - } - this.add(targetCode, this.current); - return results; - } - - public long accessed() { - return this.accessed; - } - - private boolean contains(int node) { - return this.forward ? this.targetContains(node) : - this.sourceContains(node); - } - - private boolean sourceContains(int node) { - return this.sourceLayers.peek().containsKey(node); - } - - private boolean targetContains(int node) { - return this.targetLayers.peek().containsKey(node); - } - - @Watched - private PathSet getPath(int source, int target, boolean ring) { - PathSet results = new PathSet(); - PathSet sources = this.getSourcePath(source); - PathSet targets = this.getTargetPath(target); - for (Path tpath : targets) { - tpath.reverse(); - for (Path spath : sources) { - if (!ring) { - // Avoid loop in path - if (CollectionUtils.containsAny(spath.vertices(), - tpath.vertices())) { - continue; - } - } - List ids = new ArrayList<>(spath.vertices()); - ids.addAll(tpath.vertices()); - results.add(new Path(ids)); - } - } - return results; - } - - private PathSet getPath(Stack all, int id, int layerIndex) { - PathSet results = new PathSet(); - if (layerIndex == 0) { - Id sid = this.id(id); - results.add(new Path(Lists.newArrayList(sid))); - return results; - } - - Id sid = this.id(id); - Int2IntsMap layer = all.elementAt(layerIndex); - int[] parents = layer.get(id); - for (int parent : parents) { - PathSet paths = this.getPath(all, parent, layerIndex - 1); - for (Iterator iter = paths.iterator(); iter.hasNext();) { - Path path = iter.next(); - if (path.vertices().contains(sid)) { - iter.remove(); - continue; - } - path.addToEnd(sid); - } - - results.addAll(paths); - } - return results; - } - - private PathSet getSourcePath(int source) { - return this.getPath(this.sourceLayers, source, - this.sourceLayers.size() - 1); - } - - private PathSet getTargetPath(int target) { - return this.getPath(this.targetLayers, target, - this.targetLayers.size() - 1); - } - - @Watched - private void add(int current, int parent) { - this.currentLayer.add(current, parent); - } - - @Watched - private int code(Id id) { - return this.idMapping.object2Code(id); - } - - @Watched - private Id id(int code) { - return (Id) this.idMapping.code2Object(code); - } - - public static class Int2IntsMap { - - private static final int INIT_CAPACITY = 16; - private static final int CHUNK_SIZE = 10; - - private IntIntHashMap offsetMap; - private int[] intsTable; - - private int nextBlock; - - public Int2IntsMap() { - this.offsetMap = new IntIntHashMap(INIT_CAPACITY); - this.intsTable = new int[INIT_CAPACITY * CHUNK_SIZE]; - this.nextBlock = 0; - } - - public void add(int key, int value) { - if (this.offsetMap.containsKey(key)) { - int positionIndex = this.offsetMap.get(key); - int position = this.intsTable[positionIndex]; - if (this.endOfChunk(position)) { - this.ensureCapacity(); - - this.intsTable[position] = this.nextBlock; - - this.intsTable[this.nextBlock] = value; - this.intsTable[positionIndex] = this.nextBlock + 1; - - // Update next block - this.nextBlock += CHUNK_SIZE; - - } else { - intsTable[position] = value; - this.intsTable[positionIndex]++; - } - this.intsTable[positionIndex + 1]++; - } else { - // New key, allocate 1st chunk and init - this.ensureCapacity(); - - // Allocate 1st chunk - this.offsetMap.put(key, this.nextBlock); - - // Init first chunk - this.intsTable[this.nextBlock] = this.nextBlock + 3; - this.intsTable[this.nextBlock + 1] = 1; - this.intsTable[this.nextBlock + 2] = value; - - // Update next block - this.nextBlock += CHUNK_SIZE; - } - } - - public boolean containsKey(int key) { - return this.offsetMap.containsKey(key); - } - - public int[] get(int key) { - int firstChunk = this.offsetMap.get(key); - int size = this.intsTable[firstChunk + 1]; - int[] values = new int[size]; - for (int i = 0, position = firstChunk + 2; i < size; i++) { - if (!this.endOfChunk(position)) { - values[i] = this.intsTable[position++]; - } else { - position = this.intsTable[position]; - i--; - } - } - return values; - } - - public IntIterator keyIterator() { - return this.offsetMap.keySet().intIterator(); - } - - public int size() { - return this.offsetMap.size(); - } - - private boolean endOfChunk(int position) { - return (position + 1) % CHUNK_SIZE == 0; - } - - private void ensureCapacity() { - if (this.nextBlock >= this.intsTable.length) { - this.expansion(); - } - } - - private void expansion() { - int currentSize = this.intsTable.length; - int[] newTable = new int[2 * currentSize]; - System.arraycopy(this.intsTable, 0, newTable, 0, currentSize); - this.intsTable = newTable; - } - } - - public static void main(String[] args) { - int[] array = new int[5]; - for (int i = 0; i < array.length; i++) { - System.out.println(array[i]); - } - array[3]++; - System.out.println(array[3]); - - Int2IntsMap map = new Int2IntsMap(); - - Random random = new Random(); - int i = 100; - int j = 200; - int k = 250; - int l = 255; - int m = 270; - - for (int n = 0; n < 1000; n++) { - switch (random.nextInt() % 5) { - case 0: - if (i < 200) { - map.add(1, i++); - } - break; - case 1: - if (j < 250) { - map.add(2, j++); - } - break; - case 2: - if (k < 255) { - map.add(3, k++); - } - break; - case 3: - if (l < 270) { - map.add(4, l++); - } - break; - case 4: - if (m < 300) { - map.add(5, m++); - } - break; - } - } - - for (int ii = 1; ii <= 5; ii++) { - int[] result = map.get(ii); - for (int jj : result) { - System.out.print(jj); - System.out.print(","); - } - System.out.println(); - } - } -} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecords.java new file mode 100644 index 0000000000..89dad50de2 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecords.java @@ -0,0 +1,204 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Stack; +import java.util.function.Function; + +import org.apache.commons.collections.CollectionUtils; +import org.eclipse.collections.api.iterator.IntIterator; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.perf.PerfUtil.Watched; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; +import com.baidu.hugegraph.util.collection.ObjectIntMapping; +import com.google.common.collect.Lists; + +public class ArrayPathsRecords implements Records { + + private final ObjectIntMapping idMapping; + + private final Stack sourceLayers; + private final Stack targetLayers; + + private Int2IntsMap currentLayer; + private IntIterator lastLayerKeys; + private int current; + private boolean forward; + + private int accessed; + + public ArrayPathsRecords(Id sourceV, Id targetV) { + this.idMapping = new ObjectIntMapping(); + + int sourceCode = this.code(sourceV); + int targetCode = this.code(targetV); + Int2IntsMap firstSourceLayer = new Int2IntsMap(); + Int2IntsMap firstTargetLayer = new Int2IntsMap(); + firstSourceLayer.add(sourceCode, 0); + firstTargetLayer.add(targetCode, 0); + this.sourceLayers = new Stack<>(); + this.targetLayers = new Stack<>(); + this.sourceLayers.push(firstSourceLayer); + this.targetLayers.push(firstTargetLayer); + + this.accessed = 2; + } + + public void startOneLayer(boolean forward) { + this.forward = forward; + this.currentLayer = new Int2IntsMap(); + this.lastLayerKeys = this.forward ? + this.sourceLayers.peek().keyIterator() : + this.targetLayers.peek().keyIterator(); + } + + public void finishOneLayer() { + if (this.forward) { + this.sourceLayers.push(this.currentLayer); + } else { + this.targetLayers.push(this.currentLayer); + } + this.accessed += this.currentLayer.size(); + } + + @Watched + public boolean hasNextKey() { + return this.lastLayerKeys.hasNext(); + } + + @Watched + public Id nextKey() { + this.current = this.lastLayerKeys.next(); + return this.id(current); + } + + @Watched + public PathSet findPath(Id target, Function filter, + boolean all, boolean ring) { + assert all; + PathSet results = new PathSet(); + int targetCode = this.code(target); + // If cross point exists, path found, concat them + if (this.contains(targetCode)) { + results = this.forward ? + this.concatPath(this.current, targetCode, ring) : + this.concatPath(targetCode, this.current, ring); + } + this.addPath(targetCode, this.current); + return results; + } + + public long accessed() { + return this.accessed; + } + + private boolean contains(int node) { + return this.forward ? this.targetContains(node) : + this.sourceContains(node); + } + + private boolean sourceContains(int node) { + return this.sourceLayers.peek().containsKey(node); + } + + private boolean targetContains(int node) { + return this.targetLayers.peek().containsKey(node); + } + + @Watched + private PathSet concatPath(int source, int target, boolean ring) { + PathSet results = new PathSet(); + PathSet sources = this.getSourcePath(source); + PathSet targets = this.getTargetPath(target); + for (Path tpath : targets) { + tpath.reverse(); + for (Path spath : sources) { + if (!ring) { + // Avoid loop in path + if (CollectionUtils.containsAny(spath.vertices(), + tpath.vertices())) { + continue; + } + } + List ids = new ArrayList<>(spath.vertices()); + ids.addAll(tpath.vertices()); + results.add(new Path(ids)); + } + } + return results; + } + + private PathSet concatPath(Stack all, int id, int layerIndex) { + PathSet results = new PathSet(); + if (layerIndex == 0) { + Id sid = this.id(id); + results.add(new Path(Lists.newArrayList(sid))); + return results; + } + + Id sid = this.id(id); + Int2IntsMap layer = all.elementAt(layerIndex); + int[] parents = layer.get(id); + for (int parent : parents) { + PathSet paths = this.concatPath(all, parent, layerIndex - 1); + for (Iterator iter = paths.iterator(); iter.hasNext();) { + Path path = iter.next(); + if (path.vertices().contains(sid)) { + iter.remove(); + continue; + } + path.addToEnd(sid); + } + + results.addAll(paths); + } + return results; + } + + private PathSet getSourcePath(int source) { + return this.concatPath(this.sourceLayers, source, + this.sourceLayers.size() - 1); + } + + private PathSet getTargetPath(int target) { + return this.concatPath(this.targetLayers, target, + this.targetLayers.size() - 1); + } + + @Watched + private void addPath(int current, int parent) { + this.currentLayer.add(current, parent); + } + + @Watched + private int code(Id id) { + return this.idMapping.object2Code(id); + } + + @Watched + private Id id(int code) { + return (Id) this.idMapping.code2Object(code); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Int2IntsMap.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Int2IntsMap.java new file mode 100644 index 0000000000..4f5e85a21c --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Int2IntsMap.java @@ -0,0 +1,175 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; + +public class Int2IntsMap { + + private static final int INIT_KEY_CAPACITY = 1000000; + private static final int CHUNK_SIZE = 10; + private static final int POSITION_OFFSET = 0; + private static final int SIZE_OFFSET = 1; + private static final int FIRST_CHUNK_DATA_OFFSET = 2; + private static final int EXPANSION_FACTOR = 2; + + /* + * firstChunkMap chunkTable + * + * -------- --------------- + * | 1 | 0 |--------->0 | 33 | (nextPosition) + * | 2 | 10 |-----+ 1 | 19 | (size) + * | 3 | 40 |---+ | 2 | int data | + * | . | . | | | 3 | int data | + * | x | y | | | . | ... | + * -------- | | 9 | 20 |-----------------+ + * | | --------------- | + * | +-->10 | 13 | (nextPosition) | + * | 11 | 1 | (size) | + * | 12 | int data | | nextChunk + * | 13 | 0 | | + * | . | ... | | + * | 19 | 0 | | + * | --------------- | + * | 20 | int data |<----------------+ + * | 21 | int data | + * | 22 | int data | + * | 23 | int data | + * | . | ... | + * | 29 | 30 |-----------------+ + * | --------------- | nextChunk + * | 30 | int data |<----------------+ + * | 31 | int data | + * | 32 | int data | + * | 33 | 0 | + * | . | ... | + * | 39 | 0 | + * | --------------- + * +---->40 | 48 | (nextPosition) + * 41 | 6 | (size) + * 42 | int data | + * 43 | int data | + * . | ... | + * 47 | int data | + * 48 | 0 | + * 49 | 0 | + * --------------- + * 50 | ... | + * | ... | + * | ... | + * | ... | + * + * + */ + + private IntIntHashMap firstChunkMap; + private int[] chunkTable; + + private int nextChunk; + + public Int2IntsMap() { + this.firstChunkMap = new IntIntHashMap(INIT_KEY_CAPACITY); + this.chunkTable = new int[INIT_KEY_CAPACITY * CHUNK_SIZE]; + this.nextChunk = 0; + } + + public void add(int key, int value) { + if (this.firstChunkMap.containsKey(key)) { + int firstChunk = this.firstChunkMap.get(key); + int nextPosition = this.chunkTable[firstChunk + POSITION_OFFSET]; + if (!this.endOfChunk(nextPosition)) { + chunkTable[nextPosition] = value; + this.chunkTable[firstChunk + POSITION_OFFSET]++; + } else { + this.ensureCapacity(); + + this.chunkTable[nextPosition] = this.nextChunk; + + this.chunkTable[this.nextChunk] = value; + this.chunkTable[firstChunk + POSITION_OFFSET] = this.nextChunk + 1; + + // Update next block + this.nextChunk += CHUNK_SIZE; + } + this.chunkTable[firstChunk + SIZE_OFFSET]++; + } else { + // New key, allocate 1st chunk and init + this.ensureCapacity(); + + // Allocate 1st chunk + this.firstChunkMap.put(key, this.nextChunk); + + // Init first chunk + this.chunkTable[this.nextChunk] = this.nextChunk + + FIRST_CHUNK_DATA_OFFSET + 1; + this.chunkTable[this.nextChunk + SIZE_OFFSET] = 1; + this.chunkTable[this.nextChunk + FIRST_CHUNK_DATA_OFFSET] = value; + + // Update next block + this.nextChunk += CHUNK_SIZE; + } + } + + public boolean containsKey(int key) { + return this.firstChunkMap.containsKey(key); + } + + public int[] get(int key) { + int firstChunk = this.firstChunkMap.get(key); + int size = this.chunkTable[firstChunk + SIZE_OFFSET]; + int[] values = new int[size]; + int i = 0; + int position = firstChunk + FIRST_CHUNK_DATA_OFFSET; + while (i < size) { + if (!this.endOfChunk(position)) { + values[i++] = this.chunkTable[position++]; + } else { + position = this.chunkTable[position]; + } + } + return values; + } + + public IntIterator keyIterator() { + return this.firstChunkMap.keySet().intIterator(); + } + + public int size() { + return this.firstChunkMap.size(); + } + + private boolean endOfChunk(int position) { + return (position + 1) % CHUNK_SIZE == 0; + } + + private void ensureCapacity() { + if (this.nextChunk >= this.chunkTable.length) { + this.expansion(); + } + } + + private void expansion() { + int currentSize = this.chunkTable.length; + int[] newTable = new int[currentSize * EXPANSION_FACTOR]; + System.arraycopy(this.chunkTable, 0, newTable, 0, currentSize); + this.chunkTable = newTable; + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecords.java similarity index 84% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecord.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecords.java index 3cdfe889f8..24a40c58bb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecords.java @@ -37,21 +37,21 @@ import com.baidu.hugegraph.util.collection.ObjectIntMapping; import com.google.common.collect.Lists; -public class PathsRecord implements Record { +public class PathsRecords implements Records { - private ObjectIntMapping idMapping; + private final ObjectIntMapping idMapping; private final Stack> sourceLayers; private final Stack> targetLayers; private IntObjectHashMap currentLayer; - private IntIterator iterator; + private IntIterator lastLayerKeys; private int current; private boolean forward; private int accessed; - public PathsRecord(Id sourceV, Id targetV) { + public PathsRecords(Id sourceV, Id targetV) { this.idMapping = new ObjectIntMapping(); @@ -72,7 +72,7 @@ public PathsRecord(Id sourceV, Id targetV) { public void startOneLayer(boolean forward) { this.forward = forward; this.currentLayer = new IntObjectHashMap<>(); - this.iterator = this.forward ? + this.lastLayerKeys = this.forward ? this.sourceLayers.peek().keySet().intIterator() : this.targetLayers.peek().keySet().intIterator(); } @@ -87,13 +87,13 @@ public void finishOneLayer() { } @Watched - public boolean hasNext() { - return this.iterator.hasNext(); + public boolean hasNextKey() { + return this.lastLayerKeys.hasNext(); } @Watched - public Id next() { - this.current = this.iterator.next(); + public Id nextKey() { + this.current = this.lastLayerKeys.next(); return this.id(current); } @@ -106,10 +106,10 @@ public PathSet findPath(Id target, Function filter, // If cross point exists, path found, concat them if (this.contains(targetCode)) { results = this.forward ? - this.getPath(this.current, targetCode, ring) : - this.getPath(targetCode, this.current, ring); + this.concatPath(this.current, targetCode, ring) : + this.concatPath(targetCode, this.current, ring); } - this.add(targetCode, this.current); + this.addPath(targetCode, this.current); return results; } @@ -131,7 +131,7 @@ private boolean targetContains(int node) { } @Watched - private PathSet getPath(int source, int target, boolean ring) { + private PathSet concatPath(int source, int target, boolean ring) { PathSet results = new PathSet(); PathSet sources = this.getSourcePath(source); PathSet targets = this.getTargetPath(target); @@ -153,8 +153,8 @@ private PathSet getPath(int source, int target, boolean ring) { return results; } - private PathSet getPath(Stack> all, - int id, int layerIndex) { + private PathSet concatPath(Stack> all, + int id, int layerIndex) { PathSet results = new PathSet(); if (layerIndex == 0) { Id sid = this.id(id); @@ -168,7 +168,7 @@ private PathSet getPath(Stack> all, IntIterator iterator = parents.intIterator(); while (iterator.hasNext()) { int parent = iterator.next(); - PathSet paths = this.getPath(all, parent, layerIndex - 1); + PathSet paths = this.concatPath(all, parent, layerIndex - 1); for (Iterator iter = paths.iterator(); iter.hasNext();) { Path path = iter.next(); if (path.vertices().contains(sid)) { @@ -184,17 +184,17 @@ private PathSet getPath(Stack> all, } private PathSet getSourcePath(int source) { - return this.getPath(this.sourceLayers, source, - this.sourceLayers.size() - 1); + return this.concatPath(this.sourceLayers, source, + this.sourceLayers.size() - 1); } private PathSet getTargetPath(int target) { - return this.getPath(this.targetLayers, target, - this.targetLayers.size() - 1); + return this.concatPath(this.targetLayers, target, + this.targetLayers.size() - 1); } @Watched - private void add(int current, int parent) { + private void addPath(int current, int parent) { if (this.currentLayer.containsKey(current)) { this.currentLayer.get(current).add(parent); } else { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Records.java similarity index 93% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Records.java index f26edb738f..405ed933eb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Records.java @@ -24,15 +24,15 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; -public interface Record { +public interface Records { public void startOneLayer(boolean forward); public void finishOneLayer(); - public boolean hasNext(); + public boolean hasNextKey(); - public Id next(); + public Id nextKey(); public PathSet findPath(Id target, Function filter, boolean all, boolean ring); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java index 16c129f943..91e57f2d0e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java @@ -34,7 +34,7 @@ import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; import com.baidu.hugegraph.util.collection.ObjectIntMapping; -public class ShortestPathRecord implements Record { +public class ShortestPathRecord implements Records { private ObjectIntMapping idMapping; @@ -89,11 +89,11 @@ public void finishOneLayer() { this.size += this.currentLayer.size(); } - public boolean hasNext() { + public boolean hasNextKey() { return this.iterator.hasNext(); } - public Id next() { + public Id nextKey() { this.current = this.iterator.next(); return this.id(current); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java index f6a081c0a9..d09431d12a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java @@ -31,7 +31,7 @@ public final class ObjectIntMapping { private final IntObjectHashMap int2IdMap; public ObjectIntMapping() { - this.int2IdMap = new IntObjectHashMap(); + this.int2IdMap = new IntObjectHashMap(1000000); } @Watched diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java new file mode 100644 index 0000000000..6ecf5c29fe --- /dev/null +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java @@ -0,0 +1,133 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.unit.core; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Random; +import java.util.Set; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.baidu.hugegraph.testutil.Assert; +import com.baidu.hugegraph.traversal.algorithm.records.Int2IntsMap; + +public class Int2IntsMapTest { + + @Before + public void setup() { + // pass + } + + @After + public void teardown() { + // pass + } + + @Test + public void testInt2IntsMap() { + Int2IntsMap map = new Int2IntsMap(); + + int i = 100; + int j = 200; + int k = 250; + int l = 255; + int m = 270; + Random random = new Random(); + for (int n = 0; n < 1000; n++) { + switch (random.nextInt() % 5) { + case 0: + if (i < 200) { + map.add(1, i++); + } + break; + case 1: + if (j < 250) { + map.add(2, j++); + } + break; + case 2: + if (k < 255) { + map.add(3, k++); + } + break; + case 3: + if (l < 270) { + map.add(4, l++); + } + break; + case 4: + if (m < 300) { + map.add(5, m++); + } + break; + } + } + + int[][] results = new int[5][]; + results[0] = new int[]{100,101,102,103,104,105,106,107,108,109, + 110,111,112,113,114,115,116,117,118,119, + 120,121,122,123,124,125,126,127,128,129, + 130,131,132,133,134,135,136,137,138,139, + 140,141,142,143,144,145,146,147,148,149, + 150,151,152,153,154,155,156,157,158,159, + 160,161,162,163,164,165,166,167,168,169, + 170,171,172,173,174,175,176,177,178,179, + 180,181,182,183,184,185,186,187,188,189, + 190,191,192,193,194,195,196,197,198,199}; + results[1] = new int[]{200,201,202,203,204,205,206,207,208,209, + 210,211,212,213,214,215,216,217,218,219, + 220,221,222,223,224,225,226,227,228,229, + 230,231,232,233,234,235,236,237,238,239, + 240,241,242,243,244,245,246,247,248,249}; + results[2] = new int[]{250,251,252,253,254}; + results[3] = new int[]{255,256,257,258,259,260,261,262, + 263,264,265,266,267,268,269}; + results[4] = new int[]{270,271,272,273,274,275,276,277,278,279, + 280,281,282,283,284,285,286,287,288,289, + 290, 291,292,293,294,295,296,297,298,299}; + + for (int ii = 0; ii < 5; ii++) { + int[] result = map.get(ii + 1); + Assert.assertTrue(Arrays.equals(results[ii], result)); + } + } + + @Test + public void testInt2IntsMapRandom() { + Int2IntsMap map = new Int2IntsMap(); + + Random random = new Random(); + for (int i = 0; i < 1000; i++) { + map.add(Math.abs(random.nextInt()) % 5, i); + } + + Set results = new HashSet<>(); + for (int i = 0; i < 5; i++) { + int[] result = map.get(i); + for (int j : result) { + results.add(j); + } + } + Assert.assertEquals(1000, results.size()); + } +} From affd1a5291da91c424c9a620d83dcf43316afeab Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 13 Apr 2021 15:15:51 +0800 Subject: [PATCH 19/38] improve Change-Id: Ie8b79a4c05696f1842476bd74d82b75fba4925a1 --- .../traversal/algorithm/PathsTraverser.java | 6 +- .../algorithm/ShortestPathTraverser.java | 2 +- ...rds.java => MultiPathsByArrayRecords.java} | 43 ++++++------- ...cords.java => MultiPathsBySetRecords.java} | 46 +++++++------- .../traversal/algorithm/records/Records.java | 2 + .../algorithm/records/ShortestPathRecord.java | 62 ++++++++++--------- .../collection}/Int2IntsMap.java | 50 ++++++++------- .../baidu/hugegraph/unit/UnitTestSuite.java | 2 + .../hugegraph/unit/core/Int2IntsMapTest.java | 2 +- 9 files changed, 113 insertions(+), 102 deletions(-) rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/{ArrayPathsRecords.java => MultiPathsByArrayRecords.java} (83%) rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/{PathsRecords.java => MultiPathsBySetRecords.java} (83%) rename hugegraph-core/src/main/java/com/baidu/hugegraph/{traversal/algorithm/records => util/collection}/Int2IntsMap.java (79%) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java index 5ec93d8807..12a49eeffb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java @@ -27,7 +27,7 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeEdge; -import com.baidu.hugegraph.traversal.algorithm.records.ArrayPathsRecords; +import com.baidu.hugegraph.traversal.algorithm.records.MultiPathsByArrayRecords; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; import com.google.common.collect.ImmutableList; @@ -82,7 +82,7 @@ public PathSet paths(Id sourceV, Directions sourceDir, private class Traverser { - private final ArrayPathsRecords record; + private final MultiPathsByArrayRecords record; private final Id label; private final long degree; @@ -93,7 +93,7 @@ private class Traverser { public Traverser(Id sourceV, Id targetV, Id label, long degree, long capacity, long limit) { - this.record = new ArrayPathsRecords(sourceV, targetV); + this.record = new MultiPathsByArrayRecords(sourceV, targetV); this.label = label; this.degree = degree; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java index 2e8b511bcb..91a7b437cf 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java @@ -181,7 +181,7 @@ public PathSet forward(boolean all) { Id target = edge.id().otherVertexId(); PathSet paths = this.record.findPath(target, - t -> this.superNode(t, this.direction), + t -> !this.superNode(t, this.direction), all, false); if (paths.isEmpty()) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsByArrayRecords.java similarity index 83% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecords.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsByArrayRecords.java index 89dad50de2..81d09c0ae7 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsByArrayRecords.java @@ -32,10 +32,11 @@ import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; +import com.baidu.hugegraph.util.collection.Int2IntsMap; import com.baidu.hugegraph.util.collection.ObjectIntMapping; import com.google.common.collect.Lists; -public class ArrayPathsRecords implements Records { +public class MultiPathsByArrayRecords implements Records { private final ObjectIntMapping idMapping; @@ -49,7 +50,7 @@ public class ArrayPathsRecords implements Records { private int accessed; - public ArrayPathsRecords(Id sourceV, Id targetV) { + public MultiPathsByArrayRecords(Id sourceV, Id targetV) { this.idMapping = new ObjectIntMapping(); int sourceCode = this.code(sourceV); @@ -70,8 +71,8 @@ public void startOneLayer(boolean forward) { this.forward = forward; this.currentLayer = new Int2IntsMap(); this.lastLayerKeys = this.forward ? - this.sourceLayers.peek().keyIterator() : - this.targetLayers.peek().keyIterator(); + this.sourceLayers.peek().keyIterator() : + this.targetLayers.peek().keyIterator(); } public void finishOneLayer() { @@ -103,8 +104,8 @@ public PathSet findPath(Id target, Function filter, // If cross point exists, path found, concat them if (this.contains(targetCode)) { results = this.forward ? - this.concatPath(this.current, targetCode, ring) : - this.concatPath(targetCode, this.current, ring); + this.linkPath(this.current, targetCode, ring) : + this.linkPath(targetCode, this.current, ring); } this.addPath(targetCode, this.current); return results; @@ -128,10 +129,10 @@ private boolean targetContains(int node) { } @Watched - private PathSet concatPath(int source, int target, boolean ring) { + private PathSet linkPath(int source, int target, boolean ring) { PathSet results = new PathSet(); - PathSet sources = this.getSourcePath(source); - PathSet targets = this.getTargetPath(target); + PathSet sources = this.linkSourcePath(source); + PathSet targets = this.linkTargetPath(target); for (Path tpath : targets) { tpath.reverse(); for (Path spath : sources) { @@ -150,7 +151,17 @@ private PathSet concatPath(int source, int target, boolean ring) { return results; } - private PathSet concatPath(Stack all, int id, int layerIndex) { + private PathSet linkSourcePath(int source) { + return this.linkPath(this.sourceLayers, source, + this.sourceLayers.size() - 1); + } + + private PathSet linkTargetPath(int target) { + return this.linkPath(this.targetLayers, target, + this.targetLayers.size() - 1); + } + + private PathSet linkPath(Stack all, int id, int layerIndex) { PathSet results = new PathSet(); if (layerIndex == 0) { Id sid = this.id(id); @@ -162,7 +173,7 @@ private PathSet concatPath(Stack all, int id, int layerIndex) { Int2IntsMap layer = all.elementAt(layerIndex); int[] parents = layer.get(id); for (int parent : parents) { - PathSet paths = this.concatPath(all, parent, layerIndex - 1); + PathSet paths = this.linkPath(all, parent, layerIndex - 1); for (Iterator iter = paths.iterator(); iter.hasNext();) { Path path = iter.next(); if (path.vertices().contains(sid)) { @@ -177,16 +188,6 @@ private PathSet concatPath(Stack all, int id, int layerIndex) { return results; } - private PathSet getSourcePath(int source) { - return this.concatPath(this.sourceLayers, source, - this.sourceLayers.size() - 1); - } - - private PathSet getTargetPath(int target) { - return this.concatPath(this.targetLayers, target, - this.targetLayers.size() - 1); - } - @Watched private void addPath(int current, int parent) { this.currentLayer.add(current, parent); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsBySetRecords.java similarity index 83% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecords.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsBySetRecords.java index 24a40c58bb..7a97b5ff68 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsBySetRecords.java @@ -37,7 +37,7 @@ import com.baidu.hugegraph.util.collection.ObjectIntMapping; import com.google.common.collect.Lists; -public class PathsRecords implements Records { +public class MultiPathsBySetRecords implements Records { private final ObjectIntMapping idMapping; @@ -51,7 +51,7 @@ public class PathsRecords implements Records { private int accessed; - public PathsRecords(Id sourceV, Id targetV) { + public MultiPathsBySetRecords(Id sourceV, Id targetV) { this.idMapping = new ObjectIntMapping(); @@ -71,10 +71,10 @@ public PathsRecords(Id sourceV, Id targetV) { public void startOneLayer(boolean forward) { this.forward = forward; - this.currentLayer = new IntObjectHashMap<>(); + this.currentLayer = new IntObjectHashMap<>(INIT_CAPACITY); this.lastLayerKeys = this.forward ? - this.sourceLayers.peek().keySet().intIterator() : - this.targetLayers.peek().keySet().intIterator(); + this.sourceLayers.peek().keySet().intIterator() : + this.targetLayers.peek().keySet().intIterator(); } public void finishOneLayer() { @@ -106,8 +106,8 @@ public PathSet findPath(Id target, Function filter, // If cross point exists, path found, concat them if (this.contains(targetCode)) { results = this.forward ? - this.concatPath(this.current, targetCode, ring) : - this.concatPath(targetCode, this.current, ring); + this.linkPath(this.current, targetCode, ring) : + this.linkPath(targetCode, this.current, ring); } this.addPath(targetCode, this.current); return results; @@ -131,10 +131,10 @@ private boolean targetContains(int node) { } @Watched - private PathSet concatPath(int source, int target, boolean ring) { + private PathSet linkPath(int source, int target, boolean ring) { PathSet results = new PathSet(); - PathSet sources = this.getSourcePath(source); - PathSet targets = this.getTargetPath(target); + PathSet sources = this.linkSourcePath(source); + PathSet targets = this.linkTargetPath(target); for (Path tpath : targets) { tpath.reverse(); for (Path spath : sources) { @@ -153,8 +153,18 @@ private PathSet concatPath(int source, int target, boolean ring) { return results; } - private PathSet concatPath(Stack> all, - int id, int layerIndex) { + private PathSet linkSourcePath(int source) { + return this.linkPath(this.sourceLayers, source, + this.sourceLayers.size() - 1); + } + + private PathSet linkTargetPath(int target) { + return this.linkPath(this.targetLayers, target, + this.targetLayers.size() - 1); + } + + private PathSet linkPath(Stack> all, + int id, int layerIndex) { PathSet results = new PathSet(); if (layerIndex == 0) { Id sid = this.id(id); @@ -168,7 +178,7 @@ private PathSet concatPath(Stack> all, IntIterator iterator = parents.intIterator(); while (iterator.hasNext()) { int parent = iterator.next(); - PathSet paths = this.concatPath(all, parent, layerIndex - 1); + PathSet paths = this.linkPath(all, parent, layerIndex - 1); for (Iterator iter = paths.iterator(); iter.hasNext();) { Path path = iter.next(); if (path.vertices().contains(sid)) { @@ -183,16 +193,6 @@ private PathSet concatPath(Stack> all, return results; } - private PathSet getSourcePath(int source) { - return this.concatPath(this.sourceLayers, source, - this.sourceLayers.size() - 1); - } - - private PathSet getTargetPath(int target) { - return this.concatPath(this.targetLayers, target, - this.targetLayers.size() - 1); - } - @Watched private void addPath(int current, int parent) { if (this.currentLayer.containsKey(current)) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Records.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Records.java index 405ed933eb..40cd11c820 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Records.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Records.java @@ -26,6 +26,8 @@ public interface Records { + static final int INIT_CAPACITY = 16; + public void startOneLayer(boolean forward); public void finishOneLayer(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java index 91e57f2d0e..fcb9f1bf0b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java @@ -20,7 +20,6 @@ package com.baidu.hugegraph.traversal.algorithm.records; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Stack; import java.util.function.Function; @@ -36,13 +35,13 @@ public class ShortestPathRecord implements Records { - private ObjectIntMapping idMapping; + private final ObjectIntMapping idMapping; private final Stack sourceLayers; private final Stack targetLayers; private IntIntHashMap currentLayer; - private IntIterator iterator; + private IntIterator lastLayerKeys; private int current; private boolean forward; @@ -74,10 +73,10 @@ public ShortestPathRecord(Id sourceV, Id targetV) { public void startOneLayer(boolean forward) { this.forward = forward; - this.currentLayer = new IntIntHashMap(); - this.iterator = this.forward ? - this.sourceLayers.peek().keySet().intIterator() : - this.targetLayers.peek().keySet().intIterator(); + this.currentLayer = new IntIntHashMap(INIT_CAPACITY); + this.lastLayerKeys = this.forward ? + this.sourceLayers.peek().keySet().intIterator() : + this.targetLayers.peek().keySet().intIterator(); } public void finishOneLayer() { @@ -90,11 +89,11 @@ public void finishOneLayer() { } public boolean hasNextKey() { - return this.iterator.hasNext(); + return this.lastLayerKeys.hasNext(); } public Id nextKey() { - this.current = this.iterator.next(); + this.current = this.lastLayerKeys.next(); return this.id(current); } @@ -105,15 +104,14 @@ public PathSet findPath(Id target, Function filter, int targetCode = this.code(target); // If cross point exists, shortest path found, concat them if (this.contains(targetCode)) { - if (filter.apply(target)) { + if (!filter.apply(target)) { return paths; } - Path path = this.forward ? this.getPath(this.current, targetCode) : - this.getPath(targetCode, this.current); + paths.add(this.forward ? this.linkPath(this.current, targetCode) : + this.linkPath(targetCode, this.current)); this.foundPath = true; if (!all) { - paths.add(path); return paths; } } @@ -156,24 +154,30 @@ private void addOneStep(int source, int target) { this.currentLayer.put(target, source); } - private Path getPath(int source, int target) { - int sourceLayerSize = this.sourceLayers.size(); - int targetLayerSize = this.targetLayers.size(); + private Path linkPath(int source, int target) { + Path sourcePath = this.linkSourcePath(source); + Path targetPath = this.linkTargetPath(target); + sourcePath.reverse(); + List ids = new ArrayList<>(sourcePath.vertices()); + ids.addAll(targetPath.vertices()); + return new Path(ids); + } - List ids = new ArrayList<>(sourceLayerSize + targetLayerSize); + private Path linkSourcePath(int source) { + return this.linkPath(this.sourceLayers, source); + } - ids.add(this.id(source)); - int value = source; - for (int i = sourceLayerSize - 1; i > 0 ; i--) { - IntIntHashMap layer = this.sourceLayers.elementAt(i); - value = layer.get(value); - ids.add(this.id(value)); - } - Collections.reverse(ids); - ids.add(this.id(target)); - value = target; - for (int i = this.targetLayers.size() - 1; i > 0 ; i--) { - IntIntHashMap layer = this.targetLayers.elementAt(i); + private Path linkTargetPath(int target) { + return this.linkPath(this.targetLayers, target); + } + + private Path linkPath(Stack all, int node) { + int size = all.size(); + List ids = new ArrayList<>(size); + ids.add(this.id(node)); + int value = node; + for (int i = size - 1; i > 0 ; i--) { + IntIntHashMap layer = all.elementAt(i); value = layer.get(value); ids.add(this.id(value)); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Int2IntsMap.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/Int2IntsMap.java similarity index 79% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Int2IntsMap.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/Int2IntsMap.java index 4f5e85a21c..ba0493ee01 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Int2IntsMap.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/Int2IntsMap.java @@ -17,22 +17,24 @@ * under the License. */ -package com.baidu.hugegraph.traversal.algorithm.records; +package com.baidu.hugegraph.util.collection; import org.eclipse.collections.api.iterator.IntIterator; import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; public class Int2IntsMap { - private static final int INIT_KEY_CAPACITY = 1000000; + private static final int INIT_KEY_CAPACITY = 16; private static final int CHUNK_SIZE = 10; - private static final int POSITION_OFFSET = 0; - private static final int SIZE_OFFSET = 1; - private static final int FIRST_CHUNK_DATA_OFFSET = 2; private static final int EXPANSION_FACTOR = 2; + private static final int OFFSET_POSITION = 0; + private static final int OFFSET_SIZE = 1; + private static final int OFFSET_DATA_IN_FIRST_CHUNK = 2; + + /* - * firstChunkMap chunkTable + * chunkMap chunkTable * * -------- --------------- * | 1 | 0 |--------->0 | 33 | (nextPosition) @@ -80,48 +82,48 @@ public class Int2IntsMap { * */ - private IntIntHashMap firstChunkMap; + private IntIntHashMap chunkMap; private int[] chunkTable; private int nextChunk; public Int2IntsMap() { - this.firstChunkMap = new IntIntHashMap(INIT_KEY_CAPACITY); + this.chunkMap = new IntIntHashMap(INIT_KEY_CAPACITY); this.chunkTable = new int[INIT_KEY_CAPACITY * CHUNK_SIZE]; this.nextChunk = 0; } public void add(int key, int value) { - if (this.firstChunkMap.containsKey(key)) { - int firstChunk = this.firstChunkMap.get(key); - int nextPosition = this.chunkTable[firstChunk + POSITION_OFFSET]; + if (this.chunkMap.containsKey(key)) { + int firstChunk = this.chunkMap.get(key); + int nextPosition = this.chunkTable[firstChunk + OFFSET_POSITION]; if (!this.endOfChunk(nextPosition)) { chunkTable[nextPosition] = value; - this.chunkTable[firstChunk + POSITION_OFFSET]++; + this.chunkTable[firstChunk + OFFSET_POSITION]++; } else { this.ensureCapacity(); this.chunkTable[nextPosition] = this.nextChunk; this.chunkTable[this.nextChunk] = value; - this.chunkTable[firstChunk + POSITION_OFFSET] = this.nextChunk + 1; + this.chunkTable[firstChunk + OFFSET_POSITION] = this.nextChunk + 1; // Update next block this.nextChunk += CHUNK_SIZE; } - this.chunkTable[firstChunk + SIZE_OFFSET]++; + this.chunkTable[firstChunk + OFFSET_SIZE]++; } else { // New key, allocate 1st chunk and init this.ensureCapacity(); // Allocate 1st chunk - this.firstChunkMap.put(key, this.nextChunk); + this.chunkMap.put(key, this.nextChunk); // Init first chunk this.chunkTable[this.nextChunk] = this.nextChunk + - FIRST_CHUNK_DATA_OFFSET + 1; - this.chunkTable[this.nextChunk + SIZE_OFFSET] = 1; - this.chunkTable[this.nextChunk + FIRST_CHUNK_DATA_OFFSET] = value; + OFFSET_DATA_IN_FIRST_CHUNK + 1; + this.chunkTable[this.nextChunk + OFFSET_SIZE] = 1; + this.chunkTable[this.nextChunk + OFFSET_DATA_IN_FIRST_CHUNK] = value; // Update next block this.nextChunk += CHUNK_SIZE; @@ -129,15 +131,15 @@ public void add(int key, int value) { } public boolean containsKey(int key) { - return this.firstChunkMap.containsKey(key); + return this.chunkMap.containsKey(key); } public int[] get(int key) { - int firstChunk = this.firstChunkMap.get(key); - int size = this.chunkTable[firstChunk + SIZE_OFFSET]; + int firstChunk = this.chunkMap.get(key); + int size = this.chunkTable[firstChunk + OFFSET_SIZE]; int[] values = new int[size]; int i = 0; - int position = firstChunk + FIRST_CHUNK_DATA_OFFSET; + int position = firstChunk + OFFSET_DATA_IN_FIRST_CHUNK; while (i < size) { if (!this.endOfChunk(position)) { values[i++] = this.chunkTable[position++]; @@ -149,11 +151,11 @@ public int[] get(int key) { } public IntIterator keyIterator() { - return this.firstChunkMap.keySet().intIterator(); + return this.chunkMap.keySet().intIterator(); } public int size() { - return this.firstChunkMap.size(); + return this.chunkMap.size(); } private boolean endOfChunk(int position) { diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java index e1e8673a6b..7071d960be 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java @@ -36,6 +36,7 @@ import com.baidu.hugegraph.unit.core.DataTypeTest; import com.baidu.hugegraph.unit.core.DirectionsTest; import com.baidu.hugegraph.unit.core.ExceptionTest; +import com.baidu.hugegraph.unit.core.Int2IntsMapTest; import com.baidu.hugegraph.unit.core.LocksTableTest; import com.baidu.hugegraph.unit.core.PageStateTest; import com.baidu.hugegraph.unit.core.QueryTest; @@ -103,6 +104,7 @@ BackendStoreSystemInfoTest.class, TraversalUtilTest.class, PageStateTest.class, + Int2IntsMapTest.class, /* serializer */ BytesBufferTest.class, diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java index 6ecf5c29fe..16f41e1267 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java @@ -29,7 +29,7 @@ import org.junit.Test; import com.baidu.hugegraph.testutil.Assert; -import com.baidu.hugegraph.traversal.algorithm.records.Int2IntsMap; +import com.baidu.hugegraph.util.collection.Int2IntsMap; public class Int2IntsMapTest { From 42344582596d29306186fa5853ec511a0c28b937 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 13 Apr 2021 17:47:39 +0800 Subject: [PATCH 20/38] improve Change-Id: I7509a653ad429d329165e968358cf4d01bf86742 --- .../traversal/algorithm/PathsTraverser.java | 6 +- .../algorithm/records/ArrayRecord.java | 79 +++++++ .../algorithm/records/IntSetRecord.java | 62 +++++ .../records/MultiPathsBySetRecords.java | 214 ------------------ ...rayRecords.java => MultiPathsRecords.java} | 42 ++-- .../traversal/algorithm/records/Record.java | 35 +++ .../algorithm/records/RecordFactory.java | 38 ++++ .../algorithm/records/RecordType.java | 63 ++++++ 8 files changed, 303 insertions(+), 236 deletions(-) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayRecord.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntSetRecord.java delete mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsBySetRecords.java rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/{MultiPathsByArrayRecords.java => MultiPathsRecords.java} (85%) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordFactory.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java index 12a49eeffb..19898595da 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java @@ -27,7 +27,7 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeEdge; -import com.baidu.hugegraph.traversal.algorithm.records.MultiPathsByArrayRecords; +import com.baidu.hugegraph.traversal.algorithm.records.MultiPathsRecords; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; import com.google.common.collect.ImmutableList; @@ -82,7 +82,7 @@ public PathSet paths(Id sourceV, Directions sourceDir, private class Traverser { - private final MultiPathsByArrayRecords record; + private final MultiPathsRecords record; private final Id label; private final long degree; @@ -93,7 +93,7 @@ private class Traverser { public Traverser(Id sourceV, Id targetV, Id label, long degree, long capacity, long limit) { - this.record = new MultiPathsByArrayRecords(sourceV, targetV); + this.record = new MultiPathsRecords(sourceV, targetV); this.label = label; this.degree = degree; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayRecord.java new file mode 100644 index 0000000000..f142d7d0d4 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayRecord.java @@ -0,0 +1,79 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import org.eclipse.collections.api.iterator.IntIterator; + +import com.baidu.hugegraph.util.collection.Int2IntsMap; + +public class ArrayRecord implements Record { + + private Int2IntsMap layer; + + public ArrayRecord() { + this.layer = new Int2IntsMap(); + } + + @Override + public IntIterator keys() { + return this.layer.keyIterator(); + } + + @Override + public boolean containsKey(int key) { + return this.layer.containsKey(key); + } + + @Override + public IntIterator get(int key) { + return new IntArrayIterator(this.layer.get(key)); + } + + @Override + public void addPath(int node, int parent) { + this.layer.add(node, parent); + } + + @Override + public int size() { + return this.layer.size(); + } + + public class IntArrayIterator implements IntIterator { + + private int[] array; + private int index; + + public IntArrayIterator(int[] array) { + this.array = array; + this.index = 0; + } + + @Override + public int next() { + return this.array[index++]; + } + + @Override + public boolean hasNext() { + return this.index < this.array.length; + } + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntSetRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntSetRecord.java new file mode 100644 index 0000000000..9c0cc561a9 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntSetRecord.java @@ -0,0 +1,62 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; +import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; + +public class IntSetRecord implements Record { + + private IntObjectHashMap layer; + + public IntSetRecord() { + this.layer = new IntObjectHashMap<>(); + } + + @Override + public IntIterator keys() { + return this.layer.keySet().intIterator(); + } + + + @Override + public boolean containsKey(int key) { + return this.layer.containsKey(key); + } + + @Override public IntIterator get(int key) { + return this.layer.get(key).intIterator(); + } + + @Override + public void addPath(int node, int parent) { + if (this.layer.containsKey(node)) { + this.layer.get(node).add(parent); + } else { + this.layer.put(node, IntHashSet.newSetWith(parent)); + } + } + + @Override + public int size() { + return this.layer.size(); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsBySetRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsBySetRecords.java deleted file mode 100644 index 7a97b5ff68..0000000000 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsBySetRecords.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright 2017 HugeGraph Authors - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.baidu.hugegraph.traversal.algorithm.records; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Stack; -import java.util.function.Function; - -import org.apache.commons.collections.CollectionUtils; -import org.eclipse.collections.api.iterator.IntIterator; -import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; -import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; - -import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.perf.PerfUtil.Watched; -import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; -import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; -import com.baidu.hugegraph.util.collection.ObjectIntMapping; -import com.google.common.collect.Lists; - -public class MultiPathsBySetRecords implements Records { - - private final ObjectIntMapping idMapping; - - private final Stack> sourceLayers; - private final Stack> targetLayers; - - private IntObjectHashMap currentLayer; - private IntIterator lastLayerKeys; - private int current; - private boolean forward; - - private int accessed; - - public MultiPathsBySetRecords(Id sourceV, Id targetV) { - - this.idMapping = new ObjectIntMapping(); - - int sourceCode = this.code(sourceV); - int targetCode = this.code(targetV); - IntObjectHashMap firstSourceLayer = new IntObjectHashMap<>(); - IntObjectHashMap firstTargetLayer = new IntObjectHashMap<>(); - firstSourceLayer.put(sourceCode, new IntHashSet()); - firstTargetLayer.put(targetCode, new IntHashSet()); - this.sourceLayers = new Stack<>(); - this.targetLayers = new Stack<>(); - this.sourceLayers.push(firstSourceLayer); - this.targetLayers.push(firstTargetLayer); - - this.accessed = 2; - } - - public void startOneLayer(boolean forward) { - this.forward = forward; - this.currentLayer = new IntObjectHashMap<>(INIT_CAPACITY); - this.lastLayerKeys = this.forward ? - this.sourceLayers.peek().keySet().intIterator() : - this.targetLayers.peek().keySet().intIterator(); - } - - public void finishOneLayer() { - if (this.forward) { - this.sourceLayers.push(this.currentLayer); - } else { - this.targetLayers.push(this.currentLayer); - } - this.accessed += this.currentLayer.size(); - } - - @Watched - public boolean hasNextKey() { - return this.lastLayerKeys.hasNext(); - } - - @Watched - public Id nextKey() { - this.current = this.lastLayerKeys.next(); - return this.id(current); - } - - @Watched - public PathSet findPath(Id target, Function filter, - boolean all, boolean ring) { - assert all; - PathSet results = new PathSet(); - int targetCode = this.code(target); - // If cross point exists, path found, concat them - if (this.contains(targetCode)) { - results = this.forward ? - this.linkPath(this.current, targetCode, ring) : - this.linkPath(targetCode, this.current, ring); - } - this.addPath(targetCode, this.current); - return results; - } - - public long accessed() { - return this.accessed; - } - - private boolean contains(int node) { - return this.forward ? this.targetContains(node) : - this.sourceContains(node); - } - - private boolean sourceContains(int node) { - return this.sourceLayers.peek().containsKey(node); - } - - private boolean targetContains(int node) { - return this.targetLayers.peek().containsKey(node); - } - - @Watched - private PathSet linkPath(int source, int target, boolean ring) { - PathSet results = new PathSet(); - PathSet sources = this.linkSourcePath(source); - PathSet targets = this.linkTargetPath(target); - for (Path tpath : targets) { - tpath.reverse(); - for (Path spath : sources) { - if (!ring) { - // Avoid loop in path - if (CollectionUtils.containsAny(spath.vertices(), - tpath.vertices())) { - continue; - } - } - List ids = new ArrayList<>(spath.vertices()); - ids.addAll(tpath.vertices()); - results.add(new Path(ids)); - } - } - return results; - } - - private PathSet linkSourcePath(int source) { - return this.linkPath(this.sourceLayers, source, - this.sourceLayers.size() - 1); - } - - private PathSet linkTargetPath(int target) { - return this.linkPath(this.targetLayers, target, - this.targetLayers.size() - 1); - } - - private PathSet linkPath(Stack> all, - int id, int layerIndex) { - PathSet results = new PathSet(); - if (layerIndex == 0) { - Id sid = this.id(id); - results.add(new Path(Lists.newArrayList(sid))); - return results; - } - - Id sid = this.id(id); - IntObjectHashMap layer = all.elementAt(layerIndex); - IntHashSet parents = layer.get(id); - IntIterator iterator = parents.intIterator(); - while (iterator.hasNext()) { - int parent = iterator.next(); - PathSet paths = this.linkPath(all, parent, layerIndex - 1); - for (Iterator iter = paths.iterator(); iter.hasNext();) { - Path path = iter.next(); - if (path.vertices().contains(sid)) { - iter.remove(); - continue; - } - path.addToEnd(sid); - } - - results.addAll(paths); - } - return results; - } - - @Watched - private void addPath(int current, int parent) { - if (this.currentLayer.containsKey(current)) { - this.currentLayer.get(current).add(parent); - } else { - this.currentLayer.put(current, IntHashSet.newSetWith(parent)); - } - } - - @Watched - private int code(Id id) { - return this.idMapping.object2Code(id); - } - - @Watched - private Id id(int code) { - return (Id) this.idMapping.code2Object(code); - } -} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsByArrayRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java similarity index 85% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsByArrayRecords.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java index 81d09c0ae7..b0e82f283d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsByArrayRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java @@ -32,33 +32,32 @@ import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; -import com.baidu.hugegraph.util.collection.Int2IntsMap; import com.baidu.hugegraph.util.collection.ObjectIntMapping; import com.google.common.collect.Lists; -public class MultiPathsByArrayRecords implements Records { +public class MultiPathsRecords implements Records { private final ObjectIntMapping idMapping; - private final Stack sourceLayers; - private final Stack targetLayers; + private final Stack sourceLayers; + private final Stack targetLayers; - private Int2IntsMap currentLayer; + private Record currentLayer; private IntIterator lastLayerKeys; private int current; private boolean forward; private int accessed; - public MultiPathsByArrayRecords(Id sourceV, Id targetV) { + public MultiPathsRecords(Id sourceV, Id targetV) { this.idMapping = new ObjectIntMapping(); int sourceCode = this.code(sourceV); int targetCode = this.code(targetV); - Int2IntsMap firstSourceLayer = new Int2IntsMap(); - Int2IntsMap firstTargetLayer = new Int2IntsMap(); - firstSourceLayer.add(sourceCode, 0); - firstTargetLayer.add(targetCode, 0); + Record firstSourceLayer = this.newLayer(); + Record firstTargetLayer = this.newLayer(); + firstSourceLayer.addPath(sourceCode, 0); + firstTargetLayer.addPath(targetCode, 0); this.sourceLayers = new Stack<>(); this.targetLayers = new Stack<>(); this.sourceLayers.push(firstSourceLayer); @@ -69,10 +68,9 @@ public MultiPathsByArrayRecords(Id sourceV, Id targetV) { public void startOneLayer(boolean forward) { this.forward = forward; - this.currentLayer = new Int2IntsMap(); - this.lastLayerKeys = this.forward ? - this.sourceLayers.peek().keyIterator() : - this.targetLayers.peek().keyIterator(); + this.currentLayer = newLayer(); + this.lastLayerKeys = this.forward ? this.sourceLayers.peek().keys() : + this.targetLayers.peek().keys(); } public void finishOneLayer() { @@ -130,6 +128,7 @@ private boolean targetContains(int node) { @Watched private PathSet linkPath(int source, int target, boolean ring) { + PathSet results = new PathSet(); PathSet sources = this.linkSourcePath(source); PathSet targets = this.linkTargetPath(target); @@ -161,7 +160,7 @@ private PathSet linkTargetPath(int target) { this.targetLayers.size() - 1); } - private PathSet linkPath(Stack all, int id, int layerIndex) { + private PathSet linkPath(Stack all, int id, int layerIndex) { PathSet results = new PathSet(); if (layerIndex == 0) { Id sid = this.id(id); @@ -170,9 +169,10 @@ private PathSet linkPath(Stack all, int id, int layerIndex) { } Id sid = this.id(id); - Int2IntsMap layer = all.elementAt(layerIndex); - int[] parents = layer.get(id); - for (int parent : parents) { + Record layer = all.elementAt(layerIndex); + IntIterator iterator = layer.get(id); + while (iterator.hasNext()) { + int parent = iterator.next(); PathSet paths = this.linkPath(all, parent, layerIndex - 1); for (Iterator iter = paths.iterator(); iter.hasNext();) { Path path = iter.next(); @@ -190,7 +190,7 @@ private PathSet linkPath(Stack all, int id, int layerIndex) { @Watched private void addPath(int current, int parent) { - this.currentLayer.add(current, parent); + this.currentLayer.addPath(current, parent); } @Watched @@ -202,4 +202,8 @@ private int code(Id id) { private Id id(int code) { return (Id) this.idMapping.code2Object(code); } + + public Record newLayer() { + return RecordFactory.newRecord(); + } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java new file mode 100644 index 0000000000..aec0b5241b --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java @@ -0,0 +1,35 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import org.eclipse.collections.api.iterator.IntIterator; + +public interface Record { + + public abstract IntIterator keys(); + + public abstract boolean containsKey(int key); + + public abstract IntIterator get(int key); + + public abstract void addPath(int node, int parent); + + public abstract int size(); +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordFactory.java new file mode 100644 index 0000000000..1e943ad503 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordFactory.java @@ -0,0 +1,38 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +public class RecordFactory { + + public static Record newRecord() { + return new ArrayRecord(); + } + + public static Record newRecord(RecordType type) { + switch (type) { + case ARRAY: + return new ArrayRecord(); + case SET: + return new IntSetRecord(); + default: + throw new AssertionError("Unsupported record type: " + type); + } + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java new file mode 100644 index 0000000000..2b8ec8baeb --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java @@ -0,0 +1,63 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import com.baidu.hugegraph.type.define.SerialEnum; + +public enum RecordType implements SerialEnum { + + // Java Collection Framework + ARRAY(1, "array"), + + // Eclipse Collection + SET(2, "set"); + + private final byte code; + private final String name; + + static { + SerialEnum.register(RecordType.class); + } + + RecordType(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } + + public static RecordType fromCode(byte code) { + switch (code) { + case 1: + return ARRAY; + case 2: + return SET; + default: + throw new AssertionError("Unsupported record code: " + code); + } + } +} \ No newline at end of file From fca15512f6f6e505ba980fd551bd9fbce120efaa Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 13 Apr 2021 18:48:18 +0800 Subject: [PATCH 21/38] extract Record and Records for paths and shortest path Change-Id: I4b79d0987f1eebcfb952bc4627a2f4fc8d2dcaf3 --- .../traversal/algorithm/PathsTraverser.java | 5 +- .../algorithm/ShortestPathTraverser.java | 10 +- .../{ArrayRecord.java => IntArrayRecord.java} | 4 +- .../algorithm/records/IntIntRecord.java | 61 ++++++ .../algorithm/records/MultiPathsRecords.java | 76 +++---- .../algorithm/records/RecordFactory.java | 8 +- .../algorithm/records/RecordType.java | 4 +- .../traversal/algorithm/records/Records.java | 2 - .../algorithm/records/ShortestPathRecord.java | 194 ------------------ .../records/ShortestPathRecords.java | 111 ++++++++++ 10 files changed, 225 insertions(+), 250 deletions(-) rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/{ArrayRecord.java => IntArrayRecord.java} (96%) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntIntRecord.java delete mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java index 19898595da..48dbc47a8d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java @@ -28,6 +28,7 @@ import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeEdge; import com.baidu.hugegraph.traversal.algorithm.records.MultiPathsRecords; +import com.baidu.hugegraph.traversal.algorithm.records.RecordType; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; import com.google.common.collect.ImmutableList; @@ -93,8 +94,8 @@ private class Traverser { public Traverser(Id sourceV, Id targetV, Id label, long degree, long capacity, long limit) { - this.record = new MultiPathsRecords(sourceV, targetV); - + this.record = new MultiPathsRecords(sourceV, targetV, + RecordType.ARRAY); this.label = label; this.degree = degree; this.capacity = capacity; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java index 91a7b437cf..151928964b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/ShortestPathTraverser.java @@ -30,18 +30,16 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeEdge; -import com.baidu.hugegraph.traversal.algorithm.records.ShortestPathRecord; +import com.baidu.hugegraph.traversal.algorithm.records.ShortestPathRecords; import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; -import com.baidu.hugegraph.util.collection.ObjectIntMapping; import com.google.common.collect.ImmutableList; public class ShortestPathTraverser extends HugeTraverser { public ShortestPathTraverser(HugeGraph graph) { super(graph); - this.idMapping = new ObjectIntMapping(); } @Watched @@ -142,7 +140,7 @@ public PathSet allShortestPaths(Id sourceV, Id targetV, Directions dir, private class Traverser { - private ShortestPathRecord record; + private ShortestPathRecords record; private final Directions direction; private final Map labels; private final long degree; @@ -152,7 +150,7 @@ private class Traverser { public Traverser(Id sourceV, Id targetV, Directions dir, Map labels, long degree, long skipDegree, long capacity) { - this.record = new ShortestPathRecord(sourceV, targetV); + this.record = new ShortestPathRecords(sourceV, targetV); this.direction = dir; this.labels = labels; this.degree = degree; @@ -221,7 +219,7 @@ public PathSet backward(boolean all) { Id target = edge.id().otherVertexId(); PathSet paths = this.record.findPath(target, - t -> this.superNode(t, opposite), + t -> !this.superNode(t, opposite), all, false); if (paths.isEmpty()) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntArrayRecord.java similarity index 96% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayRecord.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntArrayRecord.java index f142d7d0d4..a67e4d344b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ArrayRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntArrayRecord.java @@ -23,11 +23,11 @@ import com.baidu.hugegraph.util.collection.Int2IntsMap; -public class ArrayRecord implements Record { +public class IntArrayRecord implements Record { private Int2IntsMap layer; - public ArrayRecord() { + public IntArrayRecord() { this.layer = new Int2IntsMap(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntIntRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntIntRecord.java new file mode 100644 index 0000000000..5f9c06951c --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntIntRecord.java @@ -0,0 +1,61 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; + +public class IntIntRecord implements Record { + + private IntIntHashMap layer; + + public IntIntRecord() { + this.layer = new IntIntHashMap(); + } + + @Override + public IntIterator keys() { + return this.layer.keySet().intIterator(); + } + + @Override + public boolean containsKey(int key) { + return this.layer.containsKey(key); + } + + @Override + public IntIterator get(int key) { + return null; + } + + @Override + public void addPath(int node, int parent) { + this.layer.put(node, parent); + } + + @Override + public int size() { + return this.layer.size(); + } + + public IntIntHashMap layer() { + return this.layer; + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java index b0e82f283d..6763a80181 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java @@ -38,58 +38,59 @@ public class MultiPathsRecords implements Records { private final ObjectIntMapping idMapping; + private final RecordType type; - private final Stack sourceLayers; - private final Stack targetLayers; - - private Record currentLayer; - private IntIterator lastLayerKeys; - private int current; - private boolean forward; + protected final Stack sourceRecords; + protected final Stack targetRecords; + protected Record currentRecord; + private IntIterator lastRecordKeys; + protected int current; + protected boolean forward; private int accessed; - public MultiPathsRecords(Id sourceV, Id targetV) { + public MultiPathsRecords(Id sourceV, Id targetV, RecordType type) { this.idMapping = new ObjectIntMapping(); + this.type = type; int sourceCode = this.code(sourceV); int targetCode = this.code(targetV); - Record firstSourceLayer = this.newLayer(); - Record firstTargetLayer = this.newLayer(); - firstSourceLayer.addPath(sourceCode, 0); - firstTargetLayer.addPath(targetCode, 0); - this.sourceLayers = new Stack<>(); - this.targetLayers = new Stack<>(); - this.sourceLayers.push(firstSourceLayer); - this.targetLayers.push(firstTargetLayer); + Record firstSourceRecord = this.newRecord(); + Record firstTargetRecord = this.newRecord(); + firstSourceRecord.addPath(sourceCode, 0); + firstTargetRecord.addPath(targetCode, 0); + this.sourceRecords = new Stack<>(); + this.targetRecords = new Stack<>(); + this.sourceRecords.push(firstSourceRecord); + this.targetRecords.push(firstTargetRecord); this.accessed = 2; } public void startOneLayer(boolean forward) { this.forward = forward; - this.currentLayer = newLayer(); - this.lastLayerKeys = this.forward ? this.sourceLayers.peek().keys() : - this.targetLayers.peek().keys(); + this.currentRecord = this.newRecord(); + this.lastRecordKeys = this.forward ? this.sourceRecords.peek().keys() : + this.targetRecords.peek().keys(); } public void finishOneLayer() { if (this.forward) { - this.sourceLayers.push(this.currentLayer); + this.sourceRecords.push(this.currentRecord); } else { - this.targetLayers.push(this.currentLayer); + this.targetRecords.push(this.currentRecord); } - this.accessed += this.currentLayer.size(); + this.accessed += this.currentRecord.size(); } @Watched public boolean hasNextKey() { - return this.lastLayerKeys.hasNext(); + return this.lastRecordKeys.hasNext(); } @Watched public Id nextKey() { - this.current = this.lastLayerKeys.next(); + this.current = this.lastRecordKeys.next(); return this.id(current); } @@ -113,22 +114,21 @@ public long accessed() { return this.accessed; } - private boolean contains(int node) { + protected boolean contains(int node) { return this.forward ? this.targetContains(node) : this.sourceContains(node); } private boolean sourceContains(int node) { - return this.sourceLayers.peek().containsKey(node); + return this.sourceRecords.peek().containsKey(node); } private boolean targetContains(int node) { - return this.targetLayers.peek().containsKey(node); + return this.targetRecords.peek().containsKey(node); } @Watched private PathSet linkPath(int source, int target, boolean ring) { - PathSet results = new PathSet(); PathSet sources = this.linkSourcePath(source); PathSet targets = this.linkTargetPath(target); @@ -151,13 +151,13 @@ private PathSet linkPath(int source, int target, boolean ring) { } private PathSet linkSourcePath(int source) { - return this.linkPath(this.sourceLayers, source, - this.sourceLayers.size() - 1); + return this.linkPath(this.sourceRecords, source, + this.sourceRecords.size() - 1); } private PathSet linkTargetPath(int target) { - return this.linkPath(this.targetLayers, target, - this.targetLayers.size() - 1); + return this.linkPath(this.targetRecords, target, + this.targetRecords.size() - 1); } private PathSet linkPath(Stack all, int id, int layerIndex) { @@ -189,21 +189,21 @@ private PathSet linkPath(Stack all, int id, int layerIndex) { } @Watched - private void addPath(int current, int parent) { - this.currentLayer.addPath(current, parent); + protected void addPath(int current, int parent) { + this.currentRecord.addPath(current, parent); } @Watched - private int code(Id id) { + protected int code(Id id) { return this.idMapping.object2Code(id); } @Watched - private Id id(int code) { + protected Id id(int code) { return (Id) this.idMapping.code2Object(code); } - public Record newLayer() { - return RecordFactory.newRecord(); + private Record newRecord() { + return RecordFactory.newRecord(this.type); } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordFactory.java index 1e943ad503..563c820ed3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordFactory.java @@ -21,16 +21,14 @@ public class RecordFactory { - public static Record newRecord() { - return new ArrayRecord(); - } - public static Record newRecord(RecordType type) { switch (type) { case ARRAY: - return new ArrayRecord(); + return new IntArrayRecord(); case SET: return new IntSetRecord(); + case INT: + return new IntIntRecord(); default: throw new AssertionError("Unsupported record type: " + type); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java index 2b8ec8baeb..9a1adbd2f1 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java @@ -27,7 +27,9 @@ public enum RecordType implements SerialEnum { ARRAY(1, "array"), // Eclipse Collection - SET(2, "set"); + SET(2, "set"), + + INT(3, "int"); private final byte code; private final String name; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Records.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Records.java index 40cd11c820..405ed933eb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Records.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Records.java @@ -26,8 +26,6 @@ public interface Records { - static final int INIT_CAPACITY = 16; - public void startOneLayer(boolean forward); public void finishOneLayer(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java deleted file mode 100644 index fcb9f1bf0b..0000000000 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecord.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright 2017 HugeGraph Authors - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.baidu.hugegraph.traversal.algorithm.records; - -import java.util.ArrayList; -import java.util.List; -import java.util.Stack; -import java.util.function.Function; - -import org.eclipse.collections.api.iterator.IntIterator; -import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; -import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; - -import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; -import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; -import com.baidu.hugegraph.util.collection.ObjectIntMapping; - -public class ShortestPathRecord implements Records { - - private final ObjectIntMapping idMapping; - - private final Stack sourceLayers; - private final Stack targetLayers; - - private IntIntHashMap currentLayer; - private IntIterator lastLayerKeys; - private int current; - private boolean forward; - - private final IntHashSet accessed; - private long size; - private boolean foundPath; - - public ShortestPathRecord(Id sourceV, Id targetV) { - - this.idMapping = new ObjectIntMapping(); - - int sourceCode = this.code(sourceV); - int targetCode = this.code(targetV); - IntIntHashMap firstSourceLayer = new IntIntHashMap(); - IntIntHashMap firstTargetLayer = new IntIntHashMap(); - firstSourceLayer.put(sourceCode, 0); - firstTargetLayer.put(targetCode, 0); - this.sourceLayers = new Stack<>(); - this.targetLayers = new Stack<>(); - this.sourceLayers.push(firstSourceLayer); - this.targetLayers.push(firstTargetLayer); - - this.accessed = new IntHashSet(); - this.accessed.add(sourceCode); - this.accessed.add(targetCode); - - this.size = 2L; - } - - public void startOneLayer(boolean forward) { - this.forward = forward; - this.currentLayer = new IntIntHashMap(INIT_CAPACITY); - this.lastLayerKeys = this.forward ? - this.sourceLayers.peek().keySet().intIterator() : - this.targetLayers.peek().keySet().intIterator(); - } - - public void finishOneLayer() { - if (this.forward) { - this.sourceLayers.push(this.currentLayer); - } else { - this.targetLayers.push(this.currentLayer); - } - this.size += this.currentLayer.size(); - } - - public boolean hasNextKey() { - return this.lastLayerKeys.hasNext(); - } - - public Id nextKey() { - this.current = this.lastLayerKeys.next(); - return this.id(current); - } - - public PathSet findPath(Id target, Function filter, - boolean all, boolean ring) { - assert !ring; - PathSet paths = new PathSet(); - int targetCode = this.code(target); - // If cross point exists, shortest path found, concat them - if (this.contains(targetCode)) { - if (!filter.apply(target)) { - return paths; - } - - paths.add(this.forward ? this.linkPath(this.current, targetCode) : - this.linkPath(targetCode, this.current)); - this.foundPath = true; - if (!all) { - return paths; - } - } - - /* - * Not found shortest path yet, node is added to - * newVertices if: - * 1. not in sources and newVertices yet - * 2. path of node doesn't have loop - */ - if (!this.foundPath && this.isNew(targetCode)) { - this.addOneStep(this.current, targetCode); - } - return paths; - } - - public long accessed() { - return this.size; - } - - private boolean contains(int node) { - return this.forward ? this.targetContains(node) : - this.sourceContains(node); - } - - private boolean sourceContains(int node) { - return this.sourceLayers.peek().containsKey(node); - } - - private boolean targetContains(int node) { - return this.targetLayers.peek().containsKey(node); - } - - private boolean isNew(int node) { - return !this.currentLayer.containsKey(node) && - !this.accessed.contains(node); - } - - private void addOneStep(int source, int target) { - this.currentLayer.put(target, source); - } - - private Path linkPath(int source, int target) { - Path sourcePath = this.linkSourcePath(source); - Path targetPath = this.linkTargetPath(target); - sourcePath.reverse(); - List ids = new ArrayList<>(sourcePath.vertices()); - ids.addAll(targetPath.vertices()); - return new Path(ids); - } - - private Path linkSourcePath(int source) { - return this.linkPath(this.sourceLayers, source); - } - - private Path linkTargetPath(int target) { - return this.linkPath(this.targetLayers, target); - } - - private Path linkPath(Stack all, int node) { - int size = all.size(); - List ids = new ArrayList<>(size); - ids.add(this.id(node)); - int value = node; - for (int i = size - 1; i > 0 ; i--) { - IntIntHashMap layer = all.elementAt(i); - value = layer.get(value); - ids.add(this.id(value)); - } - return new Path(ids); - } - - private int code(Id id) { - return this.idMapping.object2Code(id); - } - - private Id id(int code) { - return (Id) this.idMapping.code2Object(code); - } -} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java new file mode 100644 index 0000000000..06a51167a7 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java @@ -0,0 +1,111 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import java.util.ArrayList; +import java.util.List; +import java.util.Stack; +import java.util.function.Function; + +import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; +import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; + +public class ShortestPathRecords extends MultiPathsRecords { + + private final IntHashSet accessedVertices; + private boolean pathFound; + + public ShortestPathRecords(Id sourceV, Id targetV) { + super(sourceV, targetV, RecordType.INT); + + this.accessedVertices = new IntHashSet(); + this.accessedVertices.add(this.code(sourceV)); + this.accessedVertices.add(this.code(targetV)); + } + + @Override + public PathSet findPath(Id target, Function filter, + boolean all, boolean ring) { + assert !ring; + PathSet paths = new PathSet(); + int targetCode = this.code(target); + // If cross point exists, shortest path found, concat them + if (this.contains(targetCode)) { + if (!filter.apply(target)) { + return paths; + } + paths.add(this.forward ? this.linkPath(this.current, targetCode) : + this.linkPath(targetCode, this.current)); + this.pathFound = true; + if (!all) { + return paths; + } + } + /* + * Not found shortest path yet, node is added to + * newVertices if: + * 1. not in sources and newVertices yet + * 2. path of node doesn't have loop + */ + if (!this.pathFound && this.isNew(targetCode)) { + this.addPath(targetCode, this.current); + } + return paths; + } + + private boolean isNew(int node) { + return !this.currentRecord.containsKey(node) && + !this.accessedVertices.contains(node); + } + + private Path linkPath(int source, int target) { + Path sourcePath = this.linkSourcePath(source); + Path targetPath = this.linkTargetPath(target); + sourcePath.reverse(); + List ids = new ArrayList<>(sourcePath.vertices()); + ids.addAll(targetPath.vertices()); + return new Path(ids); + } + + private Path linkSourcePath(int source) { + return this.linkPath(this.sourceRecords, source); + } + + private Path linkTargetPath(int target) { + return this.linkPath(this.targetRecords, target); + } + + private Path linkPath(Stack all, int node) { + int size = all.size(); + List ids = new ArrayList<>(size); + ids.add(this.id(node)); + int value = node; + for (int i = size - 1; i > 0 ; i--) { + IntIntHashMap layer = ((IntIntRecord) all.elementAt(i)).layer(); + value = layer.get(value); + ids.add(this.id(value)); + } + return new Path(ids); + } +} From 8d9e3c36b7d6a9e49d690c0108e8e183e6c57b82 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Fri, 16 Apr 2021 20:33:33 +0800 Subject: [PATCH 22/38] adapt primitive collection for kout/kneighbor Change-Id: Iffad898d0ff04f8b8937eb774352c853bae87aef --- .../api/traversers/AllShortestPathsAPI.java | 3 +- .../api/traversers/CrosspointsAPI.java | 3 +- .../hugegraph/api/traversers/PathsAPI.java | 3 +- .../hugegraph/api/traversers/RaysAPI.java | 2 +- .../hugegraph/api/traversers/RingsAPI.java | 3 +- .../baidu/hugegraph/structure/HugeEdge.java | 4 +- .../baidu/hugegraph/structure/HugeVertex.java | 5 --- .../traversal/algorithm/HugeTraverser.java | 30 +++++---------- .../traversal/algorithm/KoutTraverser.java | 12 +++--- .../algorithm/records/IntArrayRecord.java | 4 +- .../algorithm/records/IntIntRecord.java | 2 +- .../algorithm/records/IntSetRecord.java | 2 +- .../algorithm/records/MultiPathsRecords.java | 4 +- .../algorithm/records/RecordType.java | 4 +- .../records/ShortestPathRecords.java | 1 + .../type/define/CollectionImplType.java | 2 + .../util/collection/CollectionFactory.java | 37 +++++++++++++++++++ .../util/collection/ObjectIntMapping.java | 10 +++-- .../hugegraph/unit/util/JsonUtilTest.java | 6 +-- 19 files changed, 80 insertions(+), 57 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java index b6de8187f1..cfbde599ee 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java @@ -89,7 +89,6 @@ public String get(@Context GraphManager manager, HugeTraverser.PathSet paths = traverser.allShortestPaths( sourceId, targetId, dir, edgeLabels, depth, maxDegree, skipDegree, capacity); - return manager.serializer(g).writePaths("paths", paths.paths(), - false); + return manager.serializer(g).writePaths("paths", paths, false); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CrosspointsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CrosspointsAPI.java index 8efbab6917..30ca26ac52 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CrosspointsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/CrosspointsAPI.java @@ -84,7 +84,6 @@ public String get(@Context GraphManager manager, HugeTraverser.PathSet paths = traverser.paths(sourceId, dir, targetId, dir, edgeLabel, depth, maxDegree, capacity, limit); - return manager.serializer(g).writePaths("crosspoints", - paths.paths(), true); + return manager.serializer(g).writePaths("crosspoints", paths, true); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java index 9a488c304a..095463b7cb 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java @@ -97,8 +97,7 @@ public String get(@Context GraphManager manager, dir.opposite(), edgeLabel, depth, maxDegree, capacity, limit); - return manager.serializer(g).writePaths("paths", paths.paths(), - false); + return manager.serializer(g).writePaths("paths", paths, false); } @POST diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RaysAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RaysAPI.java index 8ba11cd892..bb36766782 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RaysAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RaysAPI.java @@ -83,6 +83,6 @@ public String get(@Context GraphManager manager, HugeTraverser.PathSet paths = traverser.rays(source, dir, edgeLabel, depth, maxDegree, capacity, limit); - return manager.serializer(g).writePaths("rays", paths.paths(), false); + return manager.serializer(g).writePaths("rays", paths, false); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RingsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RingsAPI.java index b576785eec..6879da007e 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RingsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/RingsAPI.java @@ -86,7 +86,6 @@ public String get(@Context GraphManager manager, HugeTraverser.PathSet paths = traverser.rings(source, dir, edgeLabel, depth, sourceInRing, maxDegree, capacity, limit); - return manager.serializer(g).writePaths("rings", paths.paths(), - false); + return manager.serializer(g).writePaths("rings", paths, false); } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java index f579fc9b13..0a90b14502 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java @@ -501,10 +501,10 @@ public static HugeEdge constructEdge(HugeVertex ownerVertex, VertexLabel otherVertexLabel; if (isOutEdge) { - ownerVertex.correctVertexLabel(srcLabel); +// ownerVertex.correctVertexLabel(srcLabel); otherVertexLabel = tgtLabel; } else { - ownerVertex.correctVertexLabel(tgtLabel); +// ownerVertex.correctVertexLabel(tgtLabel); otherVertexLabel = srcLabel; } HugeVertex otherVertex = new HugeVertex(graph, otherVertexId, diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java index 60692f41d1..c707983d8d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java @@ -35,7 +35,6 @@ import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.apache.tinkerpop.gremlin.structure.util.ElementHelper; import org.apache.tinkerpop.gremlin.structure.util.StringFactory; -import org.eclipse.collections.impl.list.mutable.FastList; import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.HugeGraph; @@ -659,10 +658,6 @@ public void resetEdges() { this.edges = newSet(); } - public void removeEdge(HugeEdge edge) { - this.edges.remove(edge); - } - public void addEdge(HugeEdge edge) { if (this.edges == EMPTY_LIST) { this.edges = newSet(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java index cab5099a1d..0f639a3dc2 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java @@ -71,7 +71,6 @@ public class HugeTraverser { private HugeGraph graph; private static CollectionFactory collectionFactory; - protected ObjectIntMapping idMapping; public static final String DEFAULT_CAPACITY = "10000000"; public static final String DEFAULT_ELEMENTS_LIMIT = "10000000"; @@ -92,7 +91,9 @@ public class HugeTraverser { public HugeTraverser(HugeGraph graph) { this.graph = graph; - collectionFactory = new CollectionFactory(this.collectionImplType()); + if (collectionFactory == null) { + collectionFactory = new CollectionFactory(this.collectionImplType()); + } } public HugeGraph graph() { @@ -507,16 +508,6 @@ protected static List joinPath(Node prev, Node back, boolean ring) { return path; } - @Watched - protected Id id(int code) { - return (Id) this.idMapping.code2Object(code); - } - - @Watched - protected int code(Id id) { - return this.idMapping.object2Code(id); - } - public static class Node { private Id id; @@ -716,10 +707,7 @@ public void clear() { this.paths.clear(); } - public boolean addAll(PathSet paths) { - return this.paths.addAll(paths.paths); - } - + @Override public boolean isEmpty() { return this.paths.isEmpty(); } @@ -729,14 +717,12 @@ public boolean contains(Object o) { return this.paths.contains(o); } + @Override public int size() { return this.paths.size(); } - public Set paths() { - return this.paths; - } - + @Override public Iterator iterator() { return this.paths.iterator(); } @@ -751,6 +737,10 @@ public T[] toArray(T[] a) { return this.paths.toArray(a); } + public boolean addAll(PathSet paths) { + return this.paths.addAll(paths.paths); + } + public Set vertices() { Set vertices = newIdSet(); for (Path path : this.paths) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java index e5b1c97872..826296b598 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java @@ -92,9 +92,9 @@ public Set kout(Id sourceV, Directions dir, String label, return latest; } - public Set customizedKout(Id source, EdgeStep step, int maxDepth, - boolean nearest, long capacity, - long limit) { + public Set customizedKout(Id source, EdgeStep step, + int maxDepth, boolean nearest, + long capacity, long limit) { E.checkNotNull(source, "source vertex id"); this.checkVertexExist(source, "source vertex"); checkPositive(maxDepth, "k-out max_depth"); @@ -114,9 +114,9 @@ public Set customizedKout(Id source, EdgeStep step, int maxDepth, return results; } - public Set customizedKout(Id source, EdgeStep step, int maxDepth, - boolean nearest, long capacity, - boolean single) { + public Set customizedKout(Id source, EdgeStep step, + int maxDepth, boolean nearest, + long capacity, boolean single) { Set latest = newSet(single); Set all = newSet(single); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntArrayRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntArrayRecord.java index a67e4d344b..2365f0f786 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntArrayRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntArrayRecord.java @@ -25,7 +25,7 @@ public class IntArrayRecord implements Record { - private Int2IntsMap layer; + private final Int2IntsMap layer; public IntArrayRecord() { this.layer = new Int2IntsMap(); @@ -58,7 +58,7 @@ public int size() { public class IntArrayIterator implements IntIterator { - private int[] array; + private final int[] array; private int index; public IntArrayIterator(int[] array) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntIntRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntIntRecord.java index 5f9c06951c..fc12a5a395 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntIntRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntIntRecord.java @@ -24,7 +24,7 @@ public class IntIntRecord implements Record { - private IntIntHashMap layer; + private final IntIntHashMap layer; public IntIntRecord() { this.layer = new IntIntHashMap(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntSetRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntSetRecord.java index 9c0cc561a9..c4c025df28 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntSetRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntSetRecord.java @@ -25,7 +25,7 @@ public class IntSetRecord implements Record { - private IntObjectHashMap layer; + private final IntObjectHashMap layer; public IntSetRecord() { this.layer = new IntObjectHashMap<>(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java index 6763a80181..6564db944e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java @@ -37,7 +37,7 @@ public class MultiPathsRecords implements Records { - private final ObjectIntMapping idMapping; + private final ObjectIntMapping idMapping; private final RecordType type; protected final Stack sourceRecords; @@ -50,7 +50,7 @@ public class MultiPathsRecords implements Records { private int accessed; public MultiPathsRecords(Id sourceV, Id targetV, RecordType type) { - this.idMapping = new ObjectIntMapping(); + this.idMapping = new ObjectIntMapping<>(); this.type = type; int sourceCode = this.code(sourceV); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java index 9a1adbd2f1..1f08cda9d5 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java @@ -21,7 +21,7 @@ import com.baidu.hugegraph.type.define.SerialEnum; -public enum RecordType implements SerialEnum { +public enum RecordType implements SerialEnum { // Java Collection Framework ARRAY(1, "array"), @@ -62,4 +62,4 @@ public static RecordType fromCode(byte code) { throw new AssertionError("Unsupported record code: " + code); } } -} \ No newline at end of file +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java index 06a51167a7..eb5be3dfe8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java @@ -42,6 +42,7 @@ public ShortestPathRecords(Id sourceV, Id targetV) { this.accessedVertices = new IntHashSet(); this.accessedVertices.add(this.code(sourceV)); this.accessedVertices.add(this.code(targetV)); + this.pathFound = false; } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java index eb23855cd8..db2bcfc27e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java @@ -57,6 +57,8 @@ public static CollectionImplType fromCode(byte code) { return JCF; case 2: return EC; + case 3: + return FU; default: throw new AssertionError( "Unsupported collection code: " + code); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java index f0f89707b1..54b928af69 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java @@ -36,6 +36,7 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.type.define.CollectionImplType; +import com.baidu.hugegraph.util.E; import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; import it.unimi.dsi.fastutil.objects.ObjectArrayList; @@ -215,6 +216,42 @@ public static MutableIntObjectMap newIntObjectMap( return new IntObjectHashMap<>(map); } + public static MutableIntObjectMap newIntObjectMap( + int key1, V value1) { + return IntObjectHashMap.newWithKeysValues(key1, value1); + } + + public static MutableIntObjectMap newIntObjectMap( + int key1, V value1, + int key2, V value2) { + return IntObjectHashMap.newWithKeysValues(key1, value1, key2, value2); + } + + public static MutableIntObjectMap newIntObjectMap( + int key1, V value1, + int key2, V value2, + int key3, V value3) { + return IntObjectHashMap.newWithKeysValues(key1, value1, key2, value2, + key3, value3); + } + + @SuppressWarnings("unchecked") + public static MutableIntObjectMap newIntObjectMap( + int key1, V value1, + int key2, V value2, + int key3, V value3, + Object... objects) { + IntObjectHashMap map = IntObjectHashMap.newWithKeysValues( + key1, value1, key2, value2, key3, value3); + E.checkArgument(objects.length % 2 == 0, + "Must provide even arguments for " + + "CollectionFactory.newIntObjectMap"); + for (int i = 0; i < objects.length; i+=2) { + map.put((int) objects[i], (V) objects[i + 1]); + } + return map; + } + public Set newIdSet() { return newIdSet(this.type); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java index d09431d12a..733b11baa4 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java @@ -25,23 +25,25 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.perf.PerfUtil.Watched; -public final class ObjectIntMapping { +public final class ObjectIntMapping { private static final int MAGIC = 1 << 16; - private final IntObjectHashMap int2IdMap; + private final IntObjectHashMap int2IdMap; public ObjectIntMapping() { - this.int2IdMap = new IntObjectHashMap(1000000); + this.int2IdMap = new IntObjectHashMap<>(1000000); } @Watched + @SuppressWarnings("unchecked") public int object2Code(Object object) { int key = object.hashCode(); + // TODO: improve hash algorithm for (int i = 1; i > 0; i <<= 1) { for (int j = 0; i >= MAGIC && j < 10; j++) { Id existed = (Id) this.int2IdMap.get(key); if (existed == null) { - this.int2IdMap.put(key, object); + this.int2IdMap.put(key, (V) object); return key; } if (existed.equals(object)) { diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/JsonUtilTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/JsonUtilTest.java index a42d55082b..3214aa7909 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/JsonUtilTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/JsonUtilTest.java @@ -25,7 +25,6 @@ import org.apache.tinkerpop.shaded.jackson.core.type.TypeReference; import org.eclipse.collections.api.map.primitive.MutableIntObjectMap; -import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; import org.junit.Test; import org.mockito.Mockito; @@ -54,6 +53,7 @@ import com.baidu.hugegraph.unit.BaseUnitTest; import com.baidu.hugegraph.unit.FakeObjects; import com.baidu.hugegraph.util.JsonUtil; +import com.baidu.hugegraph.util.collection.CollectionFactory; import com.google.common.collect.ImmutableList; public class JsonUtilTest extends BaseUnitTest { @@ -235,7 +235,7 @@ public void testSerializeVertexWithNumberId() { HugeVertex vertex = new HugeVertex(fakeObject.graph(), id, vl); MutableIntObjectMap> properties = - IntObjectHashMap.newWithKeysValues( + CollectionFactory.newIntObjectMap( (int) name.id().asLong(), new HugeVertexProperty<>(vertex, name, "marko"), (int) age.id().asLong(), @@ -287,7 +287,7 @@ public void testSerializeEdge() { Whitebox.setInternalState(edge, "targetVertex", target); MutableIntObjectMap> properties = - IntObjectHashMap.newWithKeysValues( + CollectionFactory.newIntObjectMap( (int) date.id().asLong(), new HugeEdgeProperty<>(edge, date, Utils.date("2019-03-12")), (int) weight.id().asLong(), From 35a7bc7bdbe5db5592d00c064a822b7502b3c804 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Wed, 28 Apr 2021 19:28:31 +0800 Subject: [PATCH 23/38] multiple threads adapt for kout/kneighbor Change-Id: I0791e3d307f2cff51ccc4760ea63c52468a4b8d8 --- .../api/traversers/KneighborAPI.java | 27 ++- .../hugegraph/api/traversers/KoutAPI.java | 27 +-- .../baidu/hugegraph/structure/HugeEdge.java | 4 +- .../traversal/algorithm/HugeTraverser.java | 25 --- .../algorithm/KneighborTraverser.java | 77 +++++--- .../traversal/algorithm/KoutTraverser.java | 114 ++++++----- .../traversal/algorithm/OltpTraverser.java | 51 ++++- .../traversal/algorithm/PathsTraverser.java | 10 +- .../algorithm/records/AbstractRecords.java | 61 ++++++ ...s.java => DoubleWayMultiPathsRecords.java} | 32 +--- .../algorithm/records/KneighborRecords.java | 74 +++++++ .../algorithm/records/KoutRecords.java | 63 ++++++ .../records/ShortestPathRecords.java | 5 +- .../records/SingleWayMultiPathsRecords.java | 180 ++++++++++++++++++ .../records/{ => record}/IntArrayRecord.java | 29 +-- .../records/{ => record}/IntIntRecord.java | 5 +- .../algorithm/records/record/IntIterator.java | 57 ++++++ .../records/{ => record}/IntSetRecord.java | 11 +- .../records/{ => record}/Record.java | 4 +- .../records/{ => record}/RecordFactory.java | 21 +- .../records/{ => record}/RecordType.java | 2 +- .../algorithm/records/record/SyncRecord.java | 73 +++++++ .../util/collection/ObjectIntMapping.java | 6 +- 23 files changed, 748 insertions(+), 210 deletions(-) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/{MultiPathsRecords.java => DoubleWayMultiPathsRecords.java} (89%) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/{ => record}/IntArrayRecord.java (68%) rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/{ => record}/IntIntRecord.java (90%) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntIterator.java rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/{ => record}/IntSetRecord.java (86%) rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/{ => record}/Record.java (90%) rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/{ => record}/RecordFactory.java (68%) rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/{ => record}/RecordType.java (96%) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/SyncRecord.java diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java index c912936e9a..ae382fe001 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java @@ -22,10 +22,8 @@ import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_ELEMENTS_LIMIT; -import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; -import java.util.List; import java.util.Set; import javax.inject.Singleton; @@ -50,8 +48,10 @@ import com.baidu.hugegraph.core.GraphManager; import com.baidu.hugegraph.server.RestServer; import com.baidu.hugegraph.structure.HugeVertex; +import com.baidu.hugegraph.traversal.algorithm.records.KneighborRecords; import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; import com.baidu.hugegraph.traversal.algorithm.KneighborTraverser; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; @@ -125,32 +125,27 @@ public String post(@Context GraphManager manager, EdgeStep step = step(g, request.step); - Set results; + KneighborRecords results; try (KneighborTraverser traverser = new KneighborTraverser(g)) { results = traverser.customizedKneighbor(sourceId, step, request.maxDepth, request.limit); } - Set neighbors = new HashSet<>(); - for (HugeTraverser.Node node : results) { - neighbors.add(node.id()); - } + Set neighbors = results.ids(request.limit); - List paths = new ArrayList<>(); + PathSet paths = new PathSet(); if (request.withPath) { - for (HugeTraverser.Node node : results) { - paths.add(new HugeTraverser.Path(node.path())); - } + paths.addAll(results.paths(request.limit)); } Iterator iter = QueryResults.emptyIterator(); if (request.withVertex) { Set ids = new HashSet<>(); - for (HugeTraverser.Node node : results) { - ids.add(node.id()); - } - for (HugeTraverser.Path p : paths) { - ids.addAll(p.vertices()); + ids.addAll(results.ids(request.limit)); + if (request.withPath) { + for (HugeTraverser.Path p : paths) { + ids.addAll(p.vertices()); + } } if (!ids.isEmpty()) { iter = g.vertices(ids.toArray()); diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KoutAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KoutAPI.java index c91b2fbb57..b6aa8d537f 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KoutAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KoutAPI.java @@ -23,10 +23,8 @@ import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE; import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_ELEMENTS_LIMIT; -import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; -import java.util.List; import java.util.Set; import javax.inject.Singleton; @@ -51,9 +49,9 @@ import com.baidu.hugegraph.core.GraphManager; import com.baidu.hugegraph.server.RestServer; import com.baidu.hugegraph.structure.HugeVertex; +import com.baidu.hugegraph.traversal.algorithm.records.KoutRecords; import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser; -import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Node; import com.baidu.hugegraph.traversal.algorithm.KoutTraverser; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; @@ -133,7 +131,7 @@ public String post(@Context GraphManager manager, EdgeStep step = step(g, request.step); - Set results; + KoutRecords results; try (KoutTraverser traverser = new KoutTraverser(g)) { results = traverser.customizedKout(sourceId, step, request.maxDepth, @@ -142,25 +140,20 @@ public String post(@Context GraphManager manager, request.limit); } - Set neighbors = new HashSet<>(); - for (HugeTraverser.Node node : results) { - neighbors.add(node.id()); - } + Set neighbors = results.ids(request.limit); - List paths = new ArrayList<>(); + HugeTraverser.PathSet paths = new HugeTraverser.PathSet(); if (request.withPath) { - for (HugeTraverser.Node node : results) { - paths.add(new HugeTraverser.Path(node.path())); - } + paths.addAll(results.paths(request.limit)); } Iterator iter = QueryResults.emptyIterator(); if (request.withVertex) { Set ids = new HashSet<>(); - for (Node node : results) { - ids.add(node.id()); - } - for (HugeTraverser.Path p : paths) { - ids.addAll(p.vertices()); + ids.addAll(results.ids(request.limit)); + if (request.withPath) { + for (HugeTraverser.Path p : paths) { + ids.addAll(p.vertices()); + } } if (!ids.isEmpty()) { iter = g.vertices(ids.toArray()); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java index 0a90b14502..f579fc9b13 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java @@ -501,10 +501,10 @@ public static HugeEdge constructEdge(HugeVertex ownerVertex, VertexLabel otherVertexLabel; if (isOutEdge) { -// ownerVertex.correctVertexLabel(srcLabel); + ownerVertex.correctVertexLabel(srcLabel); otherVertexLabel = tgtLabel; } else { -// ownerVertex.correctVertexLabel(tgtLabel); + ownerVertex.correctVertexLabel(tgtLabel); otherVertexLabel = srcLabel; } HugeVertex otherVertex = new HugeVertex(graph, otherVertexId, diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java index 0f639a3dc2..615d2ca8cb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java @@ -61,7 +61,6 @@ import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.InsertionOrderUtil; import com.baidu.hugegraph.util.collection.CollectionFactory; -import com.baidu.hugegraph.util.collection.ObjectIntMapping; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -157,30 +156,6 @@ protected Set adjacentVertices(Id source, EdgeStep step) { return neighbors; } - protected Set adjacentVertices(Id start, Set vertices, - EdgeStep step, Set excluded, - long remaining) { - Set neighbors = newSet(); - for (Node source : vertices) { - Iterator edges = this.edgesOfVertex(source.id(), step); - while (edges.hasNext()) { - Id target = ((HugeEdge) edges.next()).id().otherVertexId(); - KNode kNode = new KNode(target, (KNode) source); - boolean matchExcluded = (excluded != null && - excluded.contains(kNode)); - if (matchExcluded || neighbors.contains(kNode) || - start.equals(kNode.id())) { - continue; - } - neighbors.add(kNode); - if (remaining != NO_LIMIT && --remaining <= 0L) { - return neighbors; - } - } - } - return neighbors; - } - @Watched protected Iterator edgesOfVertex(Id source, Directions dir, Id label, long limit) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java index 3403a53c25..cec1b53ab3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java @@ -21,9 +21,15 @@ import java.util.Iterator; import java.util.Set; +import java.util.function.Consumer; + +import org.apache.tinkerpop.gremlin.structure.Edge; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.structure.HugeEdge; +import com.baidu.hugegraph.traversal.algorithm.records.KneighborRecords; +import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; @@ -64,8 +70,8 @@ public Set kneighbor(Id sourceV, Directions dir, return all; } - public Set customizedKneighbor(Id source, EdgeStep step, - int maxDepth, long limit) { + public KneighborRecords customizedKneighbor(Id source, EdgeStep step, + int maxDepth, long limit) { E.checkNotNull(source, "source vertex id"); this.checkVertexExist(source, "source vertex"); checkPositive(maxDepth, "k-neighbor max_depth"); @@ -73,36 +79,59 @@ public Set customizedKneighbor(Id source, EdgeStep step, boolean single = maxDepth < this.concurrentDepth() || step.direction() != Directions.BOTH; - return this.customizedKneighbor(source, step, maxDepth, - limit, single); + Traverser traverser = new Traverser(source, step, maxDepth, limit, + single); + return traverser.customizedKneighbor(); } - public Set customizedKneighbor(Id source, EdgeStep step, int maxDepth, - long limit, boolean single) { - Set latest = newSet(single); - Set all = newSet(single); + private class Traverser { - Node sourceV = new KNode(source, null); + private final KneighborRecords record; - latest.add(sourceV); + private final EdgeStep step; + private final long limit; + private final boolean single; + private int depth; - while (maxDepth-- > 0) { - long remaining = limit == NO_LIMIT ? NO_LIMIT : limit - all.size(); - latest = this.adjacentVertices(source, latest, step, all, - remaining, single); - int size = all.size() + latest.size(); - if (limit != NO_LIMIT && size >= limit) { - int subLength = (int) limit - all.size(); - Iterator iterator = latest.iterator(); - for (int i = 0; i < subLength && iterator.hasNext(); i++) { - all.add(iterator.next()); + private boolean stop; + + public Traverser(Id source, EdgeStep step, int maxDepth, + long limit, boolean single) { + this.record = new KneighborRecords(source, RecordType.INT, + true, single); + this.step = step; + this.depth = maxDepth; + this.limit = limit; + this.single = single; + this.stop = false; + } + + public KneighborRecords customizedKneighbor() { + Consumer consumer = v -> { + Iterator edges = edgesOfVertex(v, step); + while (edges.hasNext()) { + Id target = ((HugeEdge) edges.next()).id().otherVertexId(); + this.record.addPath(v, target); + this.checkLimit(this.limit, this.depth, this.record.size()); } - break; - } else { - all.addAll(latest); + }; + + while (this.depth-- > 0) { + this.record.startOneLayer(true); + traverseIds(this.record.keys(), consumer, + this.single, this.stop); + this.record.finishOneLayer(); } + return this.record; } - return all; + private void checkLimit(long limit, long depth, int size) { + if (limit == NO_LIMIT || depth > 0) { + return; + } + if (size >= limit) { + this.stop = true; + } + } } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java index 826296b598..c3a3242867 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java @@ -19,14 +19,20 @@ package com.baidu.hugegraph.traversal.algorithm; +import java.util.Iterator; import java.util.Set; +import java.util.function.Consumer; + +import org.apache.tinkerpop.gremlin.structure.Edge; import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.structure.HugeEdge; +import com.baidu.hugegraph.traversal.algorithm.records.KoutRecords; +import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.type.define.Directions; -import com.baidu.hugegraph.util.CollectionUtil; import com.baidu.hugegraph.util.E; public class KoutTraverser extends OltpTraverser { @@ -92,67 +98,87 @@ public Set kout(Id sourceV, Directions dir, String label, return latest; } - public Set customizedKout(Id source, EdgeStep step, - int maxDepth, boolean nearest, - long capacity, long limit) { + public KoutRecords customizedKout(Id source, EdgeStep step, + int maxDepth, boolean nearest, + long capacity, long limit) { E.checkNotNull(source, "source vertex id"); this.checkVertexExist(source, "source vertex"); checkPositive(maxDepth, "k-out max_depth"); checkCapacity(capacity); checkLimit(limit); - Set results; boolean single = maxDepth < this.concurrentDepth() || step.direction() != Directions.BOTH; - results = this.customizedKout(source, step, maxDepth, nearest, - capacity, single); + Traverser traverser = new Traverser(source, step, maxDepth, nearest, + capacity, limit, single); + return traverser.customizedKout(); + } - if (limit != NO_LIMIT && results.size() > limit) { - results = CollectionUtil.subSet(results, 0, (int) limit); + private class Traverser { + + private final KoutRecords record; + + private final EdgeStep step; + private final long capacity; + private final long limit; + private final boolean single; + private int depth; + + private boolean stop; + + public Traverser(Id source, EdgeStep step, int maxDepth, + boolean nearest, long capacity, long limit, + boolean single) { + this.record = new KoutRecords(source, RecordType.INT, + nearest, single); + this.step = step; + this.depth = maxDepth; + this.capacity = capacity; + this.limit = limit; + this.single = single; + this.stop = false; } - return results; - } - - public Set customizedKout(Id source, EdgeStep step, - int maxDepth, boolean nearest, - long capacity, boolean single) { - Set latest = newSet(single); - Set all = newSet(single); + public KoutRecords customizedKout() { + Consumer consumer = v -> { + Iterator edges = edgesOfVertex(v, step); + while (edges.hasNext()) { + Id target = ((HugeEdge) edges.next()).id().otherVertexId(); + this.record.addPath(v, target); - Node sourceV = new KNode(source, null); + this.checkCapacity(this.capacity, this.record.accessed(), + this.depth); + this.checkLimit(this.limit, this.depth, this.record.size()); + } + }; - latest.add(sourceV); - all.add(sourceV); + while (this.depth-- > 0) { + this.record.startOneLayer(true); + traverseIds(this.record.keys(), consumer, + this.single, this.stop); + this.record.finishOneLayer(); + } + return this.record; + } - int depth = maxDepth; - long remaining = capacity == NO_LIMIT ? - NO_LIMIT : capacity - latest.size(); - while (depth-- > 0) { - if (nearest) { - latest = this.adjacentVertices(source, latest, step, all, - remaining, single); - all.addAll(latest); - } else { - latest = this.adjacentVertices(source, latest, step, null, - remaining, single); + private void checkCapacity(long capacity, long accessed, int depth) { + if (capacity == NO_LIMIT) { + return; } - if (capacity != NO_LIMIT) { - // Update 'remaining' value to record remaining capacity - remaining -= latest.size(); - reachCapacity(remaining, capacity, depth); + if (capacity <= accessed && depth > 0) { + throw new HugeException( + "Reach capacity '%s' while remaining depth '%s'", + capacity, depth); } } - return latest; - } - - private static void reachCapacity(long remaining, long capacity, - int depth) { - if (remaining <= 0 && depth > 0) { - throw new HugeException( - "Reach capacity '%s' while remaining depth '%s'", - capacity, depth); + private void checkLimit(long limit, long depth, int size) { + if (limit == NO_LIMIT || depth > 0) { + return; + } + if (size >= limit) { + this.stop = true; + } } } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/OltpTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/OltpTraverser.java index 8dcd9909c7..f4bf1e3d8c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/OltpTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/OltpTraverser.java @@ -92,6 +92,26 @@ protected Set adjacentVertices(Id source, Set latest, } } + protected Set adjacentVertices(Set vertices, EdgeStep step, + Set excluded, long remaining) { + Set neighbors = newSet(); + for (Node source : vertices) { + Iterator edges = this.edgesOfVertex(source.id(), step); + while (edges.hasNext()) { + Id target = ((HugeEdge) edges.next()).id().otherVertexId(); + KNode kNode = new KNode(target, (KNode) source); + if (excluded != null && excluded.contains(kNode)) { + continue; + } + neighbors.add(kNode); + if (remaining != NO_LIMIT && --remaining <= 0L) { + return neighbors; + } + } + } + return neighbors; + } + protected Set adjacentVertices(Set vertices, EdgeStep step, Set excluded, AtomicLong remaining) { @@ -115,21 +135,40 @@ protected Set adjacentVertices(Set vertices, EdgeStep step, protected long traverseNodes(Iterator vertices, Consumer consumer) { - return this.traverse(vertices, consumer, "traverse-nodes"); + return this.traverse(vertices, consumer, "traverse-nodes", false); } protected long traversePairs(Iterator> pairs, Consumer> consumer) { - return this.traverse(pairs, consumer, "traverse-pairs"); + return this.traverse(pairs, consumer, "traverse-pairs", false); + } + + protected long traverseIds(Iterator ids, Consumer consumer, + boolean single, boolean stop) { + if (!single) { + return this.traverseIds(ids, consumer); + } else { + long count = 0L; + while (ids.hasNext()) { + count++; + consumer.accept(ids.next()); + } + return count; + } } protected long traverseIds(Iterator ids, Consumer consumer) { - return this.traverse(ids, consumer, "traverse-ids"); + return this.traverseIds(ids, consumer, false); + } + + protected long traverseIds(Iterator ids, Consumer consumer, + boolean stop) { + return this.traverse(ids, consumer, "traverse-ids", stop); } protected long traverse(Iterator iterator, Consumer consumer, - String name) { - if (!iterator.hasNext()) { + String name, boolean stop) { + if (!iterator.hasNext() || stop) { return 0L; } @@ -138,7 +177,7 @@ protected long traverse(Iterator iterator, Consumer consumer, consumers.start(name); long total = 0L; try { - while (iterator.hasNext()) { + while (!stop && iterator.hasNext()) { total++; K v = iterator.next(); consumers.provide(v); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java index 48dbc47a8d..4f5e839b0b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java @@ -27,8 +27,8 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeEdge; -import com.baidu.hugegraph.traversal.algorithm.records.MultiPathsRecords; -import com.baidu.hugegraph.traversal.algorithm.records.RecordType; +import com.baidu.hugegraph.traversal.algorithm.records.DoubleWayMultiPathsRecords; +import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; import com.google.common.collect.ImmutableList; @@ -83,7 +83,7 @@ public PathSet paths(Id sourceV, Directions sourceDir, private class Traverser { - private final MultiPathsRecords record; + private final DoubleWayMultiPathsRecords record; private final Id label; private final long degree; @@ -94,8 +94,8 @@ private class Traverser { public Traverser(Id sourceV, Id targetV, Id label, long degree, long capacity, long limit) { - this.record = new MultiPathsRecords(sourceV, targetV, - RecordType.ARRAY); + this.record = new DoubleWayMultiPathsRecords(sourceV, targetV, + RecordType.ARRAY); this.label = label; this.degree = degree; this.capacity = capacity; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java new file mode 100644 index 0000000000..e137ee3ae8 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java @@ -0,0 +1,61 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.perf.PerfUtil.Watched; +import com.baidu.hugegraph.traversal.algorithm.records.record.Record; +import com.baidu.hugegraph.traversal.algorithm.records.record.RecordFactory; +import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; +import com.baidu.hugegraph.util.collection.ObjectIntMapping; + +public abstract class AbstractRecords implements Records { + + private final ObjectIntMapping idMapping; + protected final RecordType type; + private final boolean single; + + public AbstractRecords(RecordType type, boolean single) { + this.idMapping = new ObjectIntMapping<>(); + this.type = type; + this.single = single; + } + + public AbstractRecords(RecordType type) { + this.idMapping = new ObjectIntMapping<>(); + this.type = type; + this.single = true; + } + + @Watched + protected int code(Id id) { + return this.idMapping.object2Code(id); + } + + @Watched + protected Id id(int code) { + return (Id) this.idMapping.code2Object(code); + } + + protected Record newRecord() { + return RecordFactory.newRecord(this.type, single); + } +} + diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java similarity index 89% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java index 6564db944e..433e7338cc 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/MultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java @@ -26,19 +26,17 @@ import java.util.function.Function; import org.apache.commons.collections.CollectionUtils; -import org.eclipse.collections.api.iterator.IntIterator; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; -import com.baidu.hugegraph.util.collection.ObjectIntMapping; +import com.baidu.hugegraph.traversal.algorithm.records.record.IntIterator; +import com.baidu.hugegraph.traversal.algorithm.records.record.Record; +import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; import com.google.common.collect.Lists; -public class MultiPathsRecords implements Records { - - private final ObjectIntMapping idMapping; - private final RecordType type; +public class DoubleWayMultiPathsRecords extends AbstractRecords { protected final Stack sourceRecords; protected final Stack targetRecords; @@ -49,9 +47,8 @@ public class MultiPathsRecords implements Records { protected boolean forward; private int accessed; - public MultiPathsRecords(Id sourceV, Id targetV, RecordType type) { - this.idMapping = new ObjectIntMapping<>(); - this.type = type; + public DoubleWayMultiPathsRecords(Id sourceV, Id targetV, RecordType type) { + super(type); int sourceCode = this.code(sourceV); int targetCode = this.code(targetV); @@ -144,7 +141,8 @@ private PathSet linkPath(int source, int target, boolean ring) { } List ids = new ArrayList<>(spath.vertices()); ids.addAll(tpath.vertices()); - results.add(new Path(ids)); + Id crosspoint = this.id(this.forward ? target : source); + results.add(new Path(crosspoint, ids)); } } return results; @@ -192,18 +190,4 @@ private PathSet linkPath(Stack all, int id, int layerIndex) { protected void addPath(int current, int parent) { this.currentRecord.addPath(current, parent); } - - @Watched - protected int code(Id id) { - return this.idMapping.object2Code(id); - } - - @Watched - protected Id id(int code) { - return (Id) this.idMapping.code2Object(code); - } - - private Record newRecord() { - return RecordFactory.newRecord(this.type); - } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java new file mode 100644 index 0000000000..f4abcb6b86 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java @@ -0,0 +1,74 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import static com.baidu.hugegraph.backend.query.Query.NO_LIMIT; + +import java.util.Set; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; +import com.baidu.hugegraph.traversal.algorithm.records.record.IntIterator; +import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; +import com.baidu.hugegraph.type.define.CollectionImplType; +import com.baidu.hugegraph.util.collection.CollectionFactory; +import com.google.common.collect.ImmutableList; + +public class KneighborRecords extends SingleWayMultiPathsRecords { + + private Id source; + + public KneighborRecords(Id source, RecordType type, + boolean nearest, boolean single) { + super(source, type, nearest, single); + this.source = source; + } + + public int size() { + return (int) this.accessed(); + } + + public Set ids(long limit) { + Set ids = CollectionFactory.newIdSet(CollectionImplType.EC); + ids.add(this.source); + for (int i = 1; i < this.records.size(); i++) { + IntIterator iterator = this.records.get(i).keys(); + while ((limit == NO_LIMIT || limit > 1L) && iterator.hasNext()) { + ids.add(this.id(iterator.next())); + limit--; + } + } + return ids; + } + + public PathSet paths(long limit) { + PathSet paths = new PathSet(); + paths.add(new HugeTraverser.Path(ImmutableList.of(this.source))); + for (int i = 1; i < this.records.size(); i++) { + IntIterator iterator = this.records.get(i).keys(); + while ((limit == NO_LIMIT || limit > 1L) && iterator.hasNext()) { + paths.add(this.getPath(i, iterator.next())); + limit--; + } + } + return paths; + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java new file mode 100644 index 0000000000..aec6f2d137 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java @@ -0,0 +1,63 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import static com.baidu.hugegraph.backend.query.Query.NO_LIMIT; + +import java.util.Set; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; +import com.baidu.hugegraph.traversal.algorithm.records.record.IntIterator; +import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; +import com.baidu.hugegraph.type.define.CollectionImplType; +import com.baidu.hugegraph.util.collection.CollectionFactory; + +public class KoutRecords extends SingleWayMultiPathsRecords { + + public KoutRecords(Id source, + RecordType type, + boolean nearest, boolean single) { + super(source, type, nearest, single); + } + + @Override + public int size() { + return this.currentRecord.size(); + } + + public Set ids(long limit) { + IntIterator iterator = this.records.peek().keys(); + Set ids = CollectionFactory.newIdSet(CollectionImplType.EC); + while ((limit == NO_LIMIT || limit-- > 0L) && iterator.hasNext()) { + ids.add(this.id(iterator.next())); + } + return ids; + } + + public PathSet paths(long limit) { + PathSet paths = new PathSet(); + IntIterator iterator = this.records.peek().keys(); + while ((limit == NO_LIMIT || limit-- > 0L) && iterator.hasNext()) { + paths.add(this.getPath(this.records.size() - 1, iterator.next())); + } + return paths; + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java index eb5be3dfe8..dcb51b292a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java @@ -30,8 +30,11 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; +import com.baidu.hugegraph.traversal.algorithm.records.record.IntIntRecord; +import com.baidu.hugegraph.traversal.algorithm.records.record.Record; +import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; -public class ShortestPathRecords extends MultiPathsRecords { +public class ShortestPathRecords extends DoubleWayMultiPathsRecords { private final IntHashSet accessedVertices; private boolean pathFound; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java new file mode 100644 index 0000000000..4a2d3aa239 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java @@ -0,0 +1,180 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.Stack; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +import org.eclipse.collections.api.set.primitive.MutableIntSet; +import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; +import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; + +import com.baidu.hugegraph.HugeException; +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.iterator.MapperIterator; +import com.baidu.hugegraph.perf.PerfUtil.Watched; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; +import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; +import com.baidu.hugegraph.traversal.algorithm.records.record.IntIntRecord; +import com.baidu.hugegraph.traversal.algorithm.records.record.IntIterator; +import com.baidu.hugegraph.traversal.algorithm.records.record.Record; +import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; + +public class SingleWayMultiPathsRecords extends AbstractRecords { + + protected final Stack records; + + private final boolean nearest; + + protected Record currentRecord; + protected IntIterator lastRecordKeys; + protected int current; + protected boolean forward; + private final Set accessedVertices; + + public SingleWayMultiPathsRecords(Id source, RecordType type, + boolean nearest, boolean single) { + super(type, single); + + this.nearest = nearest; + + int sourceCode = this.code(source); + Record firstRecord = this.newRecord(); + firstRecord.addPath(sourceCode, 0); + this.records = new Stack<>(); + this.records.push(firstRecord); + + this.accessedVertices = single ? new HashSet() : + ConcurrentHashMap.newKeySet(); + this.accessedVertices.add(sourceCode); + } + + @Override + public void startOneLayer(boolean forward) { + this.currentRecord = this.newRecord(); + this.lastRecordKeys = this.records.peek().keys(); + } + + @Override + public void finishOneLayer() { + this.records.push(this.currentRecord); + } + + @Override + public boolean hasNextKey() { + return this.lastRecordKeys.hasNext(); + } + + @Override + public Id nextKey() { + this.current = this.lastRecordKeys.next(); + return this.id(current); + } + + @Override + public PathSet findPath(Id target, Function filter, + boolean all, boolean ring) { + PathSet paths = new PathSet(); + for (int i = 1; i < this.records.size(); i++) { + IntIterator iterator = this.records.get(i).keys(); + while (iterator.hasNext()) { + paths.add(this.getPath(i, iterator.next())); + } + } + return paths; + } + + @Override + public long accessed() { + return this.accessedVertices.size(); + } + + public Iterator keys() { + return new MapperIterator<>(this.lastRecordKeys, this::id); + } + + @Watched + public void addPath(Id source, Id target) { + int sourceCode = this.code(source); + int targetCode = this.code(target); + if (this.nearest && this.accessedVertices.contains(targetCode) || + !this.nearest && this.currentRecord.containsKey(targetCode)) { + return; + } + this.currentRecord.addPath(targetCode, sourceCode); + + this.accessedVertices.add(targetCode); + } + + public int size() { + return 0; + } + + public Path getPath(int target) { + List ids = new ArrayList<>(); + for (int i = 0; i < this.records.size(); i++) { + IntIntHashMap layer = ((IntIntRecord) this.records + .elementAt(i)).layer(); + if (!layer.containsKey(target)) { + continue; + } + + ids.add(this.id(target)); + int parent = layer.get(target); + ids.add(this.id(parent)); + i--; + for (; i > 0; i--) { + layer = ((IntIntRecord) this.records.elementAt(i)).layer(); + parent = layer.get(parent); + ids.add(this.id(parent)); + } + break; + } + return new Path(ids); + } + + public Path getPath(int layerIndex, int target) { + List ids = new ArrayList<>(); + IntIntHashMap layer = ((IntIntRecord) this.records + .elementAt(layerIndex)).layer(); + if (!layer.containsKey(target)) { + throw new HugeException("Failed to get path for %s", + this.id(target)); + } + ids.add(this.id(target)); + int parent = layer.get(target); + ids.add(this.id(parent)); + layerIndex--; + for (; layerIndex > 0; layerIndex--) { + layer = ((IntIntRecord) this.records.elementAt(layerIndex)).layer(); + parent = layer.get(parent); + ids.add(this.id(parent)); + } + Collections.reverse(ids); + return new Path(ids); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntArrayRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntArrayRecord.java similarity index 68% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntArrayRecord.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntArrayRecord.java index 2365f0f786..7b10904feb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntArrayRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntArrayRecord.java @@ -17,9 +17,7 @@ * under the License. */ -package com.baidu.hugegraph.traversal.algorithm.records; - -import org.eclipse.collections.api.iterator.IntIterator; +package com.baidu.hugegraph.traversal.algorithm.records.record; import com.baidu.hugegraph.util.collection.Int2IntsMap; @@ -33,7 +31,7 @@ public IntArrayRecord() { @Override public IntIterator keys() { - return this.layer.keyIterator(); + return new IntIterator(this.layer.keyIterator()); } @Override @@ -43,7 +41,7 @@ public boolean containsKey(int key) { @Override public IntIterator get(int key) { - return new IntArrayIterator(this.layer.get(key)); + return new IntIterator(this.layer.get(key)); } @Override @@ -55,25 +53,4 @@ public void addPath(int node, int parent) { public int size() { return this.layer.size(); } - - public class IntArrayIterator implements IntIterator { - - private final int[] array; - private int index; - - public IntArrayIterator(int[] array) { - this.array = array; - this.index = 0; - } - - @Override - public int next() { - return this.array[index++]; - } - - @Override - public boolean hasNext() { - return this.index < this.array.length; - } - } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntIntRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntIntRecord.java similarity index 90% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntIntRecord.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntIntRecord.java index fc12a5a395..2cf06596ec 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntIntRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntIntRecord.java @@ -17,9 +17,8 @@ * under the License. */ -package com.baidu.hugegraph.traversal.algorithm.records; +package com.baidu.hugegraph.traversal.algorithm.records.record; -import org.eclipse.collections.api.iterator.IntIterator; import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; public class IntIntRecord implements Record { @@ -32,7 +31,7 @@ public IntIntRecord() { @Override public IntIterator keys() { - return this.layer.keySet().intIterator(); + return new IntIterator(this.layer.keySet().intIterator()); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntIterator.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntIterator.java new file mode 100644 index 0000000000..34fea4d2f8 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntIterator.java @@ -0,0 +1,57 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records.record; + +import java.util.Iterator; + +public class IntIterator implements Iterator { + + private final org.eclipse.collections.api.iterator.IntIterator iterator; + private final int[] array; + private int index; + + public IntIterator(int[] array) { + this.iterator = null; + this.array = array; + this.index = 0; + } + + public IntIterator(org.eclipse.collections.api.iterator.IntIterator intIterator) { + this.iterator = intIterator; + this.array = new int[0]; + this.index = 0; + } + + @Override + public Integer next() { + if (this.iterator != null) { + return this.iterator.next(); + } + return this.array[index++]; + } + + @Override + public boolean hasNext() { + if (this.iterator != null) { + return this.iterator.hasNext(); + } + return this.index < this.array.length; + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntSetRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntSetRecord.java similarity index 86% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntSetRecord.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntSetRecord.java index c4c025df28..9abb8d3964 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/IntSetRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntSetRecord.java @@ -17,9 +17,8 @@ * under the License. */ -package com.baidu.hugegraph.traversal.algorithm.records; +package com.baidu.hugegraph.traversal.algorithm.records.record; -import org.eclipse.collections.api.iterator.IntIterator; import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; @@ -33,17 +32,17 @@ public IntSetRecord() { @Override public IntIterator keys() { - return this.layer.keySet().intIterator(); + return new IntIterator(this.layer.keySet().intIterator()); } - @Override public boolean containsKey(int key) { return this.layer.containsKey(key); } - @Override public IntIterator get(int key) { - return this.layer.get(key).intIterator(); + @Override + public IntIterator get(int key) { + return new IntIterator(this.layer.get(key).intIterator()); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/Record.java similarity index 90% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/Record.java index aec0b5241b..947feae87a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/Record.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/Record.java @@ -17,9 +17,7 @@ * under the License. */ -package com.baidu.hugegraph.traversal.algorithm.records; - -import org.eclipse.collections.api.iterator.IntIterator; +package com.baidu.hugegraph.traversal.algorithm.records.record; public interface Record { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordFactory.java similarity index 68% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordFactory.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordFactory.java index 563c820ed3..4fd6a6f1f4 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordFactory.java @@ -17,20 +17,33 @@ * under the License. */ -package com.baidu.hugegraph.traversal.algorithm.records; +package com.baidu.hugegraph.traversal.algorithm.records.record; public class RecordFactory { public static Record newRecord(RecordType type) { + return newRecord(type, true); + } + + public static Record newRecord(RecordType type, boolean single) { + Record record; switch (type) { case ARRAY: - return new IntArrayRecord(); + record = new IntArrayRecord(); + break; case SET: - return new IntSetRecord(); + record = new IntSetRecord(); + break; case INT: - return new IntIntRecord(); + record = new IntIntRecord(); + break; default: throw new AssertionError("Unsupported record type: " + type); } + if (!single) { + record = new SyncRecord(record); + } + + return record; } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordType.java similarity index 96% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordType.java index 1f08cda9d5..8d170184be 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/RecordType.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordType.java @@ -17,7 +17,7 @@ * under the License. */ -package com.baidu.hugegraph.traversal.algorithm.records; +package com.baidu.hugegraph.traversal.algorithm.records.record; import com.baidu.hugegraph.type.define.SerialEnum; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/SyncRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/SyncRecord.java new file mode 100644 index 0000000000..128057df6b --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/SyncRecord.java @@ -0,0 +1,73 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records.record; + +public class SyncRecord implements Record { + + private final Object lock; + private final Record record; + + public SyncRecord(Record record) { + this(record, null); + } + + public SyncRecord(Record record, Object newLock) { + if (record == null) { + throw new IllegalArgumentException( + "Cannot create a SyncRecord on a null record"); + } else { + this.record = record; + this.lock = newLock == null ? this : newLock; + } + } + + @Override + public IntIterator keys() { + synchronized (this.lock) { + return this.record.keys(); + } + } + + @Override + public boolean containsKey(int key) { + synchronized (this.lock) { + return this.record.containsKey(key); + } + } + + @Override + public IntIterator get(int key) { + synchronized (this.lock) { + return this.record.get(key); + } + } + + @Override + public void addPath(int node, int parent) { + synchronized (this.lock) { + this.record.addPath(node, parent); + } + } + + @Override + public int size() { + return this.record.size(); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java index 733b11baa4..30d49bca74 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java @@ -31,12 +31,12 @@ public final class ObjectIntMapping { private final IntObjectHashMap int2IdMap; public ObjectIntMapping() { - this.int2IdMap = new IntObjectHashMap<>(1000000); + this.int2IdMap = new IntObjectHashMap<>(10000000); } @Watched @SuppressWarnings("unchecked") - public int object2Code(Object object) { + public synchronized int object2Code(Object object) { int key = object.hashCode(); // TODO: improve hash algorithm for (int i = 1; i > 0; i <<= 1) { @@ -56,7 +56,7 @@ public int object2Code(Object object) { } @Watched - public Object code2Object(int code) { + public synchronized Object code2Object(int code) { return this.int2IdMap.get(code); } } From 2501070381740d1a0206c0c9794c6bbb444a2dad Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Mon, 10 May 2021 21:52:52 +0800 Subject: [PATCH 24/38] improve Change-Id: I9e5926333dc6ad12fd081695e37994d77f26faab --- .../traversal/algorithm/FusiformSimilarityTraverser.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/FusiformSimilarityTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/FusiformSimilarityTraverser.java index 03cd92d214..39f807a21f 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/FusiformSimilarityTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/FusiformSimilarityTraverser.java @@ -308,5 +308,9 @@ public Map>> toMap() { } return results; } + + public boolean isEmpty() { + return this.similars.isEmpty(); + } } } From 9b71bd57db6537367b3c63623c530c2ab024b3f6 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 11 May 2021 19:30:57 +0800 Subject: [PATCH 25/38] improve Change-Id: I761abfa4940ebaa1dd331e9c60c80293bea5d2cb --- .../algorithm/KneighborTraverser.java | 80 +++++------- .../traversal/algorithm/KoutTraverser.java | 116 +++++++----------- .../traversal/algorithm/OltpTraverser.java | 21 ++-- .../algorithm/records/AbstractRecords.java | 20 +-- .../algorithm/records/KneighborRecords.java | 6 +- .../algorithm/records/KoutRecords.java | 4 +- .../records/SingleWayMultiPathsRecords.java | 10 +- .../records/record/RecordFactory.java | 6 +- .../algorithm/records/record/SyncRecord.java | 4 +- .../mapping/ConcurrentObjectIntMapping.java | 42 +++++++ .../collection/mapping/MappingFactory.java | 33 +++++ .../collection/mapping/ObjectIntMapping.java | 26 ++++ .../SingleObjectIntMapping.java} | 14 +-- 13 files changed, 217 insertions(+), 165 deletions(-) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/MappingFactory.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ObjectIntMapping.java rename hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/{ObjectIntMapping.java => mapping/SingleObjectIntMapping.java} (86%) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java index cec1b53ab3..54265b4a88 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java @@ -36,6 +36,8 @@ public class KneighborTraverser extends OltpTraverser { + private volatile boolean stop; + public KneighborTraverser(HugeGraph graph) { super(graph); } @@ -62,7 +64,7 @@ public Set kneighbor(Id sourceV, Directions dir, latest = this.adjacentVertices(sourceV, latest, dir, labelId, all, degree, remaining); all.addAll(latest); - if (limit != NO_LIMIT && all.size() >= limit) { + if (reachLimit(limit, all.size())) { break; } } @@ -77,61 +79,37 @@ public KneighborRecords customizedKneighbor(Id source, EdgeStep step, checkPositive(maxDepth, "k-neighbor max_depth"); checkLimit(limit); - boolean single = maxDepth < this.concurrentDepth() || - step.direction() != Directions.BOTH; - Traverser traverser = new Traverser(source, step, maxDepth, limit, - single); - return traverser.customizedKneighbor(); - } - - private class Traverser { - - private final KneighborRecords record; - - private final EdgeStep step; - private final long limit; - private final boolean single; - private int depth; - - private boolean stop; - - public Traverser(Id source, EdgeStep step, int maxDepth, - long limit, boolean single) { - this.record = new KneighborRecords(source, RecordType.INT, - true, single); - this.step = step; - this.depth = maxDepth; - this.limit = limit; - this.single = single; - this.stop = false; - } + boolean concurrent = maxDepth >= this.concurrentDepth() && + step.direction() == Directions.BOTH; - public KneighborRecords customizedKneighbor() { - Consumer consumer = v -> { - Iterator edges = edgesOfVertex(v, step); - while (edges.hasNext()) { - Id target = ((HugeEdge) edges.next()).id().otherVertexId(); - this.record.addPath(v, target); - this.checkLimit(this.limit, this.depth, this.record.size()); - } - }; - - while (this.depth-- > 0) { - this.record.startOneLayer(true); - traverseIds(this.record.keys(), consumer, - this.single, this.stop); - this.record.finishOneLayer(); - } - return this.record; - } + KneighborRecords records = new KneighborRecords(source, RecordType.INT, + true, concurrent); - private void checkLimit(long limit, long depth, int size) { - if (limit == NO_LIMIT || depth > 0) { + Consumer consumer = v -> { + if (this.stop) { return; } - if (size >= limit) { - this.stop = true; + Iterator edges = edgesOfVertex(v, step); + while (!this.stop && edges.hasNext()) { + Id target = ((HugeEdge) edges.next()).id().otherVertexId(); + records.addPath(v, target); + this.reachLimit(limit, records.size()); } + }; + + while (maxDepth-- > 0) { + records.startOneLayer(true); + traverseIds(records.keys(), consumer, concurrent); + records.finishOneLayer(); + } + return records; + } + + private boolean reachLimit(long limit, int size) { + if (limit == NO_LIMIT || size < limit) { + return false; } + this.stop = true; + return true; } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java index c3a3242867..dfe185d4d8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java @@ -37,6 +37,9 @@ public class KoutTraverser extends OltpTraverser { + private int depth; + private volatile boolean stop = false; + public KoutTraverser(HugeGraph graph) { super(graph); } @@ -51,6 +54,7 @@ public Set kout(Id sourceV, Directions dir, String label, checkDegree(degree); checkCapacity(capacity); checkLimit(limit); + this.depth = depth; if (capacity != NO_LIMIT) { // Capacity must > limit because sourceV is counted in capacity E.checkArgument(capacity >= limit && limit != NO_LIMIT, @@ -69,9 +73,9 @@ public Set kout(Id sourceV, Directions dir, String label, long remaining = capacity == NO_LIMIT ? NO_LIMIT : capacity - latest.size(); - while (depth-- > 0) { + while (this.depth-- > 0) { // Just get limit nodes in last layer if limit < remaining capacity - if (depth == 0 && limit != NO_LIMIT && + if (this.depth == 0 && limit != NO_LIMIT && (limit < remaining || remaining == NO_LIMIT)) { remaining = limit; } @@ -87,10 +91,10 @@ public Set kout(Id sourceV, Directions dir, String label, // Update 'remaining' value to record remaining capacity remaining -= latest.size(); - if (remaining <= 0 && depth > 0) { + if (remaining <= 0 && this.depth > 0) { throw new HugeException( "Reach capacity '%s' while remaining depth '%s'", - capacity, depth); + capacity, this.depth); } } } @@ -106,79 +110,51 @@ public KoutRecords customizedKout(Id source, EdgeStep step, checkPositive(maxDepth, "k-out max_depth"); checkCapacity(capacity); checkLimit(limit); + this.depth = maxDepth; + boolean concurrent = maxDepth >= this.concurrentDepth() && + step.direction() == Directions.BOTH; + KoutRecords records = new KoutRecords(source, RecordType.INT, + nearest, concurrent); + + Consumer consumer = v -> { + if (this.stop) { + return; + } + Iterator edges = edgesOfVertex(v, step); + while (!this.stop && edges.hasNext()) { + Id target = ((HugeEdge) edges.next()).id().otherVertexId(); + records.addPath(v, target); - boolean single = maxDepth < this.concurrentDepth() || - step.direction() != Directions.BOTH; - Traverser traverser = new Traverser(source, step, maxDepth, nearest, - capacity, limit, single); - return traverser.customizedKout(); - } + this.checkCapacity(capacity, records.accessed(), this.depth); + this.checkLimit(limit, this.depth, records.size()); + } + }; - private class Traverser { - - private final KoutRecords record; - - private final EdgeStep step; - private final long capacity; - private final long limit; - private final boolean single; - private int depth; - - private boolean stop; - - public Traverser(Id source, EdgeStep step, int maxDepth, - boolean nearest, long capacity, long limit, - boolean single) { - this.record = new KoutRecords(source, RecordType.INT, - nearest, single); - this.step = step; - this.depth = maxDepth; - this.capacity = capacity; - this.limit = limit; - this.single = single; - this.stop = false; + while (this.depth-- > 0) { + records.startOneLayer(true); + traverseIds(records.keys(), consumer, concurrent); + records.finishOneLayer(); } + return records; + } - public KoutRecords customizedKout() { - Consumer consumer = v -> { - Iterator edges = edgesOfVertex(v, step); - while (edges.hasNext()) { - Id target = ((HugeEdge) edges.next()).id().otherVertexId(); - this.record.addPath(v, target); - - this.checkCapacity(this.capacity, this.record.accessed(), - this.depth); - this.checkLimit(this.limit, this.depth, this.record.size()); - } - }; - - while (this.depth-- > 0) { - this.record.startOneLayer(true); - traverseIds(this.record.keys(), consumer, - this.single, this.stop); - this.record.finishOneLayer(); - } - return this.record; + private void checkCapacity(long capacity, long accessed, int depth) { + if (capacity == NO_LIMIT) { + return; } - - private void checkCapacity(long capacity, long accessed, int depth) { - if (capacity == NO_LIMIT) { - return; - } - if (capacity <= accessed && depth > 0) { - throw new HugeException( - "Reach capacity '%s' while remaining depth '%s'", - capacity, depth); - } + if (accessed >= capacity && depth > 0) { + throw new HugeException( + "Reach capacity '%s' while remaining depth '%s'", + capacity, depth); } + } - private void checkLimit(long limit, long depth, int size) { - if (limit == NO_LIMIT || depth > 0) { - return; - } - if (size >= limit) { - this.stop = true; - } + private void checkLimit(long limit, long depth, int size) { + if (limit == NO_LIMIT || depth > 0) { + return; + } + if (size >= limit) { + this.stop = true; } } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/OltpTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/OltpTraverser.java index f4bf1e3d8c..79cb7a5808 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/OltpTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/OltpTraverser.java @@ -135,17 +135,17 @@ protected Set adjacentVertices(Set vertices, EdgeStep step, protected long traverseNodes(Iterator vertices, Consumer consumer) { - return this.traverse(vertices, consumer, "traverse-nodes", false); + return this.traverse(vertices, consumer, "traverse-nodes"); } protected long traversePairs(Iterator> pairs, Consumer> consumer) { - return this.traverse(pairs, consumer, "traverse-pairs", false); + return this.traverse(pairs, consumer, "traverse-pairs"); } protected long traverseIds(Iterator ids, Consumer consumer, - boolean single, boolean stop) { - if (!single) { + boolean concurrent) { + if (concurrent) { return this.traverseIds(ids, consumer); } else { long count = 0L; @@ -158,17 +158,12 @@ protected long traverseIds(Iterator ids, Consumer consumer, } protected long traverseIds(Iterator ids, Consumer consumer) { - return this.traverseIds(ids, consumer, false); - } - - protected long traverseIds(Iterator ids, Consumer consumer, - boolean stop) { - return this.traverse(ids, consumer, "traverse-ids", stop); + return this.traverse(ids, consumer, "traverse-ids"); } protected long traverse(Iterator iterator, Consumer consumer, - String name, boolean stop) { - if (!iterator.hasNext() || stop) { + String name) { + if (!iterator.hasNext()) { return 0L; } @@ -177,7 +172,7 @@ protected long traverse(Iterator iterator, Consumer consumer, consumers.start(name); long total = 0L; try { - while (!stop && iterator.hasNext()) { + while (iterator.hasNext()) { total++; K v = iterator.next(); consumers.provide(v); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java index e137ee3ae8..fb92e854f9 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java @@ -24,24 +24,25 @@ import com.baidu.hugegraph.traversal.algorithm.records.record.Record; import com.baidu.hugegraph.traversal.algorithm.records.record.RecordFactory; import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; -import com.baidu.hugegraph.util.collection.ObjectIntMapping; +import com.baidu.hugegraph.util.collection.mapping.ObjectIntMapping; +import com.baidu.hugegraph.util.collection.mapping.MappingFactory; public abstract class AbstractRecords implements Records { private final ObjectIntMapping idMapping; - protected final RecordType type; - private final boolean single; + private final RecordType type; + private final boolean concurrent; - public AbstractRecords(RecordType type, boolean single) { - this.idMapping = new ObjectIntMapping<>(); + public AbstractRecords(RecordType type, boolean concurrent) { this.type = type; - this.single = single; + this.concurrent = concurrent; + this.idMapping = MappingFactory.newObjectIntMapping(this.concurrent); } public AbstractRecords(RecordType type) { - this.idMapping = new ObjectIntMapping<>(); this.type = type; - this.single = true; + this.concurrent = false; + this.idMapping = MappingFactory.newObjectIntMapping(); } @Watched @@ -55,7 +56,6 @@ protected Id id(int code) { } protected Record newRecord() { - return RecordFactory.newRecord(this.type, single); + return RecordFactory.newRecord(this.type, this.concurrent); } } - diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java index f4abcb6b86..723d000e83 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java @@ -34,11 +34,11 @@ public class KneighborRecords extends SingleWayMultiPathsRecords { - private Id source; + private final Id source; public KneighborRecords(Id source, RecordType type, - boolean nearest, boolean single) { - super(source, type, nearest, single); + boolean nearest, boolean concurrent) { + super(source, type, nearest, concurrent); this.source = source; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java index aec6f2d137..e48707a221 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java @@ -34,8 +34,8 @@ public class KoutRecords extends SingleWayMultiPathsRecords { public KoutRecords(Id source, RecordType type, - boolean nearest, boolean single) { - super(source, type, nearest, single); + boolean nearest, boolean concurrent) { + super(source, type, nearest, concurrent); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java index 4a2d3aa239..55e7b46c83 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java @@ -49,16 +49,16 @@ public class SingleWayMultiPathsRecords extends AbstractRecords { protected final Stack records; private final boolean nearest; + private final Set accessedVertices; protected Record currentRecord; protected IntIterator lastRecordKeys; protected int current; protected boolean forward; - private final Set accessedVertices; public SingleWayMultiPathsRecords(Id source, RecordType type, - boolean nearest, boolean single) { - super(type, single); + boolean nearest, boolean concurrent) { + super(type, concurrent); this.nearest = nearest; @@ -68,8 +68,8 @@ public SingleWayMultiPathsRecords(Id source, RecordType type, this.records = new Stack<>(); this.records.push(firstRecord); - this.accessedVertices = single ? new HashSet() : - ConcurrentHashMap.newKeySet(); + this.accessedVertices = concurrent ? ConcurrentHashMap.newKeySet() : + new HashSet(); this.accessedVertices.add(sourceCode); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordFactory.java index 4fd6a6f1f4..4f3320614e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordFactory.java @@ -22,10 +22,10 @@ public class RecordFactory { public static Record newRecord(RecordType type) { - return newRecord(type, true); + return newRecord(type, false); } - public static Record newRecord(RecordType type, boolean single) { + public static Record newRecord(RecordType type, boolean concurrent) { Record record; switch (type) { case ARRAY: @@ -40,7 +40,7 @@ record = new IntIntRecord(); default: throw new AssertionError("Unsupported record type: " + type); } - if (!single) { + if (concurrent) { record = new SyncRecord(record); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/SyncRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/SyncRecord.java index 128057df6b..b6c721e5fd 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/SyncRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/SyncRecord.java @@ -68,6 +68,8 @@ public void addPath(int node, int parent) { @Override public int size() { - return this.record.size(); + synchronized (this.lock) { + return this.record.size(); + } } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java new file mode 100644 index 0000000000..92d0837d50 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java @@ -0,0 +1,42 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.util.collection.mapping; + +import com.baidu.hugegraph.perf.PerfUtil; + +public class ConcurrentObjectIntMapping implements ObjectIntMapping { + + private SingleObjectIntMapping singleObjectIntMapping; + + public ConcurrentObjectIntMapping() { + this.singleObjectIntMapping = new SingleObjectIntMapping<>(); + } + + @PerfUtil.Watched + @SuppressWarnings("unchecked") + public synchronized int object2Code(Object object) { + return this.singleObjectIntMapping.object2Code(object); + } + + @PerfUtil.Watched + public synchronized Object code2Object(int code) { + return this.singleObjectIntMapping.code2Object(code); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/MappingFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/MappingFactory.java new file mode 100644 index 0000000000..1060ee9b53 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/MappingFactory.java @@ -0,0 +1,33 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.util.collection.mapping; + +public class MappingFactory { + + public static ObjectIntMapping newObjectIntMapping() { + return newObjectIntMapping(false); + } + + public static ObjectIntMapping newObjectIntMapping( + boolean concurrent) { + return concurrent ? new ConcurrentObjectIntMapping<>() : + new SingleObjectIntMapping<>(); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ObjectIntMapping.java new file mode 100644 index 0000000000..2bf66a4673 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ObjectIntMapping.java @@ -0,0 +1,26 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.util.collection.mapping; + +public interface ObjectIntMapping { + + public int object2Code(Object object); + public Object code2Object(int code); +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java similarity index 86% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java index 30d49bca74..9bea0178e9 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java @@ -17,24 +17,24 @@ * under the License. */ -package com.baidu.hugegraph.util.collection; +package com.baidu.hugegraph.util.collection.mapping; import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.perf.PerfUtil.Watched; +import com.baidu.hugegraph.perf.PerfUtil; -public final class ObjectIntMapping { +public class SingleObjectIntMapping implements ObjectIntMapping { private static final int MAGIC = 1 << 16; private final IntObjectHashMap int2IdMap; - public ObjectIntMapping() { - this.int2IdMap = new IntObjectHashMap<>(10000000); + public SingleObjectIntMapping() { + this.int2IdMap = new IntObjectHashMap<>(); } - @Watched + @PerfUtil.Watched @SuppressWarnings("unchecked") public synchronized int object2Code(Object object) { int key = object.hashCode(); @@ -55,7 +55,7 @@ public synchronized int object2Code(Object object) { throw new HugeException("Failed to get code for id: %s", object); } - @Watched + @PerfUtil.Watched public synchronized Object code2Object(int code) { return this.int2IdMap.get(code); } From fe483c7802681497285c429fe2d0c3acc00b3133 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 11 May 2021 22:21:49 +0800 Subject: [PATCH 26/38] add objectintmapping unit test Change-Id: Ib653623d5cc79696f58ec50ccadca8ac7039c2f1 --- .../mapping/ConcurrentObjectIntMapping.java | 10 +- .../collection/mapping/ObjectIntMapping.java | 1 + .../mapping/SingleObjectIntMapping.java | 10 +- .../unit/core/ObjectIntMappingTest.java | 142 ++++++++++++++++++ 4 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/ObjectIntMappingTest.java diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java index 92d0837d50..fb2b202b12 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java @@ -19,7 +19,7 @@ package com.baidu.hugegraph.util.collection.mapping; -import com.baidu.hugegraph.perf.PerfUtil; +import com.baidu.hugegraph.perf.PerfUtil.Watched; public class ConcurrentObjectIntMapping implements ObjectIntMapping { @@ -29,14 +29,18 @@ public ConcurrentObjectIntMapping() { this.singleObjectIntMapping = new SingleObjectIntMapping<>(); } - @PerfUtil.Watched + @Watched @SuppressWarnings("unchecked") public synchronized int object2Code(Object object) { return this.singleObjectIntMapping.object2Code(object); } - @PerfUtil.Watched + @Watched public synchronized Object code2Object(int code) { return this.singleObjectIntMapping.code2Object(code); } + + public synchronized void clear() { + this.singleObjectIntMapping.clear(); + } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ObjectIntMapping.java index 2bf66a4673..45d63b4747 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ObjectIntMapping.java @@ -23,4 +23,5 @@ public interface ObjectIntMapping { public int object2Code(Object object); public Object code2Object(int code); + public void clear(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java index 9bea0178e9..281a0ed736 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java @@ -23,7 +23,7 @@ import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.perf.PerfUtil; +import com.baidu.hugegraph.perf.PerfUtil.Watched; public class SingleObjectIntMapping implements ObjectIntMapping { @@ -34,7 +34,7 @@ public SingleObjectIntMapping() { this.int2IdMap = new IntObjectHashMap<>(); } - @PerfUtil.Watched + @Watched @SuppressWarnings("unchecked") public synchronized int object2Code(Object object) { int key = object.hashCode(); @@ -55,8 +55,12 @@ public synchronized int object2Code(Object object) { throw new HugeException("Failed to get code for id: %s", object); } - @PerfUtil.Watched + @Watched public synchronized Object code2Object(int code) { return this.int2IdMap.get(code); } + + public void clear() { + this.int2IdMap.clear(); + } } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/ObjectIntMappingTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/ObjectIntMappingTest.java new file mode 100644 index 0000000000..1f20e7e262 --- /dev/null +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/ObjectIntMappingTest.java @@ -0,0 +1,142 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.unit.core; + +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.UUID; + +import org.apache.commons.lang.RandomStringUtils; +import org.junit.After; +import org.junit.Test; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.id.IdGenerator; +import com.baidu.hugegraph.testutil.Assert; +import com.baidu.hugegraph.util.collection.mapping.MappingFactory; +import com.baidu.hugegraph.util.collection.mapping.ObjectIntMapping; + +public class ObjectIntMappingTest { + + private static ObjectIntMapping mapping = + MappingFactory.newObjectIntMapping(); + + @After + public void clear() { + mapping.clear(); + } + + @Test + public void testNumberIdMapping() { + Set codes = new LinkedHashSet<>(); + for (int i = 0; i < 1000000; i++) { + codes.add(mapping.object2Code(IdGenerator.of(i))); + } + Assert.assertEquals(1000000, codes.size()); + + int j = 0; + for (Integer code : codes) { + Assert.assertEquals(IdGenerator.of(j++), mapping.code2Object(code)); + } + } + + @Test + public void testStringIdMapping() { + Set strings = new LinkedHashSet<>(); + Set codes = new LinkedHashSet<>(); + for (int i = 0; i < 1000000; i++) { + String string = RandomStringUtils.randomAlphanumeric(10); + strings.add(string); + codes.add(mapping.object2Code(IdGenerator.of(string))); + } + Assert.assertEquals(strings.size(), codes.size()); + + Iterator strIter = strings.iterator(); + Iterator codeIter = codes.iterator(); + while (strIter.hasNext() && codeIter.hasNext()) { + Assert.assertEquals(IdGenerator.of(strIter.next()), + mapping.code2Object(codeIter.next())); + } + + Assert.assertFalse(strIter.hasNext()); + Assert.assertFalse(codeIter.hasNext()); + } + + @Test + public void testUUIDIdMapping() { + Set uuids = new LinkedHashSet<>(); + Set codes = new LinkedHashSet<>(); + for (int i = 0; i < 1000000; i++) { + UUID uuid = UUID.randomUUID(); + uuids.add(uuid); + codes.add(mapping.object2Code(IdGenerator.of(uuid))); + } + Assert.assertEquals(uuids.size(), codes.size()); + + Iterator uuidIter = uuids.iterator(); + Iterator codeIter = codes.iterator(); + while (uuidIter.hasNext() && codeIter.hasNext()) { + Assert.assertEquals(IdGenerator.of(uuidIter.next()), + mapping.code2Object(codeIter.next())); + } + + Assert.assertFalse(uuidIter.hasNext()); + Assert.assertFalse(codeIter.hasNext()); + } + + @Test + public void testMixedIdMapping() { + Set codes = new LinkedHashSet<>(); + Set objects = new LinkedHashSet<>(); + Object object; + + for (int i = 0; i < 1000000; i++) { + object = IdGenerator.of(i); + objects.add(object); + codes.add(mapping.object2Code(object)); + } + + for (int i = 0; i < 1000000; i++) { + String string = RandomStringUtils.randomAlphanumeric(10); + object = IdGenerator.of(string); + objects.add(object); + codes.add(mapping.object2Code(object)); + } + + for (int i = 0; i < 1000000; i++) { + UUID uuid = UUID.randomUUID(); + object = IdGenerator.of(uuid); + objects.add(object); + codes.add(mapping.object2Code(object)); + } + Assert.assertEquals(objects.size(), codes.size()); + + Iterator objectIter = objects.iterator(); + Iterator codeIter = codes.iterator(); + while (objectIter.hasNext() && codeIter.hasNext()) { + Assert.assertEquals(objectIter.next(), + mapping.code2Object(codeIter.next())); + } + + Assert.assertFalse(objectIter.hasNext()); + Assert.assertFalse(codeIter.hasNext()); + } +} From 00d57413b15f42507bded091b17c1a8d7d67368e Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Wed, 12 May 2021 15:07:53 +0800 Subject: [PATCH 27/38] improve Change-Id: I742ce92cea71f556d0c7b5a52229e46006f759e7 --- .../traversal/algorithm/KneighborTraverser.java | 5 +++-- .../traversal/algorithm/KoutTraverser.java | 4 ++-- .../traversal/algorithm/PathsTraverser.java | 6 +++--- .../algorithm/records/AbstractRecords.java | 6 ------ .../records/DoubleWayMultiPathsRecords.java | 6 +++--- .../algorithm/records/KneighborRecords.java | 6 +++--- .../traversal/algorithm/records/KoutRecords.java | 7 +++---- .../algorithm/records/ShortestPathRecords.java | 2 +- .../records/SingleWayMultiPathsRecords.java | 13 +++++-------- .../mapping/ConcurrentObjectIntMapping.java | 1 + .../collection/mapping/SingleObjectIntMapping.java | 1 + .../com/baidu/hugegraph/unit/UnitTestSuite.java | 2 ++ 12 files changed, 27 insertions(+), 32 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java index 54265b4a88..eaf7df377d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java @@ -82,8 +82,9 @@ public KneighborRecords customizedKneighbor(Id source, EdgeStep step, boolean concurrent = maxDepth >= this.concurrentDepth() && step.direction() == Directions.BOTH; - KneighborRecords records = new KneighborRecords(source, RecordType.INT, - true, concurrent); + KneighborRecords records = new KneighborRecords(RecordType.INT, + concurrent, + source, true); Consumer consumer = v -> { if (this.stop) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java index dfe185d4d8..9099d5e6dd 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java @@ -113,8 +113,8 @@ public KoutRecords customizedKout(Id source, EdgeStep step, this.depth = maxDepth; boolean concurrent = maxDepth >= this.concurrentDepth() && step.direction() == Directions.BOTH; - KoutRecords records = new KoutRecords(source, RecordType.INT, - nearest, concurrent); + KoutRecords records = new KoutRecords(RecordType.INT, concurrent, + source, nearest); Consumer consumer = v -> { if (this.stop) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java index 4f5e839b0b..aa07ba2d5c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java @@ -31,7 +31,6 @@ import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; -import com.google.common.collect.ImmutableList; public class PathsTraverser extends HugeTraverser { @@ -94,8 +93,9 @@ private class Traverser { public Traverser(Id sourceV, Id targetV, Id label, long degree, long capacity, long limit) { - this.record = new DoubleWayMultiPathsRecords(sourceV, targetV, - RecordType.ARRAY); + this.record = new DoubleWayMultiPathsRecords(RecordType.ARRAY, + false, + sourceV, targetV); this.label = label; this.degree = degree; this.capacity = capacity; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java index fb92e854f9..9a5431b09a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java @@ -39,12 +39,6 @@ public AbstractRecords(RecordType type, boolean concurrent) { this.idMapping = MappingFactory.newObjectIntMapping(this.concurrent); } - public AbstractRecords(RecordType type) { - this.type = type; - this.concurrent = false; - this.idMapping = MappingFactory.newObjectIntMapping(); - } - @Watched protected int code(Id id) { return this.idMapping.object2Code(id); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java index 433e7338cc..a6bc4b7a73 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java @@ -47,9 +47,9 @@ public class DoubleWayMultiPathsRecords extends AbstractRecords { protected boolean forward; private int accessed; - public DoubleWayMultiPathsRecords(Id sourceV, Id targetV, RecordType type) { - super(type); - + public DoubleWayMultiPathsRecords(RecordType type, boolean concurrent, + Id sourceV, Id targetV) { + super(type, concurrent); int sourceCode = this.code(sourceV); int targetCode = this.code(targetV); Record firstSourceRecord = this.newRecord(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java index 723d000e83..5302719b09 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java @@ -36,9 +36,9 @@ public class KneighborRecords extends SingleWayMultiPathsRecords { private final Id source; - public KneighborRecords(Id source, RecordType type, - boolean nearest, boolean concurrent) { - super(source, type, nearest, concurrent); + public KneighborRecords(RecordType type, boolean concurrent, + Id source, boolean nearest) { + super(type, concurrent, source, nearest); this.source = source; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java index e48707a221..41d6a44086 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java @@ -32,10 +32,9 @@ public class KoutRecords extends SingleWayMultiPathsRecords { - public KoutRecords(Id source, - RecordType type, - boolean nearest, boolean concurrent) { - super(source, type, nearest, concurrent); + public KoutRecords(RecordType type, boolean concurrent, + Id source, boolean nearest) { + super(type, concurrent, source, nearest); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java index dcb51b292a..336b76a3bf 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java @@ -40,7 +40,7 @@ public class ShortestPathRecords extends DoubleWayMultiPathsRecords { private boolean pathFound; public ShortestPathRecords(Id sourceV, Id targetV) { - super(sourceV, targetV, RecordType.INT); + super(RecordType.INT, false, sourceV, targetV); this.accessedVertices = new IntHashSet(); this.accessedVertices.add(this.code(sourceV)); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java index 55e7b46c83..e9185d874f 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java @@ -21,12 +21,9 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Set; import java.util.Stack; -import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import org.eclipse.collections.api.set.primitive.MutableIntSet; @@ -49,15 +46,15 @@ public class SingleWayMultiPathsRecords extends AbstractRecords { protected final Stack records; private final boolean nearest; - private final Set accessedVertices; + private final MutableIntSet accessedVertices; protected Record currentRecord; protected IntIterator lastRecordKeys; protected int current; protected boolean forward; - public SingleWayMultiPathsRecords(Id source, RecordType type, - boolean nearest, boolean concurrent) { + public SingleWayMultiPathsRecords(RecordType type, boolean concurrent, + Id source, boolean nearest) { super(type, concurrent); this.nearest = nearest; @@ -68,8 +65,8 @@ public SingleWayMultiPathsRecords(Id source, RecordType type, this.records = new Stack<>(); this.records.push(firstRecord); - this.accessedVertices = concurrent ? ConcurrentHashMap.newKeySet() : - new HashSet(); + this.accessedVertices = concurrent ? new IntHashSet().asSynchronized() : + new IntHashSet(); this.accessedVertices.add(sourceCode); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java index fb2b202b12..301fe5ffd7 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java @@ -40,6 +40,7 @@ public synchronized Object code2Object(int code) { return this.singleObjectIntMapping.code2Object(code); } + @Override public synchronized void clear() { this.singleObjectIntMapping.clear(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java index 281a0ed736..2e6bcb2a26 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java @@ -60,6 +60,7 @@ public synchronized Object code2Object(int code) { return this.int2IdMap.get(code); } + @Override public void clear() { this.int2IdMap.clear(); } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java index 7071d960be..84b88dfea2 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java @@ -39,6 +39,7 @@ import com.baidu.hugegraph.unit.core.Int2IntsMapTest; import com.baidu.hugegraph.unit.core.LocksTableTest; import com.baidu.hugegraph.unit.core.PageStateTest; +import com.baidu.hugegraph.unit.core.ObjectIntMappingTest; import com.baidu.hugegraph.unit.core.QueryTest; import com.baidu.hugegraph.unit.core.RangeTest; import com.baidu.hugegraph.unit.core.RolePermissionTest; @@ -105,6 +106,7 @@ TraversalUtilTest.class, PageStateTest.class, Int2IntsMapTest.class, + ObjectIntMappingTest.class, /* serializer */ BytesBufferTest.class, From bc74387cdfa7f5956375189425dfcb93b735223d Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 20 May 2021 11:52:23 +0800 Subject: [PATCH 28/38] add some LOG info for debug Change-Id: Ia6a8e3316c40ecf44f6f36c4cfee50db2f7d11fe --- .../com/baidu/hugegraph/backend/cache/AbstractCache.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/AbstractCache.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/AbstractCache.java index 8243b0360b..9659cd8191 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/AbstractCache.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/AbstractCache.java @@ -180,6 +180,9 @@ public long tick() { long current = now(); for (Iterator> it = this.nodes(); it.hasNext();) { CacheNode node = it.next(); + LOG.info("expireTime: {}, current: {}, node.time(): {}, " + + "current-node.time(): {}, node: {}", + expireTime, current, node.time(), current - node.time(), node); if (current - node.time() > expireTime) { // Remove item while iterating map (it must be ConcurrentMap) this.remove(node.key()); @@ -188,7 +191,7 @@ public long tick() { } if (expireItems > 0) { - LOG.debug("Cache expired {} items cost {}ms (size {}, expire {}ms)", + LOG.info("Cache expired {} items cost {}ms (size {}, expire {}ms)", expireItems, now() - current, this.size(), expireTime); } return expireItems; From bb0d39d929d3b6df547fbad447da0a0b447321af Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 20 May 2021 15:27:34 +0800 Subject: [PATCH 29/38] reduce expire time for CacheManagerTest.testCacheExpire() Change-Id: I9588f2df9a6db197dcd0d35c98b7d99993a9aabf --- .../java/com/baidu/hugegraph/unit/cache/CacheManagerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/cache/CacheManagerTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/cache/CacheManagerTest.java index ac06bd04a4..16a911e417 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/cache/CacheManagerTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/cache/CacheManagerTest.java @@ -234,7 +234,7 @@ public void testCacheListModify() { @Test public void testCacheExpire() { Cache cache1 = new RamCache(); - cache1.expire(28 * 1000L); + cache1.expire(26 * 1000L); Cache cache2 = new RamCache(); cache2.expire(0); From 448325c4ab3e890cc1d6f0f200376e0ade465e13 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 20 May 2021 16:07:01 +0800 Subject: [PATCH 30/38] remove debug LOG info Change-Id: I1358fb9216dd2a8d04be0e8772cd715c79c61b1e --- .../com/baidu/hugegraph/backend/cache/AbstractCache.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/AbstractCache.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/AbstractCache.java index 9659cd8191..8243b0360b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/AbstractCache.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/AbstractCache.java @@ -180,9 +180,6 @@ public long tick() { long current = now(); for (Iterator> it = this.nodes(); it.hasNext();) { CacheNode node = it.next(); - LOG.info("expireTime: {}, current: {}, node.time(): {}, " + - "current-node.time(): {}, node: {}", - expireTime, current, node.time(), current - node.time(), node); if (current - node.time() > expireTime) { // Remove item while iterating map (it must be ConcurrentMap) this.remove(node.key()); @@ -191,7 +188,7 @@ public long tick() { } if (expireItems > 0) { - LOG.info("Cache expired {} items cost {}ms (size {}, expire {}ms)", + LOG.debug("Cache expired {} items cost {}ms (size {}, expire {}ms)", expireItems, now() - current, this.size(), expireTime); } return expireItems; From 0ac67e97a4345412f4f429815cc9605b7b0f9871 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Fri, 21 May 2021 11:57:27 +0800 Subject: [PATCH 31/38] improve Change-Id: Icb18c665b27dced23415bdcfc582fa372d1fe2ba --- .../hugegraph/auth/HugeGraphAuthProxy.java | 3 +- .../java/com/baidu/hugegraph/HugeGraph.java | 3 +- .../baidu/hugegraph/StandardHugeGraph.java | 5 +- .../baidu/hugegraph/config/CoreOptions.java | 6 +- .../baidu/hugegraph/structure/HugeVertex.java | 26 +++++--- .../traversal/algorithm/HugeTraverser.java | 25 ++------ .../algorithm/KneighborTraverser.java | 13 +--- .../traversal/algorithm/KoutTraverser.java | 17 ++--- .../traversal/algorithm/OltpTraverser.java | 62 ------------------- .../records/DoubleWayMultiPathsRecords.java | 2 +- .../algorithm/records/KneighborRecords.java | 16 ++--- .../algorithm/records/KoutRecords.java | 4 +- .../records/SingleWayMultiPathsRecords.java | 6 +- ...ctionImplType.java => CollectionType.java} | 8 +-- .../util/collection/CollectionFactory.java | 30 +++++---- .../hugegraph/util/collection/IdSet.java | 4 +- 16 files changed, 75 insertions(+), 155 deletions(-) rename hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/{CollectionImplType.java => CollectionType.java} (88%) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java index 34fcc6d495..807d3c4f71 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java @@ -65,6 +65,7 @@ import com.baidu.hugegraph.backend.store.raft.RaftGroupManager; import com.baidu.hugegraph.config.ConfigOption; import com.baidu.hugegraph.config.HugeConfig; +import com.baidu.hugegraph.config.TypedOption; import com.baidu.hugegraph.exception.NotSupportException; import com.baidu.hugegraph.iterator.FilterIterator; import com.baidu.hugegraph.rpc.RpcServiceConfig4Client; @@ -552,7 +553,7 @@ public long now() { } @Override - public V option(ConfigOption option) { + public V option(TypedOption option) { this.verifyAnyPermission(); return this.hugegraph.option(option); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java index ed26bbb5d0..e9b93f4d86 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java @@ -38,6 +38,7 @@ import com.baidu.hugegraph.backend.store.BackendStoreSystemInfo; import com.baidu.hugegraph.backend.store.raft.RaftGroupManager; import com.baidu.hugegraph.config.ConfigOption; +import com.baidu.hugegraph.config.TypedOption; import com.baidu.hugegraph.rpc.RpcServiceConfig4Client; import com.baidu.hugegraph.rpc.RpcServiceConfig4Server; import com.baidu.hugegraph.schema.EdgeLabel; @@ -170,7 +171,7 @@ public interface HugeGraph extends Graph { public long now(); - public V option(ConfigOption option); + public V option(TypedOption option); public void registerRpcServices(RpcServiceConfig4Server serverConfig, RpcServiceConfig4Client clientConfig); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java index 1a3e87547c..a45aeb7664 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java @@ -67,6 +67,7 @@ import com.baidu.hugegraph.config.ConfigOption; import com.baidu.hugegraph.config.CoreOptions; import com.baidu.hugegraph.config.HugeConfig; +import com.baidu.hugegraph.config.TypedOption; import com.baidu.hugegraph.event.EventHub; import com.baidu.hugegraph.event.EventListener; import com.baidu.hugegraph.exception.NotAllowException; @@ -116,7 +117,7 @@ public class StandardHugeGraph implements HugeGraph { StandardHugeGraph.SysTransaction.class }; - public static final Set> ALLOWED_CONFIGS = ImmutableSet.of( + public static final Set> ALLOWED_CONFIGS = ImmutableSet.of( CoreOptions.TASK_WAIT_TIMEOUT, CoreOptions.TASK_SYNC_DELETION, CoreOptions.TASK_TTL_DELETE_BATCH, @@ -978,7 +979,7 @@ public long now() { } @Override - public V option(ConfigOption option) { + public V option(TypedOption option) { HugeConfig config = this.configuration(); if (!ALLOWED_CONFIGS.contains(option)) { throw new NotAllowException("Not allowed to access config: %s", diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java index 60eede09a8..4c90d4db0d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java @@ -26,6 +26,7 @@ import static com.baidu.hugegraph.config.OptionChecker.rangeInt; import com.baidu.hugegraph.backend.query.Query; +import com.baidu.hugegraph.type.define.CollectionType; import com.baidu.hugegraph.util.Bytes; public class CoreOptions extends OptionHolder { @@ -624,12 +625,13 @@ public static synchronized CoreOptions instance() { 10 ); - public static final ConfigOption OLTP_COLLECTION_IMPL_TYPE = - new ConfigOption<>( + public static final ConfigConvOption OLTP_COLLECTION_IMPL_TYPE = + new ConfigConvOption<>( "oltp.collection_impl_type", "The implementation type of collections " + "used in oltp algorithm.", allowValues("jcf", "ec", "fu"), + CollectionType::valueOf, "ec" ); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java index c707983d8d..63ae52b5e3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java @@ -53,7 +53,7 @@ import com.baidu.hugegraph.schema.VertexLabel; import com.baidu.hugegraph.type.HugeType; import com.baidu.hugegraph.type.define.Cardinality; -import com.baidu.hugegraph.type.define.CollectionImplType; +import com.baidu.hugegraph.type.define.CollectionType; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.type.define.IdStrategy; @@ -223,7 +223,11 @@ public Collection getEdges() { } public void resetEdges() { - this.edges = CollectionFactory.newList(CollectionImplType.EC); + /* + * Use list to hold edges for vertices to reduce memory usage and + * add operation time. + */ + this.edges = newList(); } public void removeEdge(HugeEdge edge) { @@ -232,7 +236,7 @@ public void removeEdge(HugeEdge edge) { public void addEdge(HugeEdge edge) { if (this.edges == EMPTY_LIST) { - this.edges = CollectionFactory.newList(CollectionImplType.EC); + this.edges = CollectionFactory.newList(CollectionType.EC); } this.edges.add(edge); } @@ -642,6 +646,14 @@ public static HugeVertex create(final GraphTransaction tx, return new HugeVertex4Insert(tx, id, label); } + private static Set newSet() { + return CollectionFactory.newSet(CollectionType.EC); + } + + private static List newList() { + return CollectionFactory.newList(CollectionType.EC); + } + private static final class HugeVertex4Insert extends HugeVertex { private GraphTransaction tx; @@ -649,6 +661,10 @@ private static final class HugeVertex4Insert extends HugeVertex { public HugeVertex4Insert(final GraphTransaction tx, Id id, VertexLabel label) { super(tx.graph(), id, label); + /* + * Use set to hold edges for inserted vertex + * to avoid duplicated edges + */ this.edges = newSet(); this.tx = tx; this.fresh(true); @@ -679,9 +695,5 @@ protected GraphTransaction tx() { } return null; } - - private static Set newSet() { - return CollectionFactory.newSet(CollectionImplType.EC); - } } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java index 615d2ca8cb..1b8f82be88 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java @@ -54,7 +54,7 @@ import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.traversal.optimize.TraversalUtil; import com.baidu.hugegraph.type.HugeType; -import com.baidu.hugegraph.type.define.CollectionImplType; +import com.baidu.hugegraph.type.define.CollectionType; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.util.CollectionUtil; @@ -103,9 +103,8 @@ protected int concurrentDepth() { return this.graph.option(CoreOptions.OLTP_CONCURRENT_DEPTH); } - private CollectionImplType collectionImplType() { - return CollectionImplType.valueOf(this.graph.option( - CoreOptions.OLTP_COLLECTION_IMPL_TYPE).toUpperCase()); + private CollectionType collectionImplType() { + return this.graph.option(CoreOptions.OLTP_COLLECTION_IMPL_TYPE); } protected Set adjacentVertices(Id sourceV, Set vertices, @@ -553,22 +552,6 @@ public String toString() { } } - public static class KNode extends Node { - - public KNode(Id id, KNode parent) { - super(id, parent); - } - - @Override - public boolean equals(Object object) { - if (!(object instanceof KNode)) { - return false; - } - KNode other = (KNode) object; - return Objects.equals(this.id(), other.id()); - } - } - public static class Path { public static final Path EMPTY_PATH = new Path(ImmutableList.of()); @@ -589,7 +572,7 @@ public Id crosspoint() { return this.crosspoint; } - public void addToEnd(Id id) { + public void addToLast(Id id) { this.vertices.add(id); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java index eaf7df377d..55c71c5c0a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KneighborTraverser.java @@ -36,8 +36,6 @@ public class KneighborTraverser extends OltpTraverser { - private volatile boolean stop; - public KneighborTraverser(HugeGraph graph) { super(graph); } @@ -87,14 +85,13 @@ public KneighborRecords customizedKneighbor(Id source, EdgeStep step, source, true); Consumer consumer = v -> { - if (this.stop) { + if (this.reachLimit(limit, records.size())) { return; } Iterator edges = edgesOfVertex(v, step); - while (!this.stop && edges.hasNext()) { + while (!this.reachLimit(limit, records.size()) && edges.hasNext()) { Id target = ((HugeEdge) edges.next()).id().otherVertexId(); records.addPath(v, target); - this.reachLimit(limit, records.size()); } }; @@ -107,10 +104,6 @@ public KneighborRecords customizedKneighbor(Id source, EdgeStep step, } private boolean reachLimit(long limit, int size) { - if (limit == NO_LIMIT || size < limit) { - return false; - } - this.stop = true; - return true; + return limit != NO_LIMIT && size >= limit; } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java index 9099d5e6dd..812d85924e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java @@ -38,7 +38,6 @@ public class KoutTraverser extends OltpTraverser { private int depth; - private volatile boolean stop = false; public KoutTraverser(HugeGraph graph) { super(graph); @@ -117,16 +116,15 @@ public KoutRecords customizedKout(Id source, EdgeStep step, source, nearest); Consumer consumer = v -> { - if (this.stop) { + if (this.reachLimit(limit, this.depth, records.size())) { return; } Iterator edges = edgesOfVertex(v, step); - while (!this.stop && edges.hasNext()) { + while (!this.reachLimit(limit, this.depth, records.size()) && + edges.hasNext()) { Id target = ((HugeEdge) edges.next()).id().otherVertexId(); records.addPath(v, target); - this.checkCapacity(capacity, records.accessed(), this.depth); - this.checkLimit(limit, this.depth, records.size()); } }; @@ -149,12 +147,7 @@ private void checkCapacity(long capacity, long accessed, int depth) { } } - private void checkLimit(long limit, long depth, int size) { - if (limit == NO_LIMIT || depth > 0) { - return; - } - if (size >= limit) { - this.stop = true; - } + private boolean reachLimit(long limit, long depth, int size) { + return limit != NO_LIMIT && depth <= 0 && size >= limit; } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/OltpTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/OltpTraverser.java index 79cb7a5808..af4a73cc39 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/OltpTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/OltpTraverser.java @@ -21,14 +21,11 @@ import java.util.Iterator; import java.util.List; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import org.apache.commons.lang3.tuple.Pair; -import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.Element; import org.apache.tinkerpop.gremlin.structure.Property; import org.apache.tinkerpop.gremlin.structure.Vertex; @@ -38,8 +35,6 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.config.CoreOptions; import com.baidu.hugegraph.iterator.FilterIterator; -import com.baidu.hugegraph.structure.HugeEdge; -import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.util.Consumers; import jersey.repackaged.com.google.common.base.Objects; @@ -81,63 +76,6 @@ public static void destroy() { } } - protected Set adjacentVertices(Id source, Set latest, - EdgeStep step, Set all, - long remaining, boolean single) { - if (single) { - return this.adjacentVertices(source, latest, step, all, remaining); - } else { - AtomicLong remain = new AtomicLong(remaining); - return this.adjacentVertices(latest, step, all, remain); - } - } - - protected Set adjacentVertices(Set vertices, EdgeStep step, - Set excluded, long remaining) { - Set neighbors = newSet(); - for (Node source : vertices) { - Iterator edges = this.edgesOfVertex(source.id(), step); - while (edges.hasNext()) { - Id target = ((HugeEdge) edges.next()).id().otherVertexId(); - KNode kNode = new KNode(target, (KNode) source); - if (excluded != null && excluded.contains(kNode)) { - continue; - } - neighbors.add(kNode); - if (remaining != NO_LIMIT && --remaining <= 0L) { - return neighbors; - } - } - } - return neighbors; - } - - protected Set adjacentVertices(Set vertices, EdgeStep step, - Set excluded, - AtomicLong remaining) { - Set neighbors = ConcurrentHashMap.newKeySet(); - this.traverseNodes(vertices.iterator(), v -> { - Iterator edges = this.edgesOfVertex(v.id(), step); - while (edges.hasNext()) { - Id target = ((HugeEdge) edges.next()).id().otherVertexId(); - KNode kNode = new KNode(target, (KNode) v); - if (excluded != null && excluded.contains(kNode)) { - continue; - } - neighbors.add(kNode); - if (remaining.decrementAndGet() <= 0L) { - return; - } - } - }); - return neighbors; - } - - protected long traverseNodes(Iterator vertices, - Consumer consumer) { - return this.traverse(vertices, consumer, "traverse-nodes"); - } - protected long traversePairs(Iterator> pairs, Consumer> consumer) { return this.traverse(pairs, consumer, "traverse-pairs"); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java index a6bc4b7a73..bae50817fe 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java @@ -178,7 +178,7 @@ private PathSet linkPath(Stack all, int id, int layerIndex) { iter.remove(); continue; } - path.addToEnd(sid); + path.addToLast(sid); } results.addAll(paths); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java index 5302719b09..24762006b6 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java @@ -24,34 +24,29 @@ import java.util.Set; import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.traversal.algorithm.HugeTraverser; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; import com.baidu.hugegraph.traversal.algorithm.records.record.IntIterator; import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; -import com.baidu.hugegraph.type.define.CollectionImplType; +import com.baidu.hugegraph.type.define.CollectionType; import com.baidu.hugegraph.util.collection.CollectionFactory; -import com.google.common.collect.ImmutableList; public class KneighborRecords extends SingleWayMultiPathsRecords { - private final Id source; - public KneighborRecords(RecordType type, boolean concurrent, Id source, boolean nearest) { super(type, concurrent, source, nearest); - this.source = source; } + @Override public int size() { return (int) this.accessed(); } public Set ids(long limit) { - Set ids = CollectionFactory.newIdSet(CollectionImplType.EC); - ids.add(this.source); + Set ids = CollectionFactory.newIdSet(CollectionType.EC); for (int i = 1; i < this.records.size(); i++) { IntIterator iterator = this.records.get(i).keys(); - while ((limit == NO_LIMIT || limit > 1L) && iterator.hasNext()) { + while ((limit == NO_LIMIT || limit > 0L) && iterator.hasNext()) { ids.add(this.id(iterator.next())); limit--; } @@ -61,10 +56,9 @@ public Set ids(long limit) { public PathSet paths(long limit) { PathSet paths = new PathSet(); - paths.add(new HugeTraverser.Path(ImmutableList.of(this.source))); for (int i = 1; i < this.records.size(); i++) { IntIterator iterator = this.records.get(i).keys(); - while ((limit == NO_LIMIT || limit > 1L) && iterator.hasNext()) { + while ((limit == NO_LIMIT || limit > 0L) && iterator.hasNext()) { paths.add(this.getPath(i, iterator.next())); limit--; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java index 41d6a44086..1302e9612a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java @@ -27,7 +27,7 @@ import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; import com.baidu.hugegraph.traversal.algorithm.records.record.IntIterator; import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; -import com.baidu.hugegraph.type.define.CollectionImplType; +import com.baidu.hugegraph.type.define.CollectionType; import com.baidu.hugegraph.util.collection.CollectionFactory; public class KoutRecords extends SingleWayMultiPathsRecords { @@ -44,7 +44,7 @@ public int size() { public Set ids(long limit) { IntIterator iterator = this.records.peek().keys(); - Set ids = CollectionFactory.newIdSet(CollectionImplType.EC); + Set ids = CollectionFactory.newIdSet(CollectionType.EC); while ((limit == NO_LIMIT || limit-- > 0L) && iterator.hasNext()) { ids.add(this.id(iterator.next())); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java index e9185d874f..2b53d9ed0c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java @@ -41,7 +41,7 @@ import com.baidu.hugegraph.traversal.algorithm.records.record.Record; import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; -public class SingleWayMultiPathsRecords extends AbstractRecords { +public abstract class SingleWayMultiPathsRecords extends AbstractRecords { protected final Stack records; @@ -127,9 +127,7 @@ public void addPath(Id source, Id target) { this.accessedVertices.add(targetCode); } - public int size() { - return 0; - } + public abstract int size(); public Path getPath(int target) { List ids = new ArrayList<>(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionType.java similarity index 88% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionType.java index db2bcfc27e..0d02fbb517 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionImplType.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/CollectionType.java @@ -19,7 +19,7 @@ package com.baidu.hugegraph.type.define; -public enum CollectionImplType implements SerialEnum { +public enum CollectionType implements SerialEnum { // Java Collection Framework JCF(1, "jcf"), @@ -34,10 +34,10 @@ public enum CollectionImplType implements SerialEnum { private final String name; static { - SerialEnum.register(CollectionImplType.class); + SerialEnum.register(CollectionType.class); } - CollectionImplType(int code, String name) { + CollectionType(int code, String name) { assert code < 256; this.code = (byte) code; this.name = name; @@ -51,7 +51,7 @@ public String string() { return this.name; } - public static CollectionImplType fromCode(byte code) { + public static CollectionType fromCode(byte code) { switch (code) { case 1: return JCF; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java index 54b928af69..f3aaaf1a62 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java @@ -35,7 +35,7 @@ import org.eclipse.collections.impl.set.mutable.UnifiedSet; import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.type.define.CollectionImplType; +import com.baidu.hugegraph.type.define.CollectionType; import com.baidu.hugegraph.util.E; import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; @@ -44,13 +44,13 @@ public class CollectionFactory { - private CollectionImplType type; + private CollectionType type; public CollectionFactory() { - this.type = CollectionImplType.EC; + this.type = CollectionType.EC; } - public CollectionFactory(CollectionImplType type) { + public CollectionFactory(CollectionType type) { this.type = type; } @@ -66,7 +66,7 @@ public List newList(Collection collection) { return newList(this.type, collection); } - public static List newList(CollectionImplType type) { + public static List newList(CollectionType type) { switch (type) { case EC: return new FastList<>(); @@ -80,7 +80,7 @@ public static List newList(CollectionImplType type) { } } - public static List newList(CollectionImplType type, + public static List newList(CollectionType type, int initialCapacity) { switch (type) { case EC: @@ -95,7 +95,7 @@ public static List newList(CollectionImplType type, } } - public static List newList(CollectionImplType type, + public static List newList(CollectionType type, Collection collection) { switch (type) { case EC: @@ -122,7 +122,7 @@ public Set newSet(Collection collection) { return newSet(this.type, collection); } - public static Set newSet(CollectionImplType type) { + public static Set newSet(CollectionType type) { switch (type) { case EC: return new UnifiedSet<>(); @@ -136,7 +136,7 @@ public static Set newSet(CollectionImplType type) { } } - public static Set newSet(CollectionImplType type, + public static Set newSet(CollectionType type, int initialCapacity) { switch (type) { case EC: @@ -151,7 +151,7 @@ public static Set newSet(CollectionImplType type, } } - public static Set newSet(CollectionImplType type, + public static Set newSet(CollectionType type, Collection collection) { switch (type) { case EC: @@ -174,7 +174,11 @@ public Map newMap(int initialCapacity) { return newMap(this.type, initialCapacity); } - public static Map newMap(CollectionImplType type) { + public static Map newMap(CollectionType type) { + /* + * EC is faster 10%-20% than JCF, and it's more stable & less + * memory cost(size is bigger, EC is better). + */ switch (type) { case EC: return new UnifiedMap<>(); @@ -188,7 +192,7 @@ public static Map newMap(CollectionImplType type) { } } - public static Map newMap(CollectionImplType type, + public static Map newMap(CollectionType type, int initialCapacity) { switch (type) { case EC: @@ -256,7 +260,7 @@ public Set newIdSet() { return newIdSet(this.type); } - public static Set newIdSet(CollectionImplType type) { + public static Set newIdSet(CollectionType type) { return new IdSet(type); } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java index d1c38f429a..ea1860691c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java @@ -31,7 +31,7 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.id.IdGenerator; import com.baidu.hugegraph.iterator.ExtendableIterator; -import com.baidu.hugegraph.type.define.CollectionImplType; +import com.baidu.hugegraph.type.define.CollectionType; import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; @@ -40,7 +40,7 @@ public class IdSet extends AbstractSet { private LongHashSet numberIds; private Set nonNumberIds; - public IdSet(CollectionImplType type) { + public IdSet(CollectionType type) { this.numberIds = new LongHashSet(); switch (type) { case JCF: From a4d32bf86654079882ce1bc21726a46e1b222392 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Fri, 21 May 2021 22:06:38 +0800 Subject: [PATCH 32/38] improve Change-Id: I5ca0ed20f0bba31226aeb10e22d191bc970e6805 --- .../api/traversers/KneighborAPI.java | 3 +- .../baidu/hugegraph/StandardHugeGraph.java | 3 +- .../baidu/hugegraph/config/CoreOptions.java | 4 +- .../baidu/hugegraph/structure/HugeVertex.java | 2 +- .../traversal/algorithm/HugeTraverser.java | 6 +-- .../algorithm/records/record/SyncRecord.java | 5 +++ .../util/collection/CollectionFactory.java | 40 +++++++++++++++++++ .../util/collection/Int2IntsMap.java | 24 +++++++---- .../hugegraph/unit/util/JsonUtilTest.java | 17 ++++---- 9 files changed, 76 insertions(+), 28 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java index ae382fe001..4ba7819b80 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java @@ -51,7 +51,6 @@ import com.baidu.hugegraph.traversal.algorithm.records.KneighborRecords; import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser; -import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; import com.baidu.hugegraph.traversal.algorithm.KneighborTraverser; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; @@ -134,7 +133,7 @@ public String post(@Context GraphManager manager, Set neighbors = results.ids(request.limit); - PathSet paths = new PathSet(); + HugeTraverser.PathSet paths = new HugeTraverser.PathSet(); if (request.withPath) { paths.addAll(results.paths(request.limit)); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java index a45aeb7664..08a2f998df 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java @@ -64,7 +64,6 @@ import com.baidu.hugegraph.backend.store.ram.RamTable; import com.baidu.hugegraph.backend.tx.GraphTransaction; import com.baidu.hugegraph.backend.tx.SchemaTransaction; -import com.baidu.hugegraph.config.ConfigOption; import com.baidu.hugegraph.config.CoreOptions; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.config.TypedOption; @@ -125,7 +124,7 @@ public class StandardHugeGraph implements HugeGraph { CoreOptions.TASK_RESULT_SIZE_LIMIT, CoreOptions.OLTP_CONCURRENT_THREADS, CoreOptions.OLTP_CONCURRENT_DEPTH, - CoreOptions.OLTP_COLLECTION_IMPL_TYPE, + CoreOptions.OLTP_COLLECTION_TYPE, CoreOptions.VERTEX_DEFAULT_LABEL, CoreOptions.VERTEX_ENCODE_PK_NUMBER ); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java index 4c90d4db0d..0da897f285 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java @@ -625,9 +625,9 @@ public static synchronized CoreOptions instance() { 10 ); - public static final ConfigConvOption OLTP_COLLECTION_IMPL_TYPE = + public static final ConfigConvOption OLTP_COLLECTION_TYPE = new ConfigConvOption<>( - "oltp.collection_impl_type", + "oltp.collection_type", "The implementation type of collections " + "used in oltp algorithm.", allowValues("jcf", "ec", "fu"), diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java index 63ae52b5e3..ac26be24a3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java @@ -236,7 +236,7 @@ public void removeEdge(HugeEdge edge) { public void addEdge(HugeEdge edge) { if (this.edges == EMPTY_LIST) { - this.edges = CollectionFactory.newList(CollectionType.EC); + this.edges = newList(); } this.edges.add(edge); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java index 1b8f82be88..ada4854a4c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/HugeTraverser.java @@ -91,7 +91,7 @@ public class HugeTraverser { public HugeTraverser(HugeGraph graph) { this.graph = graph; if (collectionFactory == null) { - collectionFactory = new CollectionFactory(this.collectionImplType()); + collectionFactory = new CollectionFactory(this.collectionType()); } } @@ -103,8 +103,8 @@ protected int concurrentDepth() { return this.graph.option(CoreOptions.OLTP_CONCURRENT_DEPTH); } - private CollectionType collectionImplType() { - return this.graph.option(CoreOptions.OLTP_COLLECTION_IMPL_TYPE); + private CollectionType collectionType() { + return this.graph.option(CoreOptions.OLTP_COLLECTION_TYPE); } protected Set adjacentVertices(Id sourceV, Set vertices, diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/SyncRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/SyncRecord.java index b6c721e5fd..4882fbddaa 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/SyncRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/SyncRecord.java @@ -40,6 +40,11 @@ public SyncRecord(Record record, Object newLock) { @Override public IntIterator keys() { + /* + * Another threads call addPath() will change IntIterator inner array, + * but in kout/kneighbor scenario it's ok because keys() and addPath() + * won't be called simultaneously on same Record. + */ synchronized (this.lock) { return this.record.keys(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java index f3aaaf1a62..1579acf095 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java @@ -256,6 +256,46 @@ public static MutableIntObjectMap newIntObjectMap( return map; } + public static MutableIntObjectMap newIntObjectMap(Id key1, + V value1) { + return IntObjectHashMap.newWithKeysValues((int) key1.asLong(), value1); + } + + public static MutableIntObjectMap newIntObjectMap( + Id key1, V value1, + Id key2, V value2) { + return IntObjectHashMap.newWithKeysValues((int) key1.asLong(), value1, + (int) key2.asLong(), value2); + } + + public static MutableIntObjectMap newIntObjectMap( + Id key1, V value1, + Id key2, V value2, + Id key3, V value3) { + return IntObjectHashMap.newWithKeysValues((int) key1.asLong(), value1, + (int) key2.asLong(), value2, + (int) key3.asLong(), value3); + } + + @SuppressWarnings("unchecked") + public static MutableIntObjectMap newIntObjectMap( + Id key1, V value1, + Id key2, V value2, + Id key3, V value3, + Object... objects) { + IntObjectHashMap map = IntObjectHashMap.newWithKeysValues( + (int) key1.asLong(), value1, + (int) key2.asLong(), value2, + (int) key3.asLong(), value3); + E.checkArgument(objects.length % 2 == 0, + "Must provide even arguments for " + + "CollectionFactory.newIntObjectMap"); + for (int i = 0; i < objects.length; i+=2) { + map.put((int) ((Id) objects[i]).asLong(), (V) objects[i + 1]); + } + return map; + } + public Set newIdSet() { return newIdSet(this.type); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/Int2IntsMap.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/Int2IntsMap.java index ba0493ee01..fc0eeafb74 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/Int2IntsMap.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/Int2IntsMap.java @@ -101,12 +101,19 @@ public void add(int key, int value) { chunkTable[nextPosition] = value; this.chunkTable[firstChunk + OFFSET_POSITION]++; } else { + /* + * nextPosition points to last entry of current chunk, should + * acquire a new chunk and make last entry of current chunk + * points new chunk + */ this.ensureCapacity(); - this.chunkTable[nextPosition] = this.nextChunk; + int lastEntryOfChunk = nextPosition; + this.chunkTable[lastEntryOfChunk] = this.nextChunk; - this.chunkTable[this.nextChunk] = value; - this.chunkTable[firstChunk + OFFSET_POSITION] = this.nextChunk + 1; + nextPosition = this.nextChunk; + this.chunkTable[nextPosition] = value; + this.chunkTable[firstChunk + OFFSET_POSITION] = nextPosition + 1; // Update next block this.nextChunk += CHUNK_SIZE; @@ -120,10 +127,11 @@ public void add(int key, int value) { this.chunkMap.put(key, this.nextChunk); // Init first chunk - this.chunkTable[this.nextChunk] = this.nextChunk + - OFFSET_DATA_IN_FIRST_CHUNK + 1; - this.chunkTable[this.nextChunk + OFFSET_SIZE] = 1; - this.chunkTable[this.nextChunk + OFFSET_DATA_IN_FIRST_CHUNK] = value; + int firstChunk = this.nextChunk; + int nextPosition = firstChunk + OFFSET_DATA_IN_FIRST_CHUNK; + this.chunkTable[firstChunk + OFFSET_POSITION] = nextPosition + 1; + this.chunkTable[firstChunk + OFFSET_SIZE] = 1; + this.chunkTable[nextPosition] = value; // Update next block this.nextChunk += CHUNK_SIZE; @@ -138,8 +146,8 @@ public int[] get(int key) { int firstChunk = this.chunkMap.get(key); int size = this.chunkTable[firstChunk + OFFSET_SIZE]; int[] values = new int[size]; - int i = 0; int position = firstChunk + OFFSET_DATA_IN_FIRST_CHUNK; + int i = 0; while (i < size) { if (!this.endOfChunk(position)) { values[i++] = this.chunkTable[position++]; diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/JsonUtilTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/JsonUtilTest.java index 3214aa7909..b3ccc2066b 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/JsonUtilTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/JsonUtilTest.java @@ -20,6 +20,7 @@ package com.baidu.hugegraph.unit.util; import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.UUID; @@ -236,12 +237,9 @@ public void testSerializeVertexWithNumberId() { MutableIntObjectMap> properties = CollectionFactory.newIntObjectMap( - (int) name.id().asLong(), - new HugeVertexProperty<>(vertex, name, "marko"), - (int) age.id().asLong(), - new HugeVertexProperty<>(vertex, age, 29), - (int) city.id().asLong(), - new HugeVertexProperty<>(vertex, city, "Beijing") + name.id(), new HugeVertexProperty<>(vertex, name, "marko"), + age.id(), new HugeVertexProperty<>(vertex, age, 29), + city.id(), new HugeVertexProperty<>(vertex, city, "Beijing") ); Whitebox.setInternalState(vertex, "properties", properties); @@ -286,12 +284,11 @@ public void testSerializeEdge() { Whitebox.setInternalState(edge, "sourceVertex", source); Whitebox.setInternalState(edge, "targetVertex", target); + Date dateValue = Utils.date("2019-03-12"); MutableIntObjectMap> properties = CollectionFactory.newIntObjectMap( - (int) date.id().asLong(), - new HugeEdgeProperty<>(edge, date, Utils.date("2019-03-12")), - (int) weight.id().asLong(), - new HugeEdgeProperty<>(edge, weight, 0.8) + date.id(), new HugeEdgeProperty<>(edge, date, dateValue), + weight.id(), new HugeEdgeProperty<>(edge, weight, 0.8) ); Whitebox.setInternalState(edge, "properties", properties); From 5073a0b0de171fd92de49eb1819113360c3cdc15 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Mon, 24 May 2021 19:21:42 +0800 Subject: [PATCH 33/38] improve Change-Id: I40b3e2c5b3d66206045ce6842e2a85ad2907bcdd --- .../src/main/java/com/baidu/hugegraph/config/CoreOptions.java | 4 ++-- .../hugegraph/traversal/algorithm/records/KoutRecords.java | 1 + .../algorithm/records/SingleWayMultiPathsRecords.java | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java index 0da897f285..ddff4e6019 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java @@ -630,8 +630,8 @@ public static synchronized CoreOptions instance() { "oltp.collection_type", "The implementation type of collections " + "used in oltp algorithm.", - allowValues("jcf", "ec", "fu"), + allowValues("JCF", "EC", "FU"), CollectionType::valueOf, - "ec" + "EC" ); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java index 1302e9612a..82f25db5a8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java @@ -35,6 +35,7 @@ public class KoutRecords extends SingleWayMultiPathsRecords { public KoutRecords(RecordType type, boolean concurrent, Id source, boolean nearest) { super(type, concurrent, source, nearest); + this.accessedVertices.add(this.code(source)); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java index 2b53d9ed0c..fb1309ee23 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java @@ -46,7 +46,7 @@ public abstract class SingleWayMultiPathsRecords extends AbstractRecords { protected final Stack records; private final boolean nearest; - private final MutableIntSet accessedVertices; + protected final MutableIntSet accessedVertices; protected Record currentRecord; protected IntIterator lastRecordKeys; @@ -67,7 +67,6 @@ public SingleWayMultiPathsRecords(RecordType type, boolean concurrent, this.accessedVertices = concurrent ? new IntHashSet().asSynchronized() : new IntHashSet(); - this.accessedVertices.add(sourceCode); } @Override From 0a42055954fad319527c010feaa82871f9310f49 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 25 May 2021 12:06:05 +0800 Subject: [PATCH 34/38] improve Change-Id: Ieb9a8bb4a0c2e7c6f2f18f8745783c93e51754cf --- .../baidu/hugegraph/structure/HugeVertex.java | 5 +- .../traversal/algorithm/KoutTraverser.java | 24 ++- .../traversal/algorithm/PathsTraverser.java | 14 +- .../algorithm/records/AbstractRecords.java | 6 +- .../records/DoubleWayMultiPathsRecords.java | 7 +- .../algorithm/records/KoutRecords.java | 1 - .../algorithm/records/PathsRecords.java | 30 ++++ .../records/ShortestPathRecords.java | 4 +- .../records/SingleWayMultiPathsRecords.java | 20 ++- ...tArrayRecord.java => Int2ArrayRecord.java} | 6 +- .../{IntIntRecord.java => Int2IntRecord.java} | 4 +- .../{IntSetRecord.java => Int2SetRecord.java} | 4 +- .../records/record/RecordFactory.java | 6 +- .../util/collection/CollectionFactory.java | 12 +- .../ConcurrentObjectIntMapping.java | 14 +- .../hugegraph/util/collection/IdSet.java | 51 ++---- .../util/collection/Int2IntsMap.java | 87 ++++----- .../{mapping => }/MappingFactory.java | 4 +- .../{mapping => }/ObjectIntMapping.java | 6 +- ...java => SingleThreadObjectIntMapping.java} | 10 +- .../baidu/hugegraph/unit/UnitTestSuite.java | 2 + .../baidu/hugegraph/unit/core/IdSetTest.java | 170 ++++++++++++++++++ .../hugegraph/unit/core/Int2IntsMapTest.java | 12 +- .../unit/core/ObjectIntMappingTest.java | 19 +- 24 files changed, 349 insertions(+), 169 deletions(-) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecords.java rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/{IntArrayRecord.java => Int2ArrayRecord.java} (91%) rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/{IntIntRecord.java => Int2IntRecord.java} (95%) rename hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/{IntSetRecord.java => Int2SetRecord.java} (96%) rename hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/{mapping => }/ConcurrentObjectIntMapping.java (75%) rename hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/{mapping => }/MappingFactory.java (91%) rename hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/{mapping => }/ObjectIntMapping.java (91%) rename hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/{mapping/SingleObjectIntMapping.java => SingleThreadObjectIntMapping.java} (87%) create mode 100644 hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/IdSetTest.java diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java index ac26be24a3..98e24143d3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java @@ -224,8 +224,7 @@ public Collection getEdges() { public void resetEdges() { /* - * Use list to hold edges for vertices to reduce memory usage and - * add operation time. + * Use List to hold edges to reduce memory usage and operation time. */ this.edges = newList(); } @@ -662,7 +661,7 @@ public HugeVertex4Insert(final GraphTransaction tx, Id id, VertexLabel label) { super(tx.graph(), id, label); /* - * Use set to hold edges for inserted vertex + * Use Set to hold edges inserted into vertex * to avoid duplicated edges */ this.edges = newSet(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java index 812d85924e..b434a102ad 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/KoutTraverser.java @@ -37,8 +37,6 @@ public class KoutTraverser extends OltpTraverser { - private int depth; - public KoutTraverser(HugeGraph graph) { super(graph); } @@ -53,7 +51,6 @@ public Set kout(Id sourceV, Directions dir, String label, checkDegree(degree); checkCapacity(capacity); checkLimit(limit); - this.depth = depth; if (capacity != NO_LIMIT) { // Capacity must > limit because sourceV is counted in capacity E.checkArgument(capacity >= limit && limit != NO_LIMIT, @@ -72,9 +69,9 @@ public Set kout(Id sourceV, Directions dir, String label, long remaining = capacity == NO_LIMIT ? NO_LIMIT : capacity - latest.size(); - while (this.depth-- > 0) { + while (depth-- > 0) { // Just get limit nodes in last layer if limit < remaining capacity - if (this.depth == 0 && limit != NO_LIMIT && + if (depth == 0 && limit != NO_LIMIT && (limit < remaining || remaining == NO_LIMIT)) { remaining = limit; } @@ -90,10 +87,10 @@ public Set kout(Id sourceV, Directions dir, String label, // Update 'remaining' value to record remaining capacity remaining -= latest.size(); - if (remaining <= 0 && this.depth > 0) { + if (remaining <= 0 && depth > 0) { throw new HugeException( "Reach capacity '%s' while remaining depth '%s'", - capacity, this.depth); + capacity, depth); } } } @@ -109,26 +106,27 @@ public KoutRecords customizedKout(Id source, EdgeStep step, checkPositive(maxDepth, "k-out max_depth"); checkCapacity(capacity); checkLimit(limit); - this.depth = maxDepth; + long[] depth = new long[1]; + depth[0] = maxDepth; boolean concurrent = maxDepth >= this.concurrentDepth() && step.direction() == Directions.BOTH; KoutRecords records = new KoutRecords(RecordType.INT, concurrent, source, nearest); Consumer consumer = v -> { - if (this.reachLimit(limit, this.depth, records.size())) { + if (this.reachLimit(limit, depth[0], records.size())) { return; } Iterator edges = edgesOfVertex(v, step); - while (!this.reachLimit(limit, this.depth, records.size()) && + while (!this.reachLimit(limit, depth[0], records.size()) && edges.hasNext()) { Id target = ((HugeEdge) edges.next()).id().otherVertexId(); records.addPath(v, target); - this.checkCapacity(capacity, records.accessed(), this.depth); + this.checkCapacity(capacity, records.accessed(), depth[0]); } }; - while (this.depth-- > 0) { + while (depth[0]-- > 0) { records.startOneLayer(true); traverseIds(records.keys(), consumer, concurrent); records.finishOneLayer(); @@ -136,7 +134,7 @@ public KoutRecords customizedKout(Id source, EdgeStep step, return records; } - private void checkCapacity(long capacity, long accessed, int depth) { + private void checkCapacity(long capacity, long accessed, long depth) { if (capacity == NO_LIMIT) { return; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java index aa07ba2d5c..15f33fcc35 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/PathsTraverser.java @@ -27,8 +27,7 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.structure.HugeEdge; -import com.baidu.hugegraph.traversal.algorithm.records.DoubleWayMultiPathsRecords; -import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; +import com.baidu.hugegraph.traversal.algorithm.records.PathsRecords; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; @@ -82,7 +81,7 @@ public PathSet paths(Id sourceV, Directions sourceDir, private class Traverser { - private final DoubleWayMultiPathsRecords record; + private final PathsRecords record; private final Id label; private final long degree; @@ -93,9 +92,7 @@ private class Traverser { public Traverser(Id sourceV, Id targetV, Id label, long degree, long capacity, long limit) { - this.record = new DoubleWayMultiPathsRecords(RecordType.ARRAY, - false, - sourceV, targetV); + this.record = new PathsRecords(false, sourceV, targetV); this.label = label; this.degree = degree; this.capacity = capacity; @@ -170,10 +167,7 @@ public PathSet paths() { private boolean reachLimit() { checkCapacity(this.capacity, this.record.accessed(), "paths"); - if (this.limit == NO_LIMIT || this.paths.size() < this.limit) { - return false; - } - return true; + return this.limit != NO_LIMIT && this.paths.size() >= this.limit; } } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java index 9a5431b09a..dd45208d66 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java @@ -24,8 +24,8 @@ import com.baidu.hugegraph.traversal.algorithm.records.record.Record; import com.baidu.hugegraph.traversal.algorithm.records.record.RecordFactory; import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; -import com.baidu.hugegraph.util.collection.mapping.ObjectIntMapping; -import com.baidu.hugegraph.util.collection.mapping.MappingFactory; +import com.baidu.hugegraph.util.collection.ObjectIntMapping; +import com.baidu.hugegraph.util.collection.MappingFactory; public abstract class AbstractRecords implements Records { @@ -46,7 +46,7 @@ protected int code(Id id) { @Watched protected Id id(int code) { - return (Id) this.idMapping.code2Object(code); + return this.idMapping.code2Object(code); } protected Record newRecord() { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java index bae50817fe..0757430cb3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java @@ -36,7 +36,7 @@ import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; import com.google.common.collect.Lists; -public class DoubleWayMultiPathsRecords extends AbstractRecords { +public abstract class DoubleWayMultiPathsRecords extends AbstractRecords { protected final Stack sourceRecords; protected final Stack targetRecords; @@ -64,6 +64,7 @@ public DoubleWayMultiPathsRecords(RecordType type, boolean concurrent, this.accessed = 2; } + @Override public void startOneLayer(boolean forward) { this.forward = forward; this.currentRecord = this.newRecord(); @@ -71,6 +72,7 @@ public void startOneLayer(boolean forward) { this.targetRecords.peek().keys(); } + @Override public void finishOneLayer() { if (this.forward) { this.sourceRecords.push(this.currentRecord); @@ -81,11 +83,13 @@ public void finishOneLayer() { } @Watched + @Override public boolean hasNextKey() { return this.lastRecordKeys.hasNext(); } @Watched + @Override public Id nextKey() { this.current = this.lastRecordKeys.next(); return this.id(current); @@ -107,6 +111,7 @@ public PathSet findPath(Id target, Function filter, return results; } + @Override public long accessed() { return this.accessed; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java index 82f25db5a8..1302e9612a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java @@ -35,7 +35,6 @@ public class KoutRecords extends SingleWayMultiPathsRecords { public KoutRecords(RecordType type, boolean concurrent, Id source, boolean nearest) { super(type, concurrent, source, nearest); - this.accessedVertices.add(this.code(source)); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecords.java new file mode 100644 index 0000000000..fd3cb7044b --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/PathsRecords.java @@ -0,0 +1,30 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.traversal.algorithm.records; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; + +public class PathsRecords extends DoubleWayMultiPathsRecords { + + public PathsRecords(boolean concurrent, Id sourceV, Id targetV) { + super(RecordType.ARRAY, concurrent, sourceV, targetV); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java index 336b76a3bf..23d3513f16 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java @@ -30,7 +30,7 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; -import com.baidu.hugegraph.traversal.algorithm.records.record.IntIntRecord; +import com.baidu.hugegraph.traversal.algorithm.records.record.Int2IntRecord; import com.baidu.hugegraph.traversal.algorithm.records.record.Record; import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; @@ -106,7 +106,7 @@ private Path linkPath(Stack all, int node) { ids.add(this.id(node)); int value = node; for (int i = size - 1; i > 0 ; i--) { - IntIntHashMap layer = ((IntIntRecord) all.elementAt(i)).layer(); + IntIntHashMap layer = ((Int2IntRecord) all.elementAt(i)).layer(); value = layer.get(value); ids.add(this.id(value)); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java index fb1309ee23..48b56eaa3e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java @@ -36,7 +36,7 @@ import com.baidu.hugegraph.perf.PerfUtil.Watched; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Path; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; -import com.baidu.hugegraph.traversal.algorithm.records.record.IntIntRecord; +import com.baidu.hugegraph.traversal.algorithm.records.record.Int2IntRecord; import com.baidu.hugegraph.traversal.algorithm.records.record.IntIterator; import com.baidu.hugegraph.traversal.algorithm.records.record.Record; import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; @@ -45,8 +45,9 @@ public abstract class SingleWayMultiPathsRecords extends AbstractRecords { protected final Stack records; + private final int sourceCode; private final boolean nearest; - protected final MutableIntSet accessedVertices; + private final MutableIntSet accessedVertices; protected Record currentRecord; protected IntIterator lastRecordKeys; @@ -59,9 +60,9 @@ public SingleWayMultiPathsRecords(RecordType type, boolean concurrent, this.nearest = nearest; - int sourceCode = this.code(source); + this.sourceCode = this.code(source); Record firstRecord = this.newRecord(); - firstRecord.addPath(sourceCode, 0); + firstRecord.addPath(this.sourceCode, 0); this.records = new Stack<>(); this.records.push(firstRecord); @@ -118,7 +119,8 @@ public void addPath(Id source, Id target) { int sourceCode = this.code(source); int targetCode = this.code(target); if (this.nearest && this.accessedVertices.contains(targetCode) || - !this.nearest && this.currentRecord.containsKey(targetCode)) { + !this.nearest && this.currentRecord.containsKey(targetCode) || + targetCode == this.sourceCode) { return; } this.currentRecord.addPath(targetCode, sourceCode); @@ -131,7 +133,7 @@ public void addPath(Id source, Id target) { public Path getPath(int target) { List ids = new ArrayList<>(); for (int i = 0; i < this.records.size(); i++) { - IntIntHashMap layer = ((IntIntRecord) this.records + IntIntHashMap layer = ((Int2IntRecord) this.records .elementAt(i)).layer(); if (!layer.containsKey(target)) { continue; @@ -142,7 +144,7 @@ public Path getPath(int target) { ids.add(this.id(parent)); i--; for (; i > 0; i--) { - layer = ((IntIntRecord) this.records.elementAt(i)).layer(); + layer = ((Int2IntRecord) this.records.elementAt(i)).layer(); parent = layer.get(parent); ids.add(this.id(parent)); } @@ -153,7 +155,7 @@ public Path getPath(int target) { public Path getPath(int layerIndex, int target) { List ids = new ArrayList<>(); - IntIntHashMap layer = ((IntIntRecord) this.records + IntIntHashMap layer = ((Int2IntRecord) this.records .elementAt(layerIndex)).layer(); if (!layer.containsKey(target)) { throw new HugeException("Failed to get path for %s", @@ -164,7 +166,7 @@ public Path getPath(int layerIndex, int target) { ids.add(this.id(parent)); layerIndex--; for (; layerIndex > 0; layerIndex--) { - layer = ((IntIntRecord) this.records.elementAt(layerIndex)).layer(); + layer = ((Int2IntRecord) this.records.elementAt(layerIndex)).layer(); parent = layer.get(parent); ids.add(this.id(parent)); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntArrayRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/Int2ArrayRecord.java similarity index 91% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntArrayRecord.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/Int2ArrayRecord.java index 7b10904feb..3ec20f9cdd 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntArrayRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/Int2ArrayRecord.java @@ -21,11 +21,11 @@ import com.baidu.hugegraph.util.collection.Int2IntsMap; -public class IntArrayRecord implements Record { +public class Int2ArrayRecord implements Record { private final Int2IntsMap layer; - public IntArrayRecord() { + public Int2ArrayRecord() { this.layer = new Int2IntsMap(); } @@ -41,7 +41,7 @@ public boolean containsKey(int key) { @Override public IntIterator get(int key) { - return new IntIterator(this.layer.get(key)); + return new IntIterator(this.layer.getValues(key)); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntIntRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/Int2IntRecord.java similarity index 95% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntIntRecord.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/Int2IntRecord.java index 2cf06596ec..0b8d56b435 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntIntRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/Int2IntRecord.java @@ -21,11 +21,11 @@ import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; -public class IntIntRecord implements Record { +public class Int2IntRecord implements Record { private final IntIntHashMap layer; - public IntIntRecord() { + public Int2IntRecord() { this.layer = new IntIntHashMap(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntSetRecord.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/Int2SetRecord.java similarity index 96% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntSetRecord.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/Int2SetRecord.java index 9abb8d3964..662e65aaec 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/IntSetRecord.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/Int2SetRecord.java @@ -22,11 +22,11 @@ import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; -public class IntSetRecord implements Record { +public class Int2SetRecord implements Record { private final IntObjectHashMap layer; - public IntSetRecord() { + public Int2SetRecord() { this.layer = new IntObjectHashMap<>(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordFactory.java index 4f3320614e..e7141cbf3a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/record/RecordFactory.java @@ -29,13 +29,13 @@ public static Record newRecord(RecordType type, boolean concurrent) { Record record; switch (type) { case ARRAY: - record = new IntArrayRecord(); + record = new Int2ArrayRecord(); break; case SET: - record = new IntSetRecord(); + record = new Int2SetRecord(); break; case INT: - record = new IntIntRecord(); + record = new Int2IntRecord(); break; default: throw new AssertionError("Unsupported record type: " + type); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java index 1579acf095..b3be5be893 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java @@ -44,7 +44,7 @@ public class CollectionFactory { - private CollectionType type; + private final CollectionType type; public CollectionFactory() { this.type = CollectionType.EC; @@ -250,7 +250,7 @@ public static MutableIntObjectMap newIntObjectMap( E.checkArgument(objects.length % 2 == 0, "Must provide even arguments for " + "CollectionFactory.newIntObjectMap"); - for (int i = 0; i < objects.length; i+=2) { + for (int i = 0; i < objects.length; i += 2) { map.put((int) objects[i], (V) objects[i + 1]); } return map; @@ -279,14 +279,8 @@ public static MutableIntObjectMap newIntObjectMap( @SuppressWarnings("unchecked") public static MutableIntObjectMap newIntObjectMap( - Id key1, V value1, - Id key2, V value2, - Id key3, V value3, Object... objects) { - IntObjectHashMap map = IntObjectHashMap.newWithKeysValues( - (int) key1.asLong(), value1, - (int) key2.asLong(), value2, - (int) key3.asLong(), value3); + IntObjectHashMap map = IntObjectHashMap.newMap(); E.checkArgument(objects.length % 2 == 0, "Must provide even arguments for " + "CollectionFactory.newIntObjectMap"); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ConcurrentObjectIntMapping.java similarity index 75% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ConcurrentObjectIntMapping.java index 301fe5ffd7..5d80678405 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ConcurrentObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ConcurrentObjectIntMapping.java @@ -17,31 +17,31 @@ * under the License. */ -package com.baidu.hugegraph.util.collection.mapping; +package com.baidu.hugegraph.util.collection; import com.baidu.hugegraph.perf.PerfUtil.Watched; public class ConcurrentObjectIntMapping implements ObjectIntMapping { - private SingleObjectIntMapping singleObjectIntMapping; + private SingleThreadObjectIntMapping objectIntMapping; public ConcurrentObjectIntMapping() { - this.singleObjectIntMapping = new SingleObjectIntMapping<>(); + this.objectIntMapping = new SingleThreadObjectIntMapping<>(); } @Watched @SuppressWarnings("unchecked") public synchronized int object2Code(Object object) { - return this.singleObjectIntMapping.object2Code(object); + return this.objectIntMapping.object2Code(object); } @Watched - public synchronized Object code2Object(int code) { - return this.singleObjectIntMapping.code2Object(code); + public synchronized V code2Object(int code) { + return this.objectIntMapping.code2Object(code); } @Override public synchronized void clear() { - this.singleObjectIntMapping.clear(); + this.objectIntMapping.clear(); } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java index ea1860691c..a4937ec55b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/IdSet.java @@ -20,12 +20,10 @@ package com.baidu.hugegraph.util.collection; import java.util.AbstractSet; -import java.util.HashSet; import java.util.Iterator; import java.util.Set; import org.eclipse.collections.api.iterator.MutableLongIterator; -import org.eclipse.collections.impl.set.mutable.UnifiedSet; import org.eclipse.collections.impl.set.mutable.primitive.LongHashSet; import com.baidu.hugegraph.backend.id.Id; @@ -33,29 +31,14 @@ import com.baidu.hugegraph.iterator.ExtendableIterator; import com.baidu.hugegraph.type.define.CollectionType; -import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; - public class IdSet extends AbstractSet { - private LongHashSet numberIds; - private Set nonNumberIds; + private final LongHashSet numberIds; + private final Set nonNumberIds; public IdSet(CollectionType type) { this.numberIds = new LongHashSet(); - switch (type) { - case JCF: - this.nonNumberIds = new HashSet<>(); - break; - case EC: - this.nonNumberIds = new UnifiedSet<>(); - break; - case FU: - this.nonNumberIds = new ObjectOpenHashSet<>(); - break; - default: - throw new AssertionError( - "Unsupported collection type: " + type); - } + this.nonNumberIds = CollectionFactory.newSet(type); } @Override @@ -68,12 +51,11 @@ public boolean isEmpty() { return this.numberIds.isEmpty() && this.nonNumberIds.isEmpty(); } - @Override - public boolean contains(Object o) { - if (o instanceof IdGenerator.LongId) { - return this.numberIds.contains(((IdGenerator.LongId) o).longValue()); + public boolean contains(Id id) { + if (id.type() == Id.IdType.LONG) { + return this.numberIds.contains(id.asLong()); } else { - return this.nonNumberIds.contains(o); + return this.nonNumberIds.contains(id); } } @@ -81,24 +63,23 @@ public boolean contains(Object o) { public Iterator iterator() { return new ExtendableIterator<>( this.nonNumberIds.iterator(), - new EcLongIterator(this.numberIds.longIterator())); + new EcIdIterator(this.numberIds.longIterator())); } @Override public boolean add(Id id) { - if (id instanceof IdGenerator.LongId) { - return this.numberIds.add(((IdGenerator.LongId) id).longValue()); + if (id.type() == Id.IdType.LONG) { + return this.numberIds.add(id.asLong()); } else { return this.nonNumberIds.add(id); } } - @Override - public boolean remove(Object o) { - if (o instanceof IdGenerator.LongId) { - return this.numberIds.remove(((IdGenerator.LongId) o).longValue()); + public boolean remove(Id id) { + if (id.type() == Id.IdType.LONG) { + return this.numberIds.remove(id.asLong()); } else { - return this.nonNumberIds.remove(o); + return this.nonNumberIds.remove(id); } } @@ -108,11 +89,11 @@ public void clear() { this.nonNumberIds.clear(); } - private static class EcLongIterator implements Iterator { + private static class EcIdIterator implements Iterator { private final MutableLongIterator iterator; - public EcLongIterator(MutableLongIterator iter) { + public EcIdIterator(MutableLongIterator iter) { this.iterator = iter; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/Int2IntsMap.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/Int2IntsMap.java index fc0eeafb74..fd2d2062fa 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/Int2IntsMap.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/Int2IntsMap.java @@ -22,31 +22,31 @@ import org.eclipse.collections.api.iterator.IntIterator; import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; +// TODO: move to common-module public class Int2IntsMap { private static final int INIT_KEY_CAPACITY = 16; private static final int CHUNK_SIZE = 10; private static final int EXPANSION_FACTOR = 2; - private static final int OFFSET_POSITION = 0; + private static final int OFFSET_NEXT_FREE = 0; private static final int OFFSET_SIZE = 1; - private static final int OFFSET_DATA_IN_FIRST_CHUNK = 2; - + private static final int OFFSET_FIRST_CHUNK_DATA = 2; /* - * chunkMap chunkTable + * chunkMap chunkTable * * -------- --------------- - * | 1 | 0 |--------->0 | 33 | (nextPosition) - * | 2 | 10 |-----+ 1 | 19 | (size) + * | 1 | 0 |--------->0 | 33 | nextFree (free entry pointer) + * | 2 | 10 |-----+ 1 | 19 | size (values count of key) * | 3 | 40 |---+ | 2 | int data | * | . | . | | | 3 | int data | - * | x | y | | | . | ... | + * | x | y | | | . | ... | point to nextFreeChunk * -------- | | 9 | 20 |-----------------+ * | | --------------- | - * | +-->10 | 13 | (nextPosition) | - * | 11 | 1 | (size) | - * | 12 | int data | | nextChunk + * | +-->10 | 13 | nextFree | + * | 11 | 1 | size | + * | 12 | int data | | * | 13 | 0 | | * | . | ... | | * | 19 | 0 | | @@ -55,9 +55,9 @@ public class Int2IntsMap { * | 21 | int data | * | 22 | int data | * | 23 | int data | - * | . | ... | + * | . | ... | point to nextFreeChunk * | 29 | 30 |-----------------+ - * | --------------- | nextChunk + * | --------------- | * | 30 | int data |<----------------+ * | 31 | int data | * | 32 | int data | @@ -65,8 +65,8 @@ public class Int2IntsMap { * | . | ... | * | 39 | 0 | * | --------------- - * +---->40 | 48 | (nextPosition) - * 41 | 6 | (size) + * +---->40 | 48 | nextFree + * 41 | 6 | size * 42 | int data | * 43 | int data | * . | ... | @@ -78,45 +78,47 @@ public class Int2IntsMap { * | ... | * | ... | * | ... | - * - * */ private IntIntHashMap chunkMap; private int[] chunkTable; - private int nextChunk; + private int nextFreeChunk; public Int2IntsMap() { this.chunkMap = new IntIntHashMap(INIT_KEY_CAPACITY); this.chunkTable = new int[INIT_KEY_CAPACITY * CHUNK_SIZE]; - this.nextChunk = 0; + this.nextFreeChunk = 0; } public void add(int key, int value) { if (this.chunkMap.containsKey(key)) { int firstChunk = this.chunkMap.get(key); - int nextPosition = this.chunkTable[firstChunk + OFFSET_POSITION]; - if (!this.endOfChunk(nextPosition)) { - chunkTable[nextPosition] = value; - this.chunkTable[firstChunk + OFFSET_POSITION]++; + /* + * The nextFree represent the position where the next element + * will be located. + */ + int nextFree = this.chunkTable[firstChunk + OFFSET_NEXT_FREE]; + if (!this.endOfChunk(nextFree)) { + chunkTable[nextFree] = value; + this.chunkTable[firstChunk + OFFSET_NEXT_FREE]++; } else { /* - * nextPosition points to last entry of current chunk, should - * acquire a new chunk and make last entry of current chunk - * points new chunk + * If the nextFree points to the end of last chunk, + * allocate a new chunk and let the nextFree point to + * the start of new allocated chunk. */ this.ensureCapacity(); - int lastEntryOfChunk = nextPosition; - this.chunkTable[lastEntryOfChunk] = this.nextChunk; + int lastEntryOfChunk = nextFree; + this.chunkTable[lastEntryOfChunk] = this.nextFreeChunk; - nextPosition = this.nextChunk; - this.chunkTable[nextPosition] = value; - this.chunkTable[firstChunk + OFFSET_POSITION] = nextPosition + 1; + nextFree = this.nextFreeChunk; + this.chunkTable[nextFree] = value; + this.chunkTable[firstChunk + OFFSET_NEXT_FREE] = nextFree + 1; // Update next block - this.nextChunk += CHUNK_SIZE; + this.nextFreeChunk += CHUNK_SIZE; } this.chunkTable[firstChunk + OFFSET_SIZE]++; } else { @@ -124,17 +126,17 @@ public void add(int key, int value) { this.ensureCapacity(); // Allocate 1st chunk - this.chunkMap.put(key, this.nextChunk); + this.chunkMap.put(key, this.nextFreeChunk); // Init first chunk - int firstChunk = this.nextChunk; - int nextPosition = firstChunk + OFFSET_DATA_IN_FIRST_CHUNK; - this.chunkTable[firstChunk + OFFSET_POSITION] = nextPosition + 1; + int firstChunk = this.nextFreeChunk; + int nextFree = firstChunk + OFFSET_FIRST_CHUNK_DATA; + this.chunkTable[firstChunk + OFFSET_NEXT_FREE] = nextFree + 1; this.chunkTable[firstChunk + OFFSET_SIZE] = 1; - this.chunkTable[nextPosition] = value; + this.chunkTable[nextFree] = value; // Update next block - this.nextChunk += CHUNK_SIZE; + this.nextFreeChunk += CHUNK_SIZE; } } @@ -142,11 +144,11 @@ public boolean containsKey(int key) { return this.chunkMap.containsKey(key); } - public int[] get(int key) { + public int[] getValues(int key) { int firstChunk = this.chunkMap.get(key); int size = this.chunkTable[firstChunk + OFFSET_SIZE]; int[] values = new int[size]; - int position = firstChunk + OFFSET_DATA_IN_FIRST_CHUNK; + int position = firstChunk + OFFSET_FIRST_CHUNK_DATA; int i = 0; while (i < size) { if (!this.endOfChunk(position)) { @@ -167,16 +169,17 @@ public int size() { } private boolean endOfChunk(int position) { + // The last entry of chunk is next chunk pointer return (position + 1) % CHUNK_SIZE == 0; } private void ensureCapacity() { - if (this.nextChunk >= this.chunkTable.length) { - this.expansion(); + if (this.nextFreeChunk >= this.chunkTable.length) { + this.expand(); } } - private void expansion() { + private void expand() { int currentSize = this.chunkTable.length; int[] newTable = new int[currentSize * EXPANSION_FACTOR]; System.arraycopy(this.chunkTable, 0, newTable, 0, currentSize); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/MappingFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/MappingFactory.java similarity index 91% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/MappingFactory.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/MappingFactory.java index 1060ee9b53..cc243c8df7 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/MappingFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/MappingFactory.java @@ -17,7 +17,7 @@ * under the License. */ -package com.baidu.hugegraph.util.collection.mapping; +package com.baidu.hugegraph.util.collection; public class MappingFactory { @@ -28,6 +28,6 @@ public static ObjectIntMapping newObjectIntMapping() { public static ObjectIntMapping newObjectIntMapping( boolean concurrent) { return concurrent ? new ConcurrentObjectIntMapping<>() : - new SingleObjectIntMapping<>(); + new SingleThreadObjectIntMapping<>(); } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java similarity index 91% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ObjectIntMapping.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java index 45d63b4747..21737092df 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/ObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ObjectIntMapping.java @@ -17,11 +17,13 @@ * under the License. */ -package com.baidu.hugegraph.util.collection.mapping; +package com.baidu.hugegraph.util.collection; public interface ObjectIntMapping { public int object2Code(Object object); - public Object code2Object(int code); + + public V code2Object(int code); + public void clear(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/SingleThreadObjectIntMapping.java similarity index 87% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/SingleThreadObjectIntMapping.java index 2e6bcb2a26..4cf5d0198d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/mapping/SingleObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/SingleThreadObjectIntMapping.java @@ -17,7 +17,7 @@ * under the License. */ -package com.baidu.hugegraph.util.collection.mapping; +package com.baidu.hugegraph.util.collection; import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; @@ -25,18 +25,18 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.perf.PerfUtil.Watched; -public class SingleObjectIntMapping implements ObjectIntMapping { +public class SingleThreadObjectIntMapping implements ObjectIntMapping { private static final int MAGIC = 1 << 16; private final IntObjectHashMap int2IdMap; - public SingleObjectIntMapping() { + public SingleThreadObjectIntMapping() { this.int2IdMap = new IntObjectHashMap<>(); } @Watched @SuppressWarnings("unchecked") - public synchronized int object2Code(Object object) { + public int object2Code(Object object) { int key = object.hashCode(); // TODO: improve hash algorithm for (int i = 1; i > 0; i <<= 1) { @@ -56,7 +56,7 @@ public synchronized int object2Code(Object object) { } @Watched - public synchronized Object code2Object(int code) { + public V code2Object(int code) { return this.int2IdMap.get(code); } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java index 84b88dfea2..73a7fea018 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java @@ -67,6 +67,7 @@ import com.baidu.hugegraph.unit.util.JsonUtilTest; import com.baidu.hugegraph.unit.util.StringEncodingTest; import com.baidu.hugegraph.unit.util.VersionTest; +import com.baidu.hugegraph.util.collection.IdSet; @RunWith(Suite.class) @Suite.SuiteClasses({ @@ -107,6 +108,7 @@ PageStateTest.class, Int2IntsMapTest.class, ObjectIntMappingTest.class, + IdSet.class, /* serializer */ BytesBufferTest.class, diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/IdSetTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/IdSetTest.java new file mode 100644 index 0000000000..b5ba03a869 --- /dev/null +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/IdSetTest.java @@ -0,0 +1,170 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.unit.core; + +import java.util.HashSet; +import java.util.Random; +import java.util.Set; +import java.util.UUID; + +import org.apache.commons.lang.RandomStringUtils; +import org.eclipse.collections.impl.set.mutable.primitive.LongHashSet; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.id.IdGenerator; +import com.baidu.hugegraph.testutil.Assert; +import com.baidu.hugegraph.testutil.Whitebox; +import com.baidu.hugegraph.type.define.CollectionType; +import com.baidu.hugegraph.util.collection.IdSet; + +public class IdSetTest { + + private static final int SIZE = 10000; + + private static IdSet idSet; + + @Before + public void setup() { + // pass + } + + @After + public void teardown() { + // pass + } + + @Test + public void testIdSetWithOnlyNumberId() { + Random random = new Random(); + Set numbers = new HashSet<>(); + for (CollectionType type : CollectionType.values()) { + idSet = new IdSet(type); + for (int i = 1; i < SIZE; i++) { + long number = random.nextLong(); + numbers.add(number); + idSet.add(IdGenerator.of(number)); + } + Assert.assertEquals(numbers.size(), idSet.size()); + + LongHashSet numberIds = Whitebox.getInternalState(idSet, + "numberIds"); + Assert.assertEquals(numbers.size(), numberIds.size()); + Set nonNumberIds = Whitebox.getInternalState(idSet, + "nonNumberIds"); + Assert.assertTrue(nonNumberIds.isEmpty()); + + numbers.clear(); + idSet.clear(); + } + } + + @Test + public void testIdSetWithOnlyUUIDId() { + Set uuids = new HashSet<>(); + for (CollectionType type : CollectionType.values()) { + idSet = new IdSet(type); + for (int i = 0; i < SIZE; i++) { + UUID uuid = UUID.randomUUID(); + uuids.add(uuid); + idSet.add(IdGenerator.of(uuid)); + } + Assert.assertEquals(uuids.size(), idSet.size()); + + LongHashSet numberIds = Whitebox.getInternalState(idSet, + "numberIds"); + Assert.assertTrue(numberIds.isEmpty()); + Set nonNumberIds = Whitebox.getInternalState(idSet, + "nonNumberIds"); + Assert.assertEquals(uuids.size(), nonNumberIds.size()); + + uuids.clear(); + idSet.clear(); + } + } + + + @Test + public void testIdSetWithOnlyStringId() { + Set strings = new HashSet<>(); + for (CollectionType type : CollectionType.values()) { + idSet = new IdSet(type); + for (int i = 0; i < SIZE; i++) { + String string = RandomStringUtils.randomAlphanumeric(10); + strings.add(string); + idSet.add(IdGenerator.of(string)); + } + Assert.assertEquals(strings.size(), idSet.size()); + + LongHashSet numberIds = Whitebox.getInternalState(idSet, + "numberIds"); + Assert.assertTrue(numberIds.isEmpty()); + Set nonNumberIds = Whitebox.getInternalState(idSet, + "nonNumberIds"); + Assert.assertEquals(strings.size(), nonNumberIds.size()); + + strings.clear(); + idSet.clear(); + } + } + + @Test + public void testIdSetWithMixedId() { + Random random = new Random(); + Set numbers = new HashSet<>(); + Set uuids = new HashSet<>(); + Set strings = new HashSet<>(); + for (CollectionType type : CollectionType.values()) { + idSet = new IdSet(type); + for (int i = 1; i < SIZE; i++) { + long number = random.nextLong(); + numbers.add(number); + idSet.add(IdGenerator.of(number)); + } + for (int i = 0; i < SIZE; i++) { + UUID uuid = UUID.randomUUID(); + uuids.add(uuid); + idSet.add(IdGenerator.of(uuid)); + } + for (int i = 0; i < SIZE; i++) { + String string = RandomStringUtils.randomAlphanumeric(10); + strings.add(string); + idSet.add(IdGenerator.of(string)); + } + Assert.assertEquals(numbers.size() + uuids.size() + strings.size(), + idSet.size()); + + LongHashSet numberIds = Whitebox.getInternalState(idSet, + "numberIds"); + Assert.assertEquals(numbers.size(), numberIds.size()); + Set nonNumberIds = Whitebox.getInternalState(idSet, + "nonNumberIds"); + Assert.assertEquals(uuids.size() + strings.size(), + nonNumberIds.size()); + + numbers.clear(); + uuids.clear(); + strings.clear(); + idSet.clear(); + } + } +} diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java index 16f41e1267..52b2ca1b81 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java @@ -107,7 +107,7 @@ public void testInt2IntsMap() { 290, 291,292,293,294,295,296,297,298,299}; for (int ii = 0; ii < 5; ii++) { - int[] result = map.get(ii + 1); + int[] result = map.getValues(ii + 1); Assert.assertTrue(Arrays.equals(results[ii], result)); } } @@ -121,13 +121,13 @@ public void testInt2IntsMapRandom() { map.add(Math.abs(random.nextInt()) % 5, i); } - Set results = new HashSet<>(); + Set all = new HashSet<>(); for (int i = 0; i < 5; i++) { - int[] result = map.get(i); - for (int j : result) { - results.add(j); + int[] values = map.getValues(i); + for (int j : values) { + all.add(j); } } - Assert.assertEquals(1000, results.size()); + Assert.assertEquals(1000, all.size()); } } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/ObjectIntMappingTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/ObjectIntMappingTest.java index 1f20e7e262..85181d266d 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/ObjectIntMappingTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/ObjectIntMappingTest.java @@ -31,11 +31,12 @@ import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.id.IdGenerator; import com.baidu.hugegraph.testutil.Assert; -import com.baidu.hugegraph.util.collection.mapping.MappingFactory; -import com.baidu.hugegraph.util.collection.mapping.ObjectIntMapping; +import com.baidu.hugegraph.util.collection.MappingFactory; +import com.baidu.hugegraph.util.collection.ObjectIntMapping; public class ObjectIntMappingTest { + private static final int OBJECT_NUMBER = 1000000; private static ObjectIntMapping mapping = MappingFactory.newObjectIntMapping(); @@ -47,10 +48,10 @@ public void clear() { @Test public void testNumberIdMapping() { Set codes = new LinkedHashSet<>(); - for (int i = 0; i < 1000000; i++) { + for (int i = 0; i < OBJECT_NUMBER; i++) { codes.add(mapping.object2Code(IdGenerator.of(i))); } - Assert.assertEquals(1000000, codes.size()); + Assert.assertEquals(OBJECT_NUMBER, codes.size()); int j = 0; for (Integer code : codes) { @@ -62,7 +63,7 @@ public void testNumberIdMapping() { public void testStringIdMapping() { Set strings = new LinkedHashSet<>(); Set codes = new LinkedHashSet<>(); - for (int i = 0; i < 1000000; i++) { + for (int i = 0; i < OBJECT_NUMBER; i++) { String string = RandomStringUtils.randomAlphanumeric(10); strings.add(string); codes.add(mapping.object2Code(IdGenerator.of(string))); @@ -84,7 +85,7 @@ public void testStringIdMapping() { public void testUUIDIdMapping() { Set uuids = new LinkedHashSet<>(); Set codes = new LinkedHashSet<>(); - for (int i = 0; i < 1000000; i++) { + for (int i = 0; i < OBJECT_NUMBER; i++) { UUID uuid = UUID.randomUUID(); uuids.add(uuid); codes.add(mapping.object2Code(IdGenerator.of(uuid))); @@ -108,20 +109,20 @@ public void testMixedIdMapping() { Set objects = new LinkedHashSet<>(); Object object; - for (int i = 0; i < 1000000; i++) { + for (int i = 0; i < OBJECT_NUMBER; i++) { object = IdGenerator.of(i); objects.add(object); codes.add(mapping.object2Code(object)); } - for (int i = 0; i < 1000000; i++) { + for (int i = 0; i < OBJECT_NUMBER; i++) { String string = RandomStringUtils.randomAlphanumeric(10); object = IdGenerator.of(string); objects.add(object); codes.add(mapping.object2Code(object)); } - for (int i = 0; i < 1000000; i++) { + for (int i = 0; i < OBJECT_NUMBER; i++) { UUID uuid = UUID.randomUUID(); object = IdGenerator.of(uuid); objects.add(object); From 3107bba35d2a38eb6fdeb3a76b46bce3a4bd92b9 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 25 May 2021 18:11:57 +0800 Subject: [PATCH 35/38] improve Change-Id: I1a66e5ea99a32f6cead79b80317dc7d7b3a77fa7 --- .../algorithm/records/AbstractRecords.java | 9 +++ .../records/DoubleWayMultiPathsRecords.java | 38 +++++++---- .../algorithm/records/KneighborRecords.java | 12 ++-- .../algorithm/records/KoutRecords.java | 11 ++-- .../records/ShortestPathRecords.java | 13 ++-- .../records/SingleWayMultiPathsRecords.java | 22 +++---- .../util/collection/CollectionFactory.java | 63 ++----------------- .../baidu/hugegraph/unit/UnitTestSuite.java | 3 +- 8 files changed, 75 insertions(+), 96 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java index dd45208d66..e75c94bea6 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/AbstractRecords.java @@ -32,6 +32,7 @@ public abstract class AbstractRecords implements Records { private final ObjectIntMapping idMapping; private final RecordType type; private final boolean concurrent; + private Record currentRecord; public AbstractRecords(RecordType type, boolean concurrent) { this.type = type; @@ -52,4 +53,12 @@ protected Id id(int code) { protected Record newRecord() { return RecordFactory.newRecord(this.type, this.concurrent); } + + protected Record currentRecord() { + return this.currentRecord; + } + + protected void currentRecord(Record record) { + this.currentRecord = record; + } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java index 0757430cb3..2dfb195e8e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java @@ -38,13 +38,12 @@ public abstract class DoubleWayMultiPathsRecords extends AbstractRecords { - protected final Stack sourceRecords; - protected final Stack targetRecords; + private final Stack sourceRecords; + private final Stack targetRecords; - protected Record currentRecord; private IntIterator lastRecordKeys; - protected int current; - protected boolean forward; + private int current; + private boolean forward; private int accessed; public DoubleWayMultiPathsRecords(RecordType type, boolean concurrent, @@ -67,19 +66,20 @@ public DoubleWayMultiPathsRecords(RecordType type, boolean concurrent, @Override public void startOneLayer(boolean forward) { this.forward = forward; - this.currentRecord = this.newRecord(); + this.currentRecord(this.newRecord()); this.lastRecordKeys = this.forward ? this.sourceRecords.peek().keys() : this.targetRecords.peek().keys(); } @Override public void finishOneLayer() { + Record record = this.currentRecord(); if (this.forward) { - this.sourceRecords.push(this.currentRecord); + this.sourceRecords.push(record); } else { - this.targetRecords.push(this.currentRecord); + this.targetRecords.push(record); } - this.accessed += this.currentRecord.size(); + this.accessed += record.size(); } @Watched @@ -92,7 +92,7 @@ public boolean hasNextKey() { @Override public Id nextKey() { this.current = this.lastRecordKeys.next(); - return this.id(current); + return this.id(this.current); } @Watched @@ -193,6 +193,22 @@ private PathSet linkPath(Stack all, int id, int layerIndex) { @Watched protected void addPath(int current, int parent) { - this.currentRecord.addPath(current, parent); + this.currentRecord().addPath(current, parent); + } + + protected Stack sourceRecords() { + return this.sourceRecords; + } + + protected Stack targetRecords() { + return this.targetRecords; + } + + protected boolean forward() { + return this.forward; + } + + protected int current() { + return this.current; } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java index 24762006b6..98c530bbd7 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java @@ -22,10 +22,12 @@ import static com.baidu.hugegraph.backend.query.Query.NO_LIMIT; import java.util.Set; +import java.util.Stack; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; import com.baidu.hugegraph.traversal.algorithm.records.record.IntIterator; +import com.baidu.hugegraph.traversal.algorithm.records.record.Record; import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; import com.baidu.hugegraph.type.define.CollectionType; import com.baidu.hugegraph.util.collection.CollectionFactory; @@ -44,8 +46,9 @@ public int size() { public Set ids(long limit) { Set ids = CollectionFactory.newIdSet(CollectionType.EC); - for (int i = 1; i < this.records.size(); i++) { - IntIterator iterator = this.records.get(i).keys(); + Stack records = this.records(); + for (int i = 1; i < records.size(); i++) { + IntIterator iterator = records.get(i).keys(); while ((limit == NO_LIMIT || limit > 0L) && iterator.hasNext()) { ids.add(this.id(iterator.next())); limit--; @@ -56,8 +59,9 @@ public Set ids(long limit) { public PathSet paths(long limit) { PathSet paths = new PathSet(); - for (int i = 1; i < this.records.size(); i++) { - IntIterator iterator = this.records.get(i).keys(); + Stack records = this.records(); + for (int i = 1; i < records.size(); i++) { + IntIterator iterator = records.get(i).keys(); while ((limit == NO_LIMIT || limit > 0L) && iterator.hasNext()) { paths.add(this.getPath(i, iterator.next())); limit--; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java index 1302e9612a..da58b6cd3b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KoutRecords.java @@ -22,10 +22,12 @@ import static com.baidu.hugegraph.backend.query.Query.NO_LIMIT; import java.util.Set; +import java.util.Stack; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.PathSet; import com.baidu.hugegraph.traversal.algorithm.records.record.IntIterator; +import com.baidu.hugegraph.traversal.algorithm.records.record.Record; import com.baidu.hugegraph.traversal.algorithm.records.record.RecordType; import com.baidu.hugegraph.type.define.CollectionType; import com.baidu.hugegraph.util.collection.CollectionFactory; @@ -39,11 +41,11 @@ public KoutRecords(RecordType type, boolean concurrent, @Override public int size() { - return this.currentRecord.size(); + return this.currentRecord().size(); } public Set ids(long limit) { - IntIterator iterator = this.records.peek().keys(); + IntIterator iterator = this.records().peek().keys(); Set ids = CollectionFactory.newIdSet(CollectionType.EC); while ((limit == NO_LIMIT || limit-- > 0L) && iterator.hasNext()) { ids.add(this.id(iterator.next())); @@ -53,9 +55,10 @@ public Set ids(long limit) { public PathSet paths(long limit) { PathSet paths = new PathSet(); - IntIterator iterator = this.records.peek().keys(); + Stack records = this.records(); + IntIterator iterator = records.peek().keys(); while ((limit == NO_LIMIT || limit-- > 0L) && iterator.hasNext()) { - paths.add(this.getPath(this.records.size() - 1, iterator.next())); + paths.add(this.getPath(records.size() - 1, iterator.next())); } return paths; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java index 23d3513f16..b8b69c06c6 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java @@ -59,8 +59,9 @@ public PathSet findPath(Id target, Function filter, if (!filter.apply(target)) { return paths; } - paths.add(this.forward ? this.linkPath(this.current, targetCode) : - this.linkPath(targetCode, this.current)); + paths.add(this.forward() ? + this.linkPath(this.current(), targetCode) : + this.linkPath(targetCode, this.current())); this.pathFound = true; if (!all) { return paths; @@ -73,13 +74,13 @@ public PathSet findPath(Id target, Function filter, * 2. path of node doesn't have loop */ if (!this.pathFound && this.isNew(targetCode)) { - this.addPath(targetCode, this.current); + this.addPath(targetCode, this.current()); } return paths; } private boolean isNew(int node) { - return !this.currentRecord.containsKey(node) && + return !this.currentRecord().containsKey(node) && !this.accessedVertices.contains(node); } @@ -93,11 +94,11 @@ private Path linkPath(int source, int target) { } private Path linkSourcePath(int source) { - return this.linkPath(this.sourceRecords, source); + return this.linkPath(this.sourceRecords(), source); } private Path linkTargetPath(int target) { - return this.linkPath(this.targetRecords, target); + return this.linkPath(this.targetRecords(), target); } private Path linkPath(Stack all, int node) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java index 48b56eaa3e..2e21dc1cd5 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/SingleWayMultiPathsRecords.java @@ -43,16 +43,13 @@ public abstract class SingleWayMultiPathsRecords extends AbstractRecords { - protected final Stack records; + private final Stack records; private final int sourceCode; private final boolean nearest; private final MutableIntSet accessedVertices; - protected Record currentRecord; - protected IntIterator lastRecordKeys; - protected int current; - protected boolean forward; + private IntIterator lastRecordKeys; public SingleWayMultiPathsRecords(RecordType type, boolean concurrent, Id source, boolean nearest) { @@ -72,13 +69,13 @@ public SingleWayMultiPathsRecords(RecordType type, boolean concurrent, @Override public void startOneLayer(boolean forward) { - this.currentRecord = this.newRecord(); + this.currentRecord(this.newRecord()); this.lastRecordKeys = this.records.peek().keys(); } @Override public void finishOneLayer() { - this.records.push(this.currentRecord); + this.records.push(this.currentRecord()); } @Override @@ -88,8 +85,7 @@ public boolean hasNextKey() { @Override public Id nextKey() { - this.current = this.lastRecordKeys.next(); - return this.id(current); + return this.id(this.lastRecordKeys.next()); } @Override @@ -119,11 +115,11 @@ public void addPath(Id source, Id target) { int sourceCode = this.code(source); int targetCode = this.code(target); if (this.nearest && this.accessedVertices.contains(targetCode) || - !this.nearest && this.currentRecord.containsKey(targetCode) || + !this.nearest && this.currentRecord().containsKey(targetCode) || targetCode == this.sourceCode) { return; } - this.currentRecord.addPath(targetCode, sourceCode); + this.currentRecord().addPath(targetCode, sourceCode); this.accessedVertices.add(targetCode); } @@ -173,4 +169,8 @@ public Path getPath(int layerIndex, int target) { Collections.reverse(ids); return new Path(ids); } + + public Stack records() { + return this.records; + } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java index b3be5be893..a7a021bc62 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java @@ -220,63 +220,6 @@ public static MutableIntObjectMap newIntObjectMap( return new IntObjectHashMap<>(map); } - public static MutableIntObjectMap newIntObjectMap( - int key1, V value1) { - return IntObjectHashMap.newWithKeysValues(key1, value1); - } - - public static MutableIntObjectMap newIntObjectMap( - int key1, V value1, - int key2, V value2) { - return IntObjectHashMap.newWithKeysValues(key1, value1, key2, value2); - } - - public static MutableIntObjectMap newIntObjectMap( - int key1, V value1, - int key2, V value2, - int key3, V value3) { - return IntObjectHashMap.newWithKeysValues(key1, value1, key2, value2, - key3, value3); - } - - @SuppressWarnings("unchecked") - public static MutableIntObjectMap newIntObjectMap( - int key1, V value1, - int key2, V value2, - int key3, V value3, - Object... objects) { - IntObjectHashMap map = IntObjectHashMap.newWithKeysValues( - key1, value1, key2, value2, key3, value3); - E.checkArgument(objects.length % 2 == 0, - "Must provide even arguments for " + - "CollectionFactory.newIntObjectMap"); - for (int i = 0; i < objects.length; i += 2) { - map.put((int) objects[i], (V) objects[i + 1]); - } - return map; - } - - public static MutableIntObjectMap newIntObjectMap(Id key1, - V value1) { - return IntObjectHashMap.newWithKeysValues((int) key1.asLong(), value1); - } - - public static MutableIntObjectMap newIntObjectMap( - Id key1, V value1, - Id key2, V value2) { - return IntObjectHashMap.newWithKeysValues((int) key1.asLong(), value1, - (int) key2.asLong(), value2); - } - - public static MutableIntObjectMap newIntObjectMap( - Id key1, V value1, - Id key2, V value2, - Id key3, V value3) { - return IntObjectHashMap.newWithKeysValues((int) key1.asLong(), value1, - (int) key2.asLong(), value2, - (int) key3.asLong(), value3); - } - @SuppressWarnings("unchecked") public static MutableIntObjectMap newIntObjectMap( Object... objects) { @@ -284,8 +227,10 @@ public static MutableIntObjectMap newIntObjectMap( E.checkArgument(objects.length % 2 == 0, "Must provide even arguments for " + "CollectionFactory.newIntObjectMap"); - for (int i = 0; i < objects.length; i+=2) { - map.put((int) ((Id) objects[i]).asLong(), (V) objects[i + 1]); + for (int i = 0; i < objects.length; i += 2) { + int key = objects[i] instanceof Id ? + (int) ((Id) objects[i]).asLong() : (int) objects[i]; + map.put(key, (V) objects[i + 1]); } return map; } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java index 73a7fea018..94a6f35506 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java @@ -36,6 +36,7 @@ import com.baidu.hugegraph.unit.core.DataTypeTest; import com.baidu.hugegraph.unit.core.DirectionsTest; import com.baidu.hugegraph.unit.core.ExceptionTest; +import com.baidu.hugegraph.unit.core.IdSetTest; import com.baidu.hugegraph.unit.core.Int2IntsMapTest; import com.baidu.hugegraph.unit.core.LocksTableTest; import com.baidu.hugegraph.unit.core.PageStateTest; @@ -108,7 +109,7 @@ PageStateTest.class, Int2IntsMapTest.class, ObjectIntMappingTest.class, - IdSet.class, + IdSetTest.class, /* serializer */ BytesBufferTest.class, From ff5b60e9dcc3ee357bd57e46e60ebb7d2ccfdf15 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 27 May 2021 22:07:58 +0800 Subject: [PATCH 36/38] improve Change-Id: I80bd3d4ae961794cc82139262d605fe400f8c28b --- .../records/DoubleWayMultiPathsRecords.java | 28 ++++++++----------- .../algorithm/records/KneighborRecords.java | 1 + .../records/ShortestPathRecords.java | 3 +- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java index 2dfb195e8e..ad76673e33 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/DoubleWayMultiPathsRecords.java @@ -42,7 +42,7 @@ public abstract class DoubleWayMultiPathsRecords extends AbstractRecords { private final Stack targetRecords; private IntIterator lastRecordKeys; - private int current; + private int currentKey; private boolean forward; private int accessed; @@ -91,8 +91,8 @@ public boolean hasNextKey() { @Watched @Override public Id nextKey() { - this.current = this.lastRecordKeys.next(); - return this.id(this.current); + this.currentKey = this.lastRecordKeys.next(); + return this.id(this.currentKey); } @Watched @@ -102,12 +102,13 @@ public PathSet findPath(Id target, Function filter, PathSet results = new PathSet(); int targetCode = this.code(target); // If cross point exists, path found, concat them - if (this.contains(targetCode)) { - results = this.forward ? - this.linkPath(this.current, targetCode, ring) : - this.linkPath(targetCode, this.current, ring); + if (this.forward && this.targetContains(targetCode)) { + results = this.linkPath(this.currentKey, targetCode, ring); } - this.addPath(targetCode, this.current); + if (!this.forward && this.sourceContains(targetCode)) { + results = this.linkPath(targetCode, this.currentKey, ring); + } + this.addPath(targetCode, this.currentKey); return results; } @@ -116,16 +117,11 @@ public long accessed() { return this.accessed; } - protected boolean contains(int node) { - return this.forward ? this.targetContains(node) : - this.sourceContains(node); - } - - private boolean sourceContains(int node) { + protected boolean sourceContains(int node) { return this.sourceRecords.peek().containsKey(node); } - private boolean targetContains(int node) { + protected boolean targetContains(int node) { return this.targetRecords.peek().containsKey(node); } @@ -209,6 +205,6 @@ protected boolean forward() { } protected int current() { - return this.current; + return this.currentKey; } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java index 98c530bbd7..df7eeb4fa0 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/KneighborRecords.java @@ -47,6 +47,7 @@ public int size() { public Set ids(long limit) { Set ids = CollectionFactory.newIdSet(CollectionType.EC); Stack records = this.records(); + // Not include record(i=0) to ignore source vertex for (int i = 1; i < records.size(); i++) { IntIterator iterator = records.get(i).keys(); while ((limit == NO_LIMIT || limit > 0L) && iterator.hasNext()) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java index b8b69c06c6..9f30bb035d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/algorithm/records/ShortestPathRecords.java @@ -55,7 +55,8 @@ public PathSet findPath(Id target, Function filter, PathSet paths = new PathSet(); int targetCode = this.code(target); // If cross point exists, shortest path found, concat them - if (this.contains(targetCode)) { + if (this.forward() && this.targetContains(targetCode) || + !this.forward() && this.sourceContains(targetCode)) { if (!filter.apply(target)) { return paths; } From d9b330090dac83dca0bfd32f4b838b771905fcd8 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 3 Jun 2021 17:56:17 +0800 Subject: [PATCH 37/38] improve Change-Id: I14abe49850d42f34576c6d1837ed5b620ffbda21 --- .../baidu/hugegraph/structure/HugeElement.java | 2 +- .../SingleThreadObjectIntMapping.java | 18 +++++++++++------- .../hugegraph/unit/core/Int2IntsMapTest.java | 4 ++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java index cbd03d37f3..825eb73585 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java @@ -59,7 +59,7 @@ public abstract class HugeElement implements Element, GraphType, Idfiable { private static final MutableIntObjectMap> EMPTY_MAP = - new IntObjectHashMap<>(); + CollectionFactory.newIntObjectMap(); private static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX; private final HugeGraph graph; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/SingleThreadObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/SingleThreadObjectIntMapping.java index 4cf5d0198d..b254565adb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/SingleThreadObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/SingleThreadObjectIntMapping.java @@ -28,6 +28,7 @@ public class SingleThreadObjectIntMapping implements ObjectIntMapping { private static final int MAGIC = 1 << 16; + private final IntObjectHashMap int2IdMap; public SingleThreadObjectIntMapping() { @@ -37,19 +38,22 @@ public SingleThreadObjectIntMapping() { @Watched @SuppressWarnings("unchecked") public int object2Code(Object object) { - int key = object.hashCode(); + int code = object.hashCode(); // TODO: improve hash algorithm for (int i = 1; i > 0; i <<= 1) { - for (int j = 0; i >= MAGIC && j < 10; j++) { - Id existed = (Id) this.int2IdMap.get(key); + for (int j = 0; j < 10; j++) { + Id existed = (Id) this.int2IdMap.get(code); if (existed == null) { - this.int2IdMap.put(key, (V) object); - return key; + this.int2IdMap.put(code, (V) object); + return code; } if (existed.equals(object)) { - return key; + return code; + } + code = code + i + j; + if (i < MAGIC) { + break; } - key = key + i + j; } } throw new HugeException("Failed to get code for id: %s", object); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java index 52b2ca1b81..cfe8abbcf3 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java @@ -104,11 +104,11 @@ public void testInt2IntsMap() { 263,264,265,266,267,268,269}; results[4] = new int[]{270,271,272,273,274,275,276,277,278,279, 280,281,282,283,284,285,286,287,288,289, - 290, 291,292,293,294,295,296,297,298,299}; + 290,291,292,293,294,295,296,297,298,299}; for (int ii = 0; ii < 5; ii++) { int[] result = map.getValues(ii + 1); - Assert.assertTrue(Arrays.equals(results[ii], result)); + Assert.assertArrayEquals(results[ii], result); } } From 8b8be471bf7d109f3eaf54950c52a4e381f7e957 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 8 Jun 2021 16:26:35 +0800 Subject: [PATCH 38/38] add UT for CollectionFactory and IdSet Change-Id: I580df88f57302dfe474108a93108759e33e083e2 --- .../util/collection/CollectionFactory.java | 23 +- .../ConcurrentObjectIntMapping.java | 1 - .../SingleThreadObjectIntMapping.java | 13 +- .../baidu/hugegraph/unit/UnitTestSuite.java | 5 +- .../baidu/hugegraph/unit/core/IdSetTest.java | 158 +++++++ .../hugegraph/unit/core/Int2IntsMapTest.java | 60 ++- .../unit/util/CollectionFactoryTest.java | 387 ++++++++++++++++++ 7 files changed, 630 insertions(+), 17 deletions(-) create mode 100644 hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/CollectionFactoryTest.java diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java index a7a021bc62..0f98cc8c83 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/CollectionFactory.java @@ -174,6 +174,10 @@ public Map newMap(int initialCapacity) { return newMap(this.type, initialCapacity); } + public Map newMap(Map map) { + return newMap(this.type, map); + } + public static Map newMap(CollectionType type) { /* * EC is faster 10%-20% than JCF, and it's more stable & less @@ -207,6 +211,21 @@ public static Map newMap(CollectionType type, } } + public static Map newMap(CollectionType type, + Map map) { + switch (type) { + case EC: + return new UnifiedMap<>(map); + case JCF: + return new HashMap<>(map); + case FU: + return new Object2ObjectOpenHashMap<>(map); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + public static MutableIntObjectMap newIntObjectMap() { return new IntObjectHashMap<>(); } @@ -235,11 +254,11 @@ public static MutableIntObjectMap newIntObjectMap( return map; } - public Set newIdSet() { + public IdSet newIdSet() { return newIdSet(this.type); } - public static Set newIdSet(CollectionType type) { + public static IdSet newIdSet(CollectionType type) { return new IdSet(type); } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ConcurrentObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ConcurrentObjectIntMapping.java index 5d80678405..e19864c1ff 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ConcurrentObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/ConcurrentObjectIntMapping.java @@ -30,7 +30,6 @@ public ConcurrentObjectIntMapping() { } @Watched - @SuppressWarnings("unchecked") public synchronized int object2Code(Object object) { return this.objectIntMapping.object2Code(object); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/SingleThreadObjectIntMapping.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/SingleThreadObjectIntMapping.java index b254565adb..0084d0bac3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/SingleThreadObjectIntMapping.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/collection/SingleThreadObjectIntMapping.java @@ -22,12 +22,12 @@ import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; import com.baidu.hugegraph.HugeException; -import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.perf.PerfUtil.Watched; public class SingleThreadObjectIntMapping implements ObjectIntMapping { private static final int MAGIC = 1 << 16; + private static final int MAX_OFFSET = 10; private final IntObjectHashMap int2IdMap; @@ -41,8 +41,8 @@ public int object2Code(Object object) { int code = object.hashCode(); // TODO: improve hash algorithm for (int i = 1; i > 0; i <<= 1) { - for (int j = 0; j < 10; j++) { - Id existed = (Id) this.int2IdMap.get(code); + for (int j = 0; j < MAX_OFFSET; j++) { + V existed = this.int2IdMap.get(code); if (existed == null) { this.int2IdMap.put(code, (V) object); return code; @@ -51,12 +51,17 @@ public int object2Code(Object object) { return code; } code = code + i + j; + /* + * If i < MAGIC, try (i * 2) to reduce conflicts, otherwise + * try (i + 1), (i + 2), ..., (i + 10) to try more times + * before try (i * 2). + */ if (i < MAGIC) { break; } } } - throw new HugeException("Failed to get code for id: %s", object); + throw new HugeException("Failed to get code for object: %s", object); } @Watched diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java index 94a6f35506..cb89173e74 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java @@ -64,11 +64,11 @@ import com.baidu.hugegraph.unit.serializer.StoreSerializerTest; import com.baidu.hugegraph.unit.serializer.TableBackendEntryTest; import com.baidu.hugegraph.unit.serializer.TextBackendEntryTest; +import com.baidu.hugegraph.unit.util.CollectionFactoryTest; import com.baidu.hugegraph.unit.util.CompressUtilTest; import com.baidu.hugegraph.unit.util.JsonUtilTest; import com.baidu.hugegraph.unit.util.StringEncodingTest; import com.baidu.hugegraph.unit.util.VersionTest; -import com.baidu.hugegraph.util.collection.IdSet; @RunWith(Suite.class) @Suite.SuiteClasses({ @@ -136,7 +136,8 @@ VersionTest.class, JsonUtilTest.class, StringEncodingTest.class, - CompressUtilTest.class + CompressUtilTest.class, + CollectionFactoryTest.class }) public class UnitTestSuite { } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/IdSetTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/IdSetTest.java index b5ba03a869..be2af1c0d9 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/IdSetTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/IdSetTest.java @@ -20,6 +20,7 @@ package com.baidu.hugegraph.unit.core; import java.util.HashSet; +import java.util.Iterator; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -167,4 +168,161 @@ public void testIdSetWithMixedId() { idSet.clear(); } } + + @Test + public void testIdSetContains() { + Random random = new Random(); + Set numbers = new HashSet<>(); + Set uuids = new HashSet<>(); + Set strings = new HashSet<>(); + long number = 0L; + UUID uuid = null; + String string = null; + for (CollectionType type : CollectionType.values()) { + idSet = new IdSet(type); + for (int i = 1; i < SIZE; i++) { + number = random.nextLong(); + numbers.add(number); + idSet.add(IdGenerator.of(number)); + } + for (int i = 0; i < SIZE; i++) { + uuid = UUID.randomUUID(); + uuids.add(uuid); + idSet.add(IdGenerator.of(uuid)); + } + for (int i = 0; i < SIZE; i++) { + string = RandomStringUtils.randomAlphanumeric(10); + strings.add(string); + idSet.add(IdGenerator.of(string)); + } + Assert.assertEquals(numbers.size() + uuids.size() + strings.size(), + idSet.size()); + + LongHashSet numberIds = Whitebox.getInternalState(idSet, + "numberIds"); + Assert.assertEquals(numbers.size(), numberIds.size()); + Set nonNumberIds = Whitebox.getInternalState(idSet, + "nonNumberIds"); + Assert.assertEquals(uuids.size() + strings.size(), + nonNumberIds.size()); + + Assert.assertTrue(idSet.contains(IdGenerator.of(number))); + Assert.assertTrue(idSet.contains(IdGenerator.of(uuid))); + Assert.assertTrue(idSet.contains(IdGenerator.of(string))); + + numbers.clear(); + uuids.clear(); + strings.clear(); + idSet.clear(); + } + } + + @Test + public void testIdSetIterator() { + Random random = new Random(); + Set numbers = new HashSet<>(); + Set uuids = new HashSet<>(); + Set strings = new HashSet<>(); + for (CollectionType type : CollectionType.values()) { + idSet = new IdSet(type); + for (int i = 1; i < SIZE; i++) { + long number = random.nextLong(); + numbers.add(number); + idSet.add(IdGenerator.of(number)); + } + for (int i = 0; i < SIZE; i++) { + UUID uuid = UUID.randomUUID(); + uuids.add(uuid); + idSet.add(IdGenerator.of(uuid)); + } + for (int i = 0; i < SIZE; i++) { + String string = RandomStringUtils.randomAlphanumeric(10); + strings.add(string); + idSet.add(IdGenerator.of(string)); + } + Assert.assertEquals(numbers.size() + uuids.size() + strings.size(), + idSet.size()); + + LongHashSet numberIds = Whitebox.getInternalState(idSet, + "numberIds"); + Assert.assertEquals(numbers.size(), numberIds.size()); + Set nonNumberIds = Whitebox.getInternalState(idSet, + "nonNumberIds"); + Assert.assertEquals(uuids.size() + strings.size(), + nonNumberIds.size()); + + Iterator iterator = idSet.iterator(); + while (iterator.hasNext()) { + Id id = iterator.next(); + if (id instanceof IdGenerator.LongId) { + Assert.assertTrue(numbers.contains(id.asLong())); + } else if (id instanceof IdGenerator.UuidId) { + Assert.assertTrue(id.uuid() && + uuids.contains(id.asObject())); + } else { + Assert.assertTrue(id instanceof IdGenerator.StringId); + Assert.assertTrue(strings.contains(id.asString())); + } + } + + numbers.clear(); + uuids.clear(); + strings.clear(); + idSet.clear(); + } + } + + @Test + public void testIdSetRemove() { + Random random = new Random(); + Set numbers = new HashSet<>(); + Set uuids = new HashSet<>(); + Set strings = new HashSet<>(); + for (CollectionType type : CollectionType.values()) { + idSet = new IdSet(type); + for (int i = 1; i < SIZE; i++) { + long number = random.nextLong(); + numbers.add(number); + idSet.add(IdGenerator.of(number)); + } + for (int i = 0; i < SIZE; i++) { + UUID uuid = UUID.randomUUID(); + uuids.add(uuid); + idSet.add(IdGenerator.of(uuid)); + } + for (int i = 0; i < SIZE; i++) { + String string = RandomStringUtils.randomAlphanumeric(10); + strings.add(string); + idSet.add(IdGenerator.of(string)); + } + Assert.assertEquals(numbers.size() + uuids.size() + strings.size(), + idSet.size()); + + LongHashSet numberIds = Whitebox.getInternalState(idSet, + "numberIds"); + Assert.assertEquals(numbers.size(), numberIds.size()); + Set nonNumberIds = Whitebox.getInternalState(idSet, + "nonNumberIds"); + Assert.assertEquals(uuids.size() + strings.size(), + nonNumberIds.size()); + + for (long number : numbers) { + idSet.remove(IdGenerator.of(number)); + } + Assert.assertEquals(nonNumberIds.size(), idSet.size()); + + for (UUID uuid : uuids) { + idSet.remove(IdGenerator.of(uuid)); + } + for (String string : strings) { + idSet.remove(IdGenerator.of(string)); + } + Assert.assertTrue(idSet.isEmpty()); + + numbers.clear(); + uuids.clear(); + strings.clear(); + idSet.clear(); + } + } } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java index cfe8abbcf3..a8573565a0 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/Int2IntsMapTest.java @@ -19,7 +19,6 @@ package com.baidu.hugegraph.unit.core; -import java.util.Arrays; import java.util.HashSet; import java.util.Random; import java.util.Set; @@ -33,6 +32,9 @@ public class Int2IntsMapTest { + private static final int KEY_NUMBER = 5; + private static final int VALUE_NUMBER = 1000; + @Before public void setup() { // pass @@ -53,8 +55,8 @@ public void testInt2IntsMap() { int l = 255; int m = 270; Random random = new Random(); - for (int n = 0; n < 1000; n++) { - switch (random.nextInt() % 5) { + for (int n = 0; n < VALUE_NUMBER; n++) { + switch (random.nextInt() % KEY_NUMBER) { case 0: if (i < 200) { map.add(1, i++); @@ -83,7 +85,7 @@ public void testInt2IntsMap() { } } - int[][] results = new int[5][]; + int[][] results = new int[KEY_NUMBER][]; results[0] = new int[]{100,101,102,103,104,105,106,107,108,109, 110,111,112,113,114,115,116,117,118,119, 120,121,122,123,124,125,126,127,128,129, @@ -106,7 +108,7 @@ public void testInt2IntsMap() { 280,281,282,283,284,285,286,287,288,289, 290,291,292,293,294,295,296,297,298,299}; - for (int ii = 0; ii < 5; ii++) { + for (int ii = 0; ii < KEY_NUMBER; ii++) { int[] result = map.getValues(ii + 1); Assert.assertArrayEquals(results[ii], result); } @@ -117,17 +119,59 @@ public void testInt2IntsMapRandom() { Int2IntsMap map = new Int2IntsMap(); Random random = new Random(); - for (int i = 0; i < 1000; i++) { - map.add(Math.abs(random.nextInt()) % 5, i); + for (int i = 0; i < VALUE_NUMBER; i++) { + map.add(Math.abs(random.nextInt()) % KEY_NUMBER, i); + } + + Set all = new HashSet<>(); + for (int i = 0; i < KEY_NUMBER; i++) { + int[] values = map.getValues(i); + for (int j : values) { + all.add(j); + } + } + Assert.assertEquals(VALUE_NUMBER, all.size()); + } + + @Test + public void testInt2IntsMapContainsKey() { + Int2IntsMap map = new Int2IntsMap(); + + Random random = new Random(); + for (int i = 0; i < VALUE_NUMBER; i++) { + map.add(Math.abs(random.nextInt()) % KEY_NUMBER, i); } Set all = new HashSet<>(); + for (int i = 0; i < KEY_NUMBER; i++) { + int[] values = map.getValues(i); + for (int j : values) { + all.add(j); + } + } + Assert.assertEquals(VALUE_NUMBER, all.size()); for (int i = 0; i < 5; i++) { + Assert.assertTrue(map.containsKey(i)); + } + } + + @Test + public void testInt2IntsMapSize() { + Int2IntsMap map = new Int2IntsMap(); + + Random random = new Random(); + for (int i = 0; i < VALUE_NUMBER; i++) { + map.add(Math.abs(random.nextInt()) % KEY_NUMBER, i); + } + + Set all = new HashSet<>(); + for (int i = 0; i < KEY_NUMBER; i++) { int[] values = map.getValues(i); for (int j : values) { all.add(j); } } - Assert.assertEquals(1000, all.size()); + Assert.assertEquals(VALUE_NUMBER, all.size()); + Assert.assertEquals(KEY_NUMBER, map.size()); } } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/CollectionFactoryTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/CollectionFactoryTest.java new file mode 100644 index 0000000000..80aa061713 --- /dev/null +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/CollectionFactoryTest.java @@ -0,0 +1,387 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.baidu.hugegraph.unit.util; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.eclipse.collections.api.map.primitive.MutableIntObjectMap; +import org.eclipse.collections.impl.list.mutable.FastList; +import org.eclipse.collections.impl.map.mutable.UnifiedMap; +import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; +import org.eclipse.collections.impl.set.mutable.UnifiedSet; +import org.eclipse.collections.impl.set.mutable.primitive.LongHashSet; +import org.junit.Test; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.testutil.Assert; +import com.baidu.hugegraph.testutil.Whitebox; +import com.baidu.hugegraph.type.define.CollectionType; +import com.baidu.hugegraph.util.collection.CollectionFactory; +import com.baidu.hugegraph.util.collection.IdSet; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.objects.ObjectArrayList; +import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; + +public class CollectionFactoryTest { + + private static CollectionFactory factory; + + @Test + public void testCollectionFactoryConstructor() { + factory = new CollectionFactory(); + CollectionType type = Whitebox.getInternalState(factory, "type"); + Assert.assertEquals(CollectionType.EC, type); + + factory = new CollectionFactory(CollectionType.EC); + type = Whitebox.getInternalState(factory, "type"); + Assert.assertEquals(CollectionType.EC, type); + + factory = new CollectionFactory(CollectionType.FU); + type = Whitebox.getInternalState(factory, "type"); + Assert.assertEquals(CollectionType.FU, type); + + factory = new CollectionFactory(CollectionType.JCF); + type = Whitebox.getInternalState(factory, "type"); + Assert.assertEquals(CollectionType.JCF, type); + } + + @Test + public void testNewList() { + // With type + factory = new CollectionFactory(); + List list = factory.newList(); + Assert.assertInstanceOf(FastList.class, list); + + factory = new CollectionFactory(CollectionType.EC); + list = factory.newList(); + Assert.assertInstanceOf(FastList.class, list); + + factory = new CollectionFactory(CollectionType.FU); + list = factory.newList(); + Assert.assertInstanceOf(ObjectArrayList.class, list); + + factory = new CollectionFactory(CollectionType.JCF); + list = factory.newList(); + Assert.assertInstanceOf(ArrayList.class, list); + + // With initial capacity + int initialCapacity = 10; + + factory = new CollectionFactory(); + list = factory.newList(initialCapacity); + Assert.assertInstanceOf(FastList.class, list); + Object[] items = Whitebox.getInternalState(list, "items"); + Assert.assertEquals(initialCapacity, items.length); + + factory = new CollectionFactory(CollectionType.EC); + list = factory.newList(initialCapacity); + Assert.assertInstanceOf(FastList.class, list); + items = Whitebox.getInternalState(list, "items"); + Assert.assertEquals(initialCapacity, items.length); + + factory = new CollectionFactory(CollectionType.FU); + list = factory.newList(initialCapacity); + Assert.assertInstanceOf(ObjectArrayList.class, list); + items = Whitebox.getInternalState(list, "a"); + Assert.assertEquals(initialCapacity, items.length); + + factory = new CollectionFactory(CollectionType.JCF); + list = factory.newList(initialCapacity); + Assert.assertInstanceOf(ArrayList.class, list); + items = Whitebox.getInternalState(list, "elementData"); + Assert.assertEquals(initialCapacity, items.length); + + // With collection + Collection integers = ImmutableSet.of(1, 2, 3, 4); + + factory = new CollectionFactory(); + list = factory.newList(integers); + Assert.assertInstanceOf(FastList.class, list); + Assert.assertEquals(integers.size(), list.size()); + for (int integer : integers) { + Assert.assertTrue(list.contains(integer)); + } + + factory = new CollectionFactory(CollectionType.EC); + list = factory.newList(integers); + Assert.assertInstanceOf(FastList.class, list); + Assert.assertEquals(integers.size(), list.size()); + for (int integer : integers) { + Assert.assertTrue(list.contains(integer)); + } + + factory = new CollectionFactory(CollectionType.FU); + list = factory.newList(integers); + Assert.assertInstanceOf(ObjectArrayList.class, list); + Assert.assertEquals(integers.size(), list.size()); + for (int integer : integers) { + Assert.assertTrue(list.contains(integer)); + } + + factory = new CollectionFactory(CollectionType.JCF); + list = factory.newList(integers); + Assert.assertInstanceOf(ArrayList.class, list); + Assert.assertEquals(integers.size(), list.size()); + for (int integer : integers) { + Assert.assertTrue(list.contains(integer)); + } + } + + @Test + public void testNewSet() { + // With type + factory = new CollectionFactory(); + Set set = factory.newSet(); + Assert.assertInstanceOf(UnifiedSet.class, set); + + factory = new CollectionFactory(CollectionType.EC); + set = factory.newSet(); + Assert.assertInstanceOf(UnifiedSet.class, set); + + factory = new CollectionFactory(CollectionType.FU); + set = factory.newSet(); + Assert.assertInstanceOf(ObjectOpenHashSet.class, set); + + factory = new CollectionFactory(CollectionType.JCF); + set = factory.newSet(); + Assert.assertInstanceOf(HashSet.class, set); + + // With initial capacity + int initialCapacity = 10; + + factory = new CollectionFactory(); + set = factory.newSet(initialCapacity); + Assert.assertInstanceOf(UnifiedSet.class, set); + Object[] items = Whitebox.getInternalState(set, "table"); + // Initial size of UnifiedSet is (initialCapacity / 0.75) + Assert.assertEquals(16, items.length); + + factory = new CollectionFactory(CollectionType.EC); + set = factory.newSet(initialCapacity); + Assert.assertInstanceOf(UnifiedSet.class, set); + items = Whitebox.getInternalState(set, "table"); + // Initial size of UnifiedSet is (initialCapacity / 0.75) + Assert.assertEquals(16, items.length); + + factory = new CollectionFactory(CollectionType.FU); + set = factory.newSet(initialCapacity); + Assert.assertInstanceOf(ObjectOpenHashSet.class, set); + items = Whitebox.getInternalState(set, "key"); + Assert.assertEquals(17, items.length); + + factory = new CollectionFactory(CollectionType.JCF); + set = factory.newSet(initialCapacity); + Assert.assertInstanceOf(HashSet.class, set); + Map map = Whitebox.getInternalState(set, "map"); + Assert.assertInstanceOf(HashMap.class, map); + Assert.assertEquals(0, map.size()); + + // With collection + Collection integers = ImmutableSet.of(1, 2, 3, 4); + + factory = new CollectionFactory(); + set = factory.newSet(integers); + Assert.assertInstanceOf(UnifiedSet.class, set); + Assert.assertEquals(integers.size(), set.size()); + for (int integer : integers) { + Assert.assertTrue(set.contains(integer)); + } + + factory = new CollectionFactory(CollectionType.EC); + set = factory.newSet(integers); + Assert.assertInstanceOf(UnifiedSet.class, set); + Assert.assertEquals(integers.size(), set.size()); + for (int integer : integers) { + Assert.assertTrue(set.contains(integer)); + } + + factory = new CollectionFactory(CollectionType.FU); + set = factory.newSet(integers); + Assert.assertInstanceOf(ObjectOpenHashSet.class, set); + Assert.assertEquals(integers.size(), set.size()); + for (int integer : integers) { + Assert.assertTrue(set.contains(integer)); + } + + factory = new CollectionFactory(CollectionType.JCF); + set = factory.newSet(integers); + Assert.assertInstanceOf(HashSet.class, set); + Assert.assertEquals(integers.size(), set.size()); + for (int integer : integers) { + Assert.assertTrue(set.contains(integer)); + } + } + + @Test + public void testNewMap() { + // With type + factory = new CollectionFactory(); + Map map = factory.newMap(); + Assert.assertInstanceOf(UnifiedMap.class, map); + + factory = new CollectionFactory(CollectionType.EC); + map = factory.newMap(); + Assert.assertInstanceOf(UnifiedMap.class, map); + + factory = new CollectionFactory(CollectionType.FU); + map = factory.newMap(); + Assert.assertInstanceOf(Object2ObjectOpenHashMap.class, map); + + factory = new CollectionFactory(CollectionType.JCF); + map = factory.newMap(); + Assert.assertInstanceOf(HashMap.class, map); + + // With initial capacity + int initialCapacity = 10; + + factory = new CollectionFactory(); + map = factory.newMap(initialCapacity); + Assert.assertInstanceOf(UnifiedMap.class, map); + Object[] items = Whitebox.getInternalState(map, "table"); + // Initial size of UnifiedSet is (initialCapacity / 0.75) * 2 + Assert.assertEquals(32, items.length); + + factory = new CollectionFactory(CollectionType.EC); + map = factory.newMap(initialCapacity); + Assert.assertInstanceOf(UnifiedMap.class, map); + items = Whitebox.getInternalState(map, "table"); + // Initial size of UnifiedSet is (initialCapacity / 0.75) * 2 + Assert.assertEquals(32, items.length); + + factory = new CollectionFactory(CollectionType.FU); + map = factory.newMap(initialCapacity); + Assert.assertInstanceOf(Object2ObjectOpenHashMap.class, map); + items = Whitebox.getInternalState(map, "key"); + Assert.assertEquals(17, items.length); + items = Whitebox.getInternalState(map, "value"); + Assert.assertEquals(17, items.length); + + factory = new CollectionFactory(CollectionType.JCF); + map = factory.newMap(initialCapacity); + Assert.assertInstanceOf(HashMap.class, map); + items = Whitebox.getInternalState(map, "table"); + Assert.assertNull(items); + + // With collection + Map keyValues = ImmutableMap.of(1, "A", 2, "B", + 3, "C", 4, "D"); + + factory = new CollectionFactory(); + map = factory.newMap(keyValues); + Assert.assertInstanceOf(UnifiedMap.class, map); + Assert.assertEquals(keyValues.size(), map.size()); + for (Map.Entry entry : keyValues.entrySet()) { + Assert.assertEquals(entry.getValue(), map.get(entry.getKey())); + } + + factory = new CollectionFactory(CollectionType.EC); + map = factory.newMap(keyValues); + Assert.assertInstanceOf(UnifiedMap.class, map); + Assert.assertEquals(keyValues.size(), map.size()); + for (Map.Entry entry : keyValues.entrySet()) { + Assert.assertEquals(entry.getValue(), map.get(entry.getKey())); + } + + factory = new CollectionFactory(CollectionType.FU); + map = factory.newMap(keyValues); + Assert.assertInstanceOf(Object2ObjectOpenHashMap.class, map); + Assert.assertEquals(keyValues.size(), map.size()); + for (Map.Entry entry : keyValues.entrySet()) { + Assert.assertEquals(entry.getValue(), map.get(entry.getKey())); + } + + factory = new CollectionFactory(CollectionType.JCF); + map = factory.newMap(keyValues); + Assert.assertInstanceOf(HashMap.class, map); + Assert.assertEquals(keyValues.size(), map.size()); + for (Map.Entry entry : keyValues.entrySet()) { + Assert.assertEquals(entry.getValue(), map.get(entry.getKey())); + } + } + + @Test + public void testIntObjectMap() { + MutableIntObjectMap map = CollectionFactory.newIntObjectMap(); + Assert.assertInstanceOf(IntObjectHashMap.class, map); + + map = CollectionFactory.newIntObjectMap(10); + Assert.assertInstanceOf(IntObjectHashMap.class, map); + int[] keys = Whitebox.getInternalState(map, "keys"); + Assert.assertEquals(32, keys.length); + Object[] values = Whitebox.getInternalState(map, "values"); + Assert.assertEquals(32, values.length); + + map = CollectionFactory.newIntObjectMap(1, "A", 2, "B"); + map = CollectionFactory.newIntObjectMap(map); + Assert.assertInstanceOf(IntObjectHashMap.class, map); + Assert.assertEquals(2, map.size()); + Assert.assertEquals("A", map.get(1)); + Assert.assertEquals("B", map.get(2)); + } + + @Test + public void testIdSet() { + factory = new CollectionFactory(CollectionType.EC); + IdSet idSet = factory.newIdSet(); + Set ids = Whitebox.getInternalState(idSet, "nonNumberIds"); + Assert.assertInstanceOf(UnifiedSet.class, ids); + Assert.assertInstanceOf(LongHashSet.class, + Whitebox.getInternalState(idSet, "numberIds")); + + factory = new CollectionFactory(CollectionType.FU); + idSet = factory.newIdSet(); + ids = Whitebox.getInternalState(idSet, "nonNumberIds"); + Assert.assertInstanceOf(ObjectOpenHashSet.class, ids); + Assert.assertInstanceOf(LongHashSet.class, + Whitebox.getInternalState(idSet, "numberIds")); + + factory = new CollectionFactory(CollectionType.JCF); + idSet = factory.newIdSet(); + ids = Whitebox.getInternalState(idSet, "nonNumberIds"); + Assert.assertInstanceOf(HashSet.class, ids); + Assert.assertInstanceOf(LongHashSet.class, + Whitebox.getInternalState(idSet, "numberIds")); + + idSet = CollectionFactory.newIdSet(CollectionType.EC); + ids = Whitebox.getInternalState(idSet, "nonNumberIds"); + Assert.assertInstanceOf(UnifiedSet.class, ids); + Assert.assertInstanceOf(LongHashSet.class, + Whitebox.getInternalState(idSet, "numberIds")); + + idSet = CollectionFactory.newIdSet(CollectionType.FU); + ids = Whitebox.getInternalState(idSet, "nonNumberIds"); + Assert.assertInstanceOf(ObjectOpenHashSet.class, ids); + Assert.assertInstanceOf(LongHashSet.class, + Whitebox.getInternalState(idSet, "numberIds")); + + idSet = CollectionFactory.newIdSet(CollectionType.JCF); + ids = Whitebox.getInternalState(idSet, "nonNumberIds"); + Assert.assertInstanceOf(HashSet.class, ids); + Assert.assertInstanceOf(LongHashSet.class, + Whitebox.getInternalState(idSet, "numberIds")); + } +}