Collection optimize for OLTP algorithm and HugeElement (apache#1409)

* add Eclipse Collection dependency * use primitive map for properties of HugeElement * from Map<Id, HugeProperty<?>> to MutableIntObjectMap<HugeProperty<?>> * allow config oltp collections(JCF or EC) * add fastutil choice for collection * change Set<HugeEdge> of HugeVertex to List<HugeEdge> * use IntObjectMap and IntIntMap to improve shortest path * improve paths api by remove redundant sourcesAll and targetsAll * improve paths api performance by using IntObjectHashMap<IntHashSet> * extract shortestpath and paths record to hold traversed paths * implement Int2IntsMap for array paths record * extract Record and Records for paths and shortest path * adapt primitive collection for kout/kneighbor * add objectintmapping unit test * add UT for CollectionFactory and IdSet Change-Id: I501a18a70054fad3c40bd6b9e852a82a5c185949
zhoney · Jun 9, 2021 · 23e3c31 · 23e3c31
1 parent 83f6612
commit 23e3c31
Show file tree

Hide file tree

Showing 75 changed files with 3,663 additions and 518 deletions.
diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/AllShortestPathsAPI.java
@@ -19,11 +19,11 @@
 
 package com.baidu.hugegraph.api.traversers;
 
-import java.util.List;
-
 import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY;
 import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
 
+import java.util.List;
+
 import javax.inject.Singleton;
 import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;

diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KneighborAPI.java
@@ -22,10 +22,8 @@
 import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
 import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_ELEMENTS_LIMIT;
 
-import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Set;
 
 import javax.inject.Singleton;
@@ -50,6 +48,7 @@
 import com.baidu.hugegraph.core.GraphManager;
 import com.baidu.hugegraph.server.RestServer;
 import com.baidu.hugegraph.structure.HugeVertex;
+import com.baidu.hugegraph.traversal.algorithm.records.KneighborRecords;
 import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 import com.baidu.hugegraph.traversal.algorithm.KneighborTraverser;
@@ -125,32 +124,27 @@ public String post(@Context GraphManager manager,
 
         EdgeStep step = step(g, request.step);
 
-        Set<HugeTraverser.Node> results;
+        KneighborRecords results;
         try (KneighborTraverser traverser = new KneighborTraverser(g)) {
             results = traverser.customizedKneighbor(sourceId, step,
                                                     request.maxDepth,
                                                     request.limit);
         }
 
-        Set<Id> neighbors = new HashSet<>();
-        for (HugeTraverser.Node node : results) {
-            neighbors.add(node.id());
-        }
+        Set<Id> neighbors = results.ids(request.limit);
 
-        List<HugeTraverser.Path> paths = new ArrayList<>();
+        HugeTraverser.PathSet paths = new HugeTraverser.PathSet();
         if (request.withPath) {
-            for (HugeTraverser.Node node : results) {
-                paths.add(new HugeTraverser.Path(node.path()));
-            }
+            paths.addAll(results.paths(request.limit));
         }
         Iterator<Vertex> iter = QueryResults.emptyIterator();
         if (request.withVertex) {
             Set<Id> ids = new HashSet<>();
-            for (HugeTraverser.Node node : results) {
-                ids.add(node.id());
-            }
-            for (HugeTraverser.Path p : paths) {
-                ids.addAll(p.vertices());
+            ids.addAll(results.ids(request.limit));
+            if (request.withPath) {
+                for (HugeTraverser.Path p : paths) {
+                    ids.addAll(p.vertices());
+                }
             }
             if (!ids.isEmpty()) {
                 iter = g.vertices(ids.toArray());

diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KoutAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/KoutAPI.java
@@ -23,10 +23,8 @@
 import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
 import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_ELEMENTS_LIMIT;
 
-import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Set;
 
 import javax.inject.Singleton;
@@ -51,9 +49,9 @@
 import com.baidu.hugegraph.core.GraphManager;
 import com.baidu.hugegraph.server.RestServer;
 import com.baidu.hugegraph.structure.HugeVertex;
+import com.baidu.hugegraph.traversal.algorithm.records.KoutRecords;
 import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
-import com.baidu.hugegraph.traversal.algorithm.HugeTraverser.Node;
 import com.baidu.hugegraph.traversal.algorithm.KoutTraverser;
 import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.util.E;
@@ -133,7 +131,7 @@ public String post(@Context GraphManager manager,
 
         EdgeStep step = step(g, request.step);
 
-        Set<HugeTraverser.Node> results;
+        KoutRecords results;
         try (KoutTraverser traverser = new KoutTraverser(g)) {
             results = traverser.customizedKout(sourceId, step,
                                                request.maxDepth,
@@ -142,25 +140,20 @@ public String post(@Context GraphManager manager,
                                                request.limit);
         }
 
-        Set<Id> neighbors = new HashSet<>();
-        for (HugeTraverser.Node node : results) {
-            neighbors.add(node.id());
-        }
+        Set<Id> neighbors = results.ids(request.limit);
 
-        List<HugeTraverser.Path> paths = new ArrayList<>();
+        HugeTraverser.PathSet paths = new HugeTraverser.PathSet();
         if (request.withPath) {
-            for (HugeTraverser.Node node : results) {
-                paths.add(new HugeTraverser.Path(node.path()));
-            }
+            paths.addAll(results.paths(request.limit));
         }
         Iterator<Vertex> iter = QueryResults.emptyIterator();
         if (request.withVertex) {
             Set<Id> ids = new HashSet<>();
-            for (Node node : results) {
-                ids.add(node.id());
-            }
-            for (HugeTraverser.Path p : paths) {
-                ids.addAll(p.vertices());
+            ids.addAll(results.ids(request.limit));
+            if (request.withPath) {
+                for (HugeTraverser.Path p : paths) {
+                    ids.addAll(p.vertices());
+                }
             }
             if (!ids.isEmpty()) {
                 iter = g.vertices(ids.toArray());

diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PathsAPI.java
@@ -19,15 +19,15 @@
 
 package com.baidu.hugegraph.api.traversers;
 
+import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY;
+import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
+import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT;
+
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
 
-import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY;
-import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
-import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT;
-
 import javax.inject.Singleton;
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DefaultValue;
@@ -50,9 +50,9 @@
 import com.baidu.hugegraph.core.GraphManager;
 import com.baidu.hugegraph.server.RestServer;
 import com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser;
-import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 import com.baidu.hugegraph.traversal.algorithm.PathsTraverser;
+import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep;
 import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.util.E;
 import com.baidu.hugegraph.util.Log;

diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/PersonalRankAPI.java
@@ -19,6 +19,11 @@
 
 package com.baidu.hugegraph.api.traversers;
 
+import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_LIMIT;
+import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
+import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEPTH;
+import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.NO_LIMIT;
+
 import java.util.Map;
 
 import javax.inject.Singleton;
@@ -43,11 +48,6 @@
 import com.codahale.metrics.annotation.Timed;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
-import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
-import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_LIMIT;
-import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEPTH;
-import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.NO_LIMIT;
-
 @Path("graphs/{graph}/traversers/personalrank")
 @Singleton
 public class PersonalRankAPI extends API {

diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/ShortestPathAPI.java
@@ -19,11 +19,11 @@
 
 package com.baidu.hugegraph.api.traversers;
 
-import java.util.List;
-
 import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY;
 import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
 
+import java.util.List;
+
 import javax.inject.Singleton;
 import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
@@ -76,7 +76,6 @@ public String get(@Context GraphManager manager,
                   "max degree '{}', skipped maxDegree '{}' and capacity '{}'",
                   graph, source, target, direction, edgeLabel, depth,
                   maxDegree, skipDegree, capacity);
-
         Id sourceId = VertexAPI.checkAndParseVertexId(source);
         Id targetId = VertexAPI.checkAndParseVertexId(target);
         Directions dir = Directions.convert(EdgeAPI.parseDirection(direction));

diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/TraverserAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/TraverserAPI.java
@@ -19,6 +19,8 @@
 
 package com.baidu.hugegraph.api.traversers;
 
+import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
+
 import java.util.List;
 import java.util.Map;
 
@@ -29,8 +31,6 @@
 import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
-import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
-
 public class TraverserAPI extends API {
 
     protected static EdgeStep step(HugeGraph graph, Step step) {

diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java
@@ -65,6 +65,7 @@
 import com.baidu.hugegraph.backend.store.raft.RaftGroupManager;
 import com.baidu.hugegraph.config.ConfigOption;
 import com.baidu.hugegraph.config.HugeConfig;
+import com.baidu.hugegraph.config.TypedOption;
 import com.baidu.hugegraph.exception.NotSupportException;
 import com.baidu.hugegraph.iterator.FilterIterator;
 import com.baidu.hugegraph.rpc.RpcServiceConfig4Client;
@@ -552,7 +553,7 @@ public long now() {
     }
 
     @Override
-    public <V> V option(ConfigOption<V> option) {
+    public <K, V> V option(TypedOption<K, V> option) {
         this.verifyAnyPermission();
         return this.hugegraph.option(option);
     }

diff --git a/hugegraph-core/pom.xml b/hugegraph-core/pom.xml
@@ -157,6 +157,23 @@
             <artifactId>commons-compress</artifactId>
             <version>1.20</version>
         </dependency>
+        <dependency>
+            <groupId>org.eclipse.collections</groupId>
+            <artifactId>eclipse-collections-api</artifactId>
+            <version>10.4.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.collections</groupId>
+            <artifactId>eclipse-collections</artifactId>
+            <version>10.4.0</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/it.unimi.dsi/fastutil -->
+        <dependency>
+            <groupId>it.unimi.dsi</groupId>
+            <artifactId>fastutil</artifactId>
+            <version>8.1.0</version>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java
@@ -38,6 +38,7 @@
 import com.baidu.hugegraph.backend.store.BackendStoreSystemInfo;
 import com.baidu.hugegraph.backend.store.raft.RaftGroupManager;
 import com.baidu.hugegraph.config.ConfigOption;
+import com.baidu.hugegraph.config.TypedOption;
 import com.baidu.hugegraph.rpc.RpcServiceConfig4Client;
 import com.baidu.hugegraph.rpc.RpcServiceConfig4Server;
 import com.baidu.hugegraph.schema.EdgeLabel;
@@ -170,7 +171,7 @@ public interface HugeGraph extends Graph {
 
     public long now();
 
-    public <V> V option(ConfigOption<V> option);
+    public <K, V> V option(TypedOption<K, V> option);
 
     public void registerRpcServices(RpcServiceConfig4Server serverConfig,
                                     RpcServiceConfig4Client clientConfig);

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java
@@ -64,13 +64,14 @@
 import com.baidu.hugegraph.backend.store.ram.RamTable;
 import com.baidu.hugegraph.backend.tx.GraphTransaction;
 import com.baidu.hugegraph.backend.tx.SchemaTransaction;
-import com.baidu.hugegraph.config.ConfigOption;
 import com.baidu.hugegraph.config.CoreOptions;
 import com.baidu.hugegraph.config.HugeConfig;
+import com.baidu.hugegraph.config.TypedOption;
 import com.baidu.hugegraph.event.EventHub;
 import com.baidu.hugegraph.event.EventListener;
 import com.baidu.hugegraph.exception.NotAllowException;
 import com.baidu.hugegraph.io.HugeGraphIoRegistry;
+import com.baidu.hugegraph.perf.PerfUtil.Watched;
 import com.baidu.hugegraph.rpc.RpcServiceConfig4Client;
 import com.baidu.hugegraph.rpc.RpcServiceConfig4Server;
 import com.baidu.hugegraph.schema.EdgeLabel;
@@ -115,14 +116,15 @@ public class StandardHugeGraph implements HugeGraph {
            StandardHugeGraph.SysTransaction.class
     };
 
-    public static final Set<ConfigOption<?>> ALLOWED_CONFIGS = ImmutableSet.of(
+    public static final Set<TypedOption<?, ?>> ALLOWED_CONFIGS = ImmutableSet.of(
            CoreOptions.TASK_WAIT_TIMEOUT,
            CoreOptions.TASK_SYNC_DELETION,
            CoreOptions.TASK_TTL_DELETE_BATCH,
            CoreOptions.TASK_INPUT_SIZE_LIMIT,
            CoreOptions.TASK_RESULT_SIZE_LIMIT,
            CoreOptions.OLTP_CONCURRENT_THREADS,
            CoreOptions.OLTP_CONCURRENT_DEPTH,
+           CoreOptions.OLTP_COLLECTION_TYPE,
            CoreOptions.VERTEX_DEFAULT_LABEL,
            CoreOptions.VERTEX_ENCODE_PK_NUMBER
     );
@@ -453,6 +455,7 @@ private BackendStore loadSystemStore() {
         return this.storeProvider.loadSystemStore(name);
     }
 
+    @Watched
     private SchemaTransaction schemaTransaction() {
         this.checkGraphNotClosed();
         /*
@@ -472,6 +475,7 @@ private SysTransaction systemTransaction() {
         return this.tx.systemTransaction();
     }
 
+    @Watched
     private GraphTransaction graphTransaction() {
         this.checkGraphNotClosed();
         /*
@@ -656,6 +660,7 @@ public Iterator<Edge> edges(Object... objects) {
     }
 
     @Override
+    @Watched
     public Iterator<Edge> edges(Query query) {
         return this.graphTransaction().queryEdges(query);
     }
@@ -727,6 +732,7 @@ public Collection<VertexLabel> vertexLabels() {
     }
 
     @Override
+    @Watched
     public VertexLabel vertexLabelOrNone(Id id) {
         VertexLabel vl = this.schemaTransaction().getVertexLabel(id);
         if (vl == null) {
@@ -972,7 +978,7 @@ public long now() {
     }
 
     @Override
-    public <V> V option(ConfigOption<V> option) {
+    public <K, V> V option(TypedOption<K, V> option) {
         HugeConfig config = this.configuration();
         if (!ALLOWED_CONFIGS.contains(option)) {
             throw new NotAllowException("Not allowed to access config: %s",

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java
@@ -44,6 +44,7 @@
 import com.baidu.hugegraph.exception.NotSupportException;
 import com.baidu.hugegraph.iterator.ExtendableIterator;
 import com.baidu.hugegraph.iterator.ListIterator;
+import com.baidu.hugegraph.perf.PerfUtil.Watched;
 import com.baidu.hugegraph.schema.IndexLabel;
 import com.baidu.hugegraph.structure.HugeEdge;
 import com.baidu.hugegraph.structure.HugeVertex;
@@ -264,6 +265,7 @@ private Iterator<HugeVertex> queryVerticesByIds(IdQuery query) {
     }
 
     @Override
+    @Watched
     protected final Iterator<HugeEdge> queryEdgesFromBackend(Query query) {
         RamTable ramtable = this.params().ramtable();
         if (ramtable != null && ramtable.matched(query)) {

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java
@@ -33,6 +33,7 @@
 import com.baidu.hugegraph.config.CoreOptions;
 import com.baidu.hugegraph.event.EventHub;
 import com.baidu.hugegraph.event.EventListener;
+import com.baidu.hugegraph.perf.PerfUtil.Watched;
 import com.baidu.hugegraph.schema.SchemaElement;
 import com.baidu.hugegraph.type.HugeType;
 import com.baidu.hugegraph.util.E;
@@ -348,6 +349,7 @@ public void updateIfNeeded(V schema) {
             }
         }
 
+        @Watched
         public V get(HugeType type, Id id) {
             assert id.number();
             long longId = id.asLong();