Skip to content

Commit

Permalink
feat(api&core): in oltp apis, add statistics info and support full in…
Browse files Browse the repository at this point in the history 
…fo about vertices and edges (#2262)

* chore: improve gitignore file

* feat: add ApiMeasure to collect runtime data

ApiMeasure will count the number of vertices and edges traversed at runtime, and the time the api takes to execute

* feat: Add ApiMeasure to JsonSerializer and Modify the Serializer interface

* JsonSerializer: return measure information in api response

* Serializer: fit the feature that returns complete information about vertices and edges

* refactor: format code based on hugegraph-style.xml

* feat: Add statistics information in all oltp restful apis response and Support full information about vertices and edges

Statistics information:

* add vertexIterCounter and edgeIterCounter in HugeTraverser.java to track traversed vertices and edges at run time

* modify all oltp restful apis to add statistics information in response

Full information about vertices and edges:

* add 'with_vertex' and 'with_edge' parameter option in apis

* modify oltp apis to support vertex and edge information in api response

* add EdgeRecord in HugeTraverser.java to record edges at run time and generate the edge information returned in api response

* modify Path and PathSet in HugeTraverser.java to support full edge information storage

* modify all traversers to support track of edge information at run time

* fix: numeric cast

* fix: Jaccard Similarity api test

* fix: adjust the code style and naming convention

* Empty commit

* Empty commit

* fix:
1. change System.currentTimeMillis() to System.nanoTime();
2. modify addCount()

* fix: rollback change in .gitignore

* fix: rollback ServerOptions.java code style

* fix: rollback API.java code style and add exception in else branch

* fix: fix code style

* fix: name style & code style
* rename edgeRecord to edgeResults
* fix Request class code style in SameNeighborsAPI.java
  • Loading branch information
@DanGuge
DanGuge authored Aug 19, 2023
1 parent b02c2bd commit d12f573
Show file tree
Hide file tree
Showing 44 changed files with 2,268 additions and 1,222 deletions.
89 changes: 67 additions & 22 deletions hugegraph-api/src/main/java/org/apache/hugegraph/api/API.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,41 +22,39 @@
import java.util.concurrent.Callable;
import java.util.function.Consumer;

import jakarta.ws.rs.ForbiddenException;
import jakarta.ws.rs.NotFoundException;
import jakarta.ws.rs.NotSupportedException;
import jakarta.ws.rs.core.MediaType;

import org.apache.commons.lang.mutable.MutableLong;
import org.apache.hugegraph.HugeException;
import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.core.GraphManager;
import org.apache.hugegraph.define.Checkable;
import org.apache.hugegraph.metrics.MetricsUtil;
import org.slf4j.Logger;

import org.apache.hugegraph.HugeException;
import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.InsertionOrderUtil;
import org.apache.hugegraph.util.JsonUtil;
import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;

import com.codahale.metrics.Meter;
import com.google.common.collect.ImmutableMap;

public class API {
import jakarta.ws.rs.ForbiddenException;
import jakarta.ws.rs.NotFoundException;
import jakarta.ws.rs.NotSupportedException;
import jakarta.ws.rs.core.MediaType;

protected static final Logger LOG = Log.logger(API.class);
public class API {

public static final String CHARSET = "UTF-8";

public static final String TEXT_PLAIN = MediaType.TEXT_PLAIN;
public static final String APPLICATION_JSON = MediaType.APPLICATION_JSON;
public static final String APPLICATION_JSON_WITH_CHARSET =
APPLICATION_JSON + ";charset=" + CHARSET;
public static final String JSON = MediaType.APPLICATION_JSON_TYPE
.getSubtype();

public static final String ACTION_APPEND = "append";
public static final String ACTION_ELIMINATE = "eliminate";
public static final String ACTION_CLEAR = "clear";

protected static final Logger LOG = Log.logger(API.class);
private static final Meter SUCCEED_METER =
MetricsUtil.registerMeter(API.class, "commit-succeed");
private static final Meter ILLEGAL_ARG_ERROR_METER =
Expand All @@ -69,8 +67,7 @@ public class API {
public static HugeGraph graph(GraphManager manager, String graph) {
HugeGraph g = manager.graph(graph);
if (g == null) {
throw new NotFoundException(String.format(
"Graph '%s' does not exist", graph));
throw new NotFoundException(String.format("Graph '%s' does not exist", graph));
}
return g;
}
Expand Down Expand Up @@ -140,8 +137,7 @@ protected static void checkUpdatingBody(Checkable body) {
body.checkUpdate();
}

protected static void checkCreatingBody(
Collection<? extends Checkable> bodies) {
protected static void checkCreatingBody(Collection<? extends Checkable> bodies) {
E.checkArgumentNotNull(bodies, "The request body can't be empty");
for (Checkable body : bodies) {
E.checkArgument(body != null,
Expand All @@ -150,8 +146,7 @@ protected static void checkCreatingBody(
}
}

protected static void checkUpdatingBody(
Collection<? extends Checkable> bodies) {
protected static void checkUpdatingBody(Collection<? extends Checkable> bodies) {
E.checkArgumentNotNull(bodies, "The request body can't be empty");
for (Checkable body : bodies) {
E.checkArgumentNotNull(body,
Expand Down Expand Up @@ -186,8 +181,58 @@ public static boolean checkAndParseAction(String action) {
} else if (action.equals(ACTION_ELIMINATE)) {
return false;
} else {
throw new NotSupportedException(
String.format("Not support action '%s'", action));
throw new NotSupportedException(String.format("Not support action '%s'", action));
}
}

public static class ApiMeasurer {

public static final String EDGE_ITER = "edge_iterations";
public static final String VERTICE_ITER = "vertice_iterations";
public static final String COST = "cost(ns)";
private final long timeStart;
private final Map<String, Object> measures;

public ApiMeasurer() {
this.timeStart = System.nanoTime();
this.measures = InsertionOrderUtil.newMap();
}

public Map<String, Object> measures() {
measures.put(COST, System.nanoTime() - timeStart);
return measures;
}

public void put(String key, String value) {
this.measures.put(key, value);
}

public void put(String key, long value) {
this.measures.put(key, value);
}

public void put(String key, int value) {
this.measures.put(key, value);
}

protected void addCount(String key, long value) {
Object current = measures.get(key);
if (current == null) {
measures.put(key, new MutableLong(value));
} else if (current instanceof MutableLong) {
((MutableLong) measures.computeIfAbsent(key, MutableLong::new)).add(value);
} else if (current instanceof Long) {
Long currentLong = (Long) current;
measures.put(key, new MutableLong(currentLong + value));
} else {
throw new NotSupportedException("addCount() method's 'value' datatype must be " +
"Long or MutableLong");
}
}

public void addIterCount(long verticeIters, long edgeIters) {
this.addCount(EDGE_ITER, edgeIters);
this.addCount(VERTICE_ITER, verticeIters);
}
}
}
73 changes: 55 additions & 18 deletions hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/AllShortestPathsAPI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,10 @@
import static org.apache.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY;
import static org.apache.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;

import java.util.HashSet;
import java.util.Iterator;
import java.util.List;

import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.inject.Singleton;
import jakarta.ws.rs.DefaultValue;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.core.Context;

import org.slf4j.Logger;
import java.util.Set;

import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.api.API;
Expand All @@ -44,9 +35,22 @@
import org.apache.hugegraph.traversal.algorithm.ShortestPathTraverser;
import org.apache.hugegraph.type.define.Directions;
import org.apache.hugegraph.util.Log;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.slf4j.Logger;

import com.codahale.metrics.annotation.Timed;
import com.google.common.collect.ImmutableList;

import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.inject.Singleton;
import jakarta.ws.rs.DefaultValue;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.core.Context;

@Path("graphs/{graph}/traversers/allshortestpaths")
@Singleton
@Tag(name = "AllShortestPathsAPI")
Expand All @@ -68,13 +72,20 @@ public String get(@Context GraphManager manager,
@DefaultValue(DEFAULT_MAX_DEGREE) long maxDegree,
@QueryParam("skip_degree")
@DefaultValue("0") long skipDegree,
@QueryParam("with_vertex")
@DefaultValue("false") boolean withVertex,
@QueryParam("with_edge")
@DefaultValue("false") boolean withEdge,
@QueryParam("capacity")
@DefaultValue(DEFAULT_CAPACITY) long capacity) {
LOG.debug("Graph [{}] get shortest path from '{}', to '{}' with " +
"direction {}, edge label {}, max depth '{}', " +
"max degree '{}', skipped degree '{}' and capacity '{}'",
"max degree '{}', skipped degree '{}', capacity '{}', " +
"with_vertex '{}' and with_edge '{}'",
graph, source, target, direction, edgeLabel, depth,
maxDegree, skipDegree, capacity);
maxDegree, skipDegree, capacity, withVertex, withEdge);

ApiMeasurer measure = new ApiMeasurer();

Id sourceId = VertexAPI.checkAndParseVertexId(source);
Id targetId = VertexAPI.checkAndParseVertexId(target);
Expand All @@ -85,9 +96,35 @@ public String get(@Context GraphManager manager,
ShortestPathTraverser traverser = new ShortestPathTraverser(g);
List<String> edgeLabels = edgeLabel == null ? ImmutableList.of() :
ImmutableList.of(edgeLabel);
HugeTraverser.PathSet paths = traverser.allShortestPaths(
sourceId, targetId, dir, edgeLabels,
depth, maxDegree, skipDegree, capacity);
return manager.serializer(g).writePaths("paths", paths, false);
HugeTraverser.PathSet paths = traverser.allShortestPaths(sourceId, targetId, dir,
edgeLabels, depth, maxDegree,
skipDegree, capacity);

measure.addIterCount(traverser.vertexIterCounter.get(),
traverser.edgeIterCounter.get());

Iterator<?> iterVertex;
Set<Id> vertexIds = new HashSet<>();
for (HugeTraverser.Path path : paths) {
vertexIds.addAll(path.vertices());
}
if (withVertex && !vertexIds.isEmpty()) {
iterVertex = g.vertices(vertexIds.toArray());
measure.addIterCount(vertexIds.size(), 0L);
} else {
iterVertex = vertexIds.iterator();
}

Iterator<?> iterEdge;
Set<Edge> edges = paths.getEdges();
if (withEdge && !edges.isEmpty()) {
iterEdge = edges.iterator();
} else {
iterEdge = HugeTraverser.EdgeRecord.getEdgeIds(edges).iterator();
}

return manager.serializer(g, measure.measures())
.writePaths("paths", paths, false,
iterVertex, iterEdge);
}
}
30 changes: 17 additions & 13 deletions hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/CrosspointsAPI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,6 @@
import static org.apache.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
import static org.apache.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT;

import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.inject.Singleton;
import jakarta.ws.rs.DefaultValue;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.core.Context;

import org.slf4j.Logger;

import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.api.API;
import org.apache.hugegraph.api.graph.EdgeAPI;
Expand All @@ -43,8 +31,20 @@
import org.apache.hugegraph.traversal.algorithm.PathsTraverser;
import org.apache.hugegraph.type.define.Directions;
import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;

import com.codahale.metrics.annotation.Timed;

import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.inject.Singleton;
import jakarta.ws.rs.DefaultValue;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.core.Context;

@Path("graphs/{graph}/traversers/crosspoints")
@Singleton
@Tag(name = "CrosspointsAPI")
Expand Down Expand Up @@ -74,6 +74,7 @@ public String get(@Context GraphManager manager,
graph, source, target, direction, edgeLabel,
depth, maxDegree, capacity, limit);

ApiMeasurer measure = new ApiMeasurer();
Id sourceId = VertexAPI.checkAndParseVertexId(source);
Id targetId = VertexAPI.checkAndParseVertexId(target);
Directions dir = Directions.convert(EdgeAPI.parseDirection(direction));
Expand All @@ -84,6 +85,9 @@ public String get(@Context GraphManager manager,
dir, edgeLabel, depth,
maxDegree, capacity,
limit);
return manager.serializer(g).writePaths("crosspoints", paths, true);
measure.addIterCount(traverser.vertexIterCounter.get(),
traverser.edgeIterCounter.get());
return manager.serializer(g, measure.measures())
.writePaths("crosspoints", paths, true);
}
}
Loading

0 comments on commit d12f573

Please sign in to comment.