Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(api&core): in oltp apis, add statistics info and support full info about vertices and edges #2262

Merged
merged 16 commits into from
Aug 19, 2023
Merged

feat(api&core): in oltp apis, add statistics info and support full info about vertices and edges #2262

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
8a8f9c7
chore: improve gitignore file
DanGuge Jul 28, 2023
8f3f9db
feat: add ApiMeasure to collect runtime data
DanGuge Jul 28, 2023
eef9da4
feat: Add ApiMeasure to JsonSerializer and Modify the Serializer inte…
DanGuge Jul 28, 2023
023acf6
refactor: format code based on hugegraph-style.xml
DanGuge Jul 28, 2023
fc94aa3
feat: Add statistics information in all oltp restful apis response an…
DanGuge Jul 28, 2023
c6ab9f4
fix: numeric cast
DanGuge Jul 28, 2023
c7825a6
fix: Jaccard Similarity api test
DanGuge Jul 28, 2023
e671963
fix: adjust the code style and naming convention
DanGuge Jul 31, 2023
1e33cda
Empty commit
DanGuge Jul 31, 2023
262ae05
Empty commit
DanGuge Jul 31, 2023
793ead8
fix:
DanGuge Aug 11, 2023
5bb446a
fix: rollback change in .gitignore
DanGuge Aug 11, 2023
cea5db2
fix: rollback ServerOptions.java code style
DanGuge Aug 12, 2023
4cc5f46
fix: rollback API.java code style and add exception in else branch
DanGuge Aug 12, 2023
92ba7c8
fix: fix code style
DanGuge Aug 12, 2023
559bff1
fix: name style & code style
DanGuge Aug 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ upload-files/
demo*
gen-java
*.class
swagger-ui-*
hugegraph-dist/dist.sh
Copy link
Member

@imbajin imbajin Aug 11, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, but we need fix/remove it in the dist-script rather than exclude them in git (will be fixed it in another PR) @VGalaxies

yzBIrgUZmN

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, fixed and rollback change


### STS ###
.apt_generated
Expand Down
81 changes: 64 additions & 17 deletions hugegraph-api/src/main/java/org/apache/hugegraph/api/API.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,24 +22,26 @@
import java.util.concurrent.Callable;
import java.util.function.Consumer;

import jakarta.ws.rs.ForbiddenException;
import jakarta.ws.rs.NotFoundException;
import jakarta.ws.rs.NotSupportedException;
import jakarta.ws.rs.core.MediaType;

import org.apache.commons.lang.mutable.MutableLong;
import org.apache.hugegraph.HugeException;
import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.core.GraphManager;
import org.apache.hugegraph.define.Checkable;
import org.apache.hugegraph.metrics.MetricsUtil;
import org.slf4j.Logger;

import org.apache.hugegraph.HugeException;
import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.InsertionOrderUtil;
import org.apache.hugegraph.util.JsonUtil;
import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;

import com.codahale.metrics.Meter;
import com.google.common.collect.ImmutableMap;

import jakarta.ws.rs.ForbiddenException;
import jakarta.ws.rs.NotFoundException;
import jakarta.ws.rs.NotSupportedException;
import jakarta.ws.rs.core.MediaType;

public class API {

protected static final Logger LOG = Log.logger(API.class);
Expand Down Expand Up @@ -69,8 +71,7 @@ public class API {
public static HugeGraph graph(GraphManager manager, String graph) {
HugeGraph g = manager.graph(graph);
if (g == null) {
throw new NotFoundException(String.format(
"Graph '%s' does not exist", graph));
throw new NotFoundException(String.format("Graph '%s' does not exist", graph));
}
return g;
}
Expand Down Expand Up @@ -140,8 +141,7 @@ protected static void checkUpdatingBody(Checkable body) {
body.checkUpdate();
}

protected static void checkCreatingBody(
Collection<? extends Checkable> bodies) {
protected static void checkCreatingBody(Collection<? extends Checkable> bodies) {
E.checkArgumentNotNull(bodies, "The request body can't be empty");
for (Checkable body : bodies) {
E.checkArgument(body != null,
Expand All @@ -150,8 +150,7 @@ protected static void checkCreatingBody(
}
}

protected static void checkUpdatingBody(
Collection<? extends Checkable> bodies) {
protected static void checkUpdatingBody(Collection<? extends Checkable> bodies) {
E.checkArgumentNotNull(bodies, "The request body can't be empty");
for (Checkable body : bodies) {
E.checkArgumentNotNull(body,
Expand Down Expand Up @@ -186,8 +185,56 @@ public static boolean checkAndParseAction(String action) {
} else if (action.equals(ACTION_ELIMINATE)) {
return false;
} else {
throw new NotSupportedException(
String.format("Not support action '%s'", action));
throw new NotSupportedException(String.format("Not support action '%s'", action));
}
}

public static class ApiMeasurer {

public static final String EDGE_ITER = "edge_iterations";
public static final String VERTICE_ITER = "vertice_iterations";
public static final String COST = "cost";
private final long timeStart;
private final Map<String, Object> measures;

public ApiMeasurer() {
this.timeStart = System.currentTimeMillis();
this.measures = InsertionOrderUtil.newMap();
}

public Map<String, Object> measures() {
measures.put(COST, System.currentTimeMillis() - timeStart);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

compute duration use System.nanoTime perfered

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

return measures;
}

public void put(String key, String value) {
this.measures.put(key, value);
}

public void put(String key, long value) {
this.measures.put(key, value);
}

public void put(String key, int value) {
this.measures.put(key, value);
}

protected void addCount(String key, long value) {
Object current = measures.get(key);
if (current == null) {
measures.put(key, new MutableLong(value));
} else if (current instanceof MutableLong) {
MutableLong currentMutableLong = (MutableLong) current;
currentMutableLong.add(value);
} else if (current instanceof Long) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

measures.computeIfAbsent(key, MutableLong.new).add(value)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add else branch then throw an exception

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

Long currentLong = (Long) current;
measures.put(key, new MutableLong(currentLong + value));
}
}

public void addIterCount(long verticeIters, long edgeIters) {
this.addCount(EDGE_ITER, edgeIters);
this.addCount(VERTICE_ITER, verticeIters);
}
}
}
73 changes: 55 additions & 18 deletions hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/AllShortestPathsAPI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,10 @@
import static org.apache.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY;
import static org.apache.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;

import java.util.HashSet;
import java.util.Iterator;
import java.util.List;

import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.inject.Singleton;
import jakarta.ws.rs.DefaultValue;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.core.Context;

import org.slf4j.Logger;
import java.util.Set;

import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.api.API;
Expand All @@ -44,9 +35,22 @@
import org.apache.hugegraph.traversal.algorithm.ShortestPathTraverser;
import org.apache.hugegraph.type.define.Directions;
import org.apache.hugegraph.util.Log;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.slf4j.Logger;

import com.codahale.metrics.annotation.Timed;
import com.google.common.collect.ImmutableList;

import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.inject.Singleton;
import jakarta.ws.rs.DefaultValue;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.core.Context;

@Path("graphs/{graph}/traversers/allshortestpaths")
@Singleton
@Tag(name = "AllShortestPathsAPI")
Expand All @@ -68,13 +72,20 @@ public String get(@Context GraphManager manager,
@DefaultValue(DEFAULT_MAX_DEGREE) long maxDegree,
@QueryParam("skip_degree")
@DefaultValue("0") long skipDegree,
@QueryParam("with_vertex")
@DefaultValue("false") boolean withVertex,
@QueryParam("with_edge")
@DefaultValue("false") boolean withEdge,
@QueryParam("capacity")
@DefaultValue(DEFAULT_CAPACITY) long capacity) {
LOG.debug("Graph [{}] get shortest path from '{}', to '{}' with " +
"direction {}, edge label {}, max depth '{}', " +
"max degree '{}', skipped degree '{}' and capacity '{}'",
"max degree '{}', skipped degree '{}', capacity '{}', " +
"with vertex '{}' and with edge '{}'",
graph, source, target, direction, edgeLabel, depth,
maxDegree, skipDegree, capacity);
maxDegree, skipDegree, capacity, withVertex, withEdge);

ApiMeasurer measure = new ApiMeasurer();

Id sourceId = VertexAPI.checkAndParseVertexId(source);
Id targetId = VertexAPI.checkAndParseVertexId(target);
Expand All @@ -85,9 +96,35 @@ public String get(@Context GraphManager manager,
ShortestPathTraverser traverser = new ShortestPathTraverser(g);
List<String> edgeLabels = edgeLabel == null ? ImmutableList.of() :
ImmutableList.of(edgeLabel);
HugeTraverser.PathSet paths = traverser.allShortestPaths(
sourceId, targetId, dir, edgeLabels,
depth, maxDegree, skipDegree, capacity);
return manager.serializer(g).writePaths("paths", paths, false);
HugeTraverser.PathSet paths = traverser.allShortestPaths(sourceId, targetId, dir,
edgeLabels, depth, maxDegree,
skipDegree, capacity);

measure.addIterCount(traverser.vertexIterCounter.get(),
traverser.edgeIterCounter.get());

Iterator<?> iterVertex;
Set<Id> vertexIds = new HashSet<>();
for (HugeTraverser.Path path : paths) {
vertexIds.addAll(path.vertices());
}
if (withVertex && !vertexIds.isEmpty()) {
iterVertex = g.vertices(vertexIds.toArray());
measure.addIterCount(vertexIds.size(), 0L);
} else {
iterVertex = vertexIds.iterator();
}

Iterator<?> iterEdge;
Set<Edge> edges = paths.getEdges();
if (withEdge && !edges.isEmpty()) {
iterEdge = edges.iterator();
} else {
iterEdge = HugeTraverser.EdgeRecord.getEdgeIds(edges).iterator();
}

return manager.serializer(g, measure.measures())
.writePaths("paths", paths, false,
iterVertex, iterEdge);
}
}
30 changes: 17 additions & 13 deletions hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/CrosspointsAPI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,6 @@
import static org.apache.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
import static org.apache.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT;

import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.inject.Singleton;
import jakarta.ws.rs.DefaultValue;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.core.Context;

import org.slf4j.Logger;

import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.api.API;
import org.apache.hugegraph.api.graph.EdgeAPI;
Expand All @@ -43,8 +31,20 @@
import org.apache.hugegraph.traversal.algorithm.PathsTraverser;
import org.apache.hugegraph.type.define.Directions;
import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;

import com.codahale.metrics.annotation.Timed;

import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.inject.Singleton;
import jakarta.ws.rs.DefaultValue;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.core.Context;

@Path("graphs/{graph}/traversers/crosspoints")
@Singleton
@Tag(name = "CrosspointsAPI")
Expand Down Expand Up @@ -74,6 +74,7 @@ public String get(@Context GraphManager manager,
graph, source, target, direction, edgeLabel,
depth, maxDegree, capacity, limit);

ApiMeasurer measure = new ApiMeasurer();
Id sourceId = VertexAPI.checkAndParseVertexId(source);
Id targetId = VertexAPI.checkAndParseVertexId(target);
Directions dir = Directions.convert(EdgeAPI.parseDirection(direction));
Expand All @@ -84,6 +85,9 @@ public String get(@Context GraphManager manager,
dir, edgeLabel, depth,
maxDegree, capacity,
limit);
return manager.serializer(g).writePaths("crosspoints", paths, true);
measure.addIterCount(traverser.vertexIterCounter.get(),
traverser.edgeIterCounter.get());
return manager.serializer(g, measure.measures())
.writePaths("crosspoints", paths, true);
}
}
Loading