Skip to content

Commit

Permalink
Support to create graph dynamically (#1065)
Browse files Browse the repository at this point in the history 
* Support to create graph dynamically
* merge gremlin-server graphs and specified graphs
* Just use graphs directory as the graph source
* use notify() instead of call() of Eventhub for graph add/remove
* merge clone graph into create graph

Implement #165

Change-Id: I03d3e606d6292acec46a7725094c05fe84a6c9cd

Co-authored-by: zhangyi51 <[email protected]>
  • Loading branch information
@Linary @zhoney
Linary and zhoney authored Dec 24, 2021
1 parent 2dd2d7c commit d213166
Show file tree
Hide file tree
Showing 24 changed files with 502 additions and 164 deletions.
2 changes: 1 addition & 1 deletion hugegraph-api/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@
</addDefaultSpecificationEntries>
</manifest>
<manifestEntries>
<Implementation-Version>0.66.0.0</Implementation-Version>
<Implementation-Version>0.67.0.0</Implementation-Version>
</manifestEntries>
</archive>
</configuration>
Expand Down
50 changes: 45 additions & 5 deletions hugegraph-api/src/main/java/com/baidu/hugegraph/api/profile/GraphsAPI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import javax.ws.rs.ForbiddenException;
import javax.ws.rs.GET;
import javax.ws.rs.NotSupportedException;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
Expand Down Expand Up @@ -63,6 +64,7 @@ public class GraphsAPI extends API {
private static final Logger LOG = Log.logger(RestServer.class);

private static final String CONFIRM_CLEAR = "I'm sure to delete all data";
private static final String CONFIRM_DROP = "I'm sure to drop the graph";

@GET
@Timed
Expand Down Expand Up @@ -100,6 +102,47 @@ public Object get(@Context GraphManager manager,
return ImmutableMap.of("name", g.name(), "backend", g.backend());
}

@DELETE
@Timed
@Path("{name}")
@Produces(APPLICATION_JSON_WITH_CHARSET)
@RolesAllowed({"admin", "$owner=$name"})
public void drop(@Context GraphManager manager,
@PathParam("name") String name,
@QueryParam("confirm_message") String message) {
LOG.debug("Drop graph by name '{}'", name);

E.checkArgument(CONFIRM_DROP.equals(message),
"Please take the message: %s", CONFIRM_DROP);
manager.dropGraph(name);
}

@POST
@Timed
@Path("{name}")
@Consumes(APPLICATION_JSON_WITH_CHARSET)
@Produces(APPLICATION_JSON_WITH_CHARSET)
@RolesAllowed({"admin"})
public Object create(@Context GraphManager manager,
@PathParam("name") String name,
@QueryParam("clone_graph_name") String clone,
String configText) {
E.checkArgument(configText != null && !configText.isEmpty(),
"The config text can't be null or empty");
HugeGraph graph;
if (clone != null && !clone.isEmpty()) {
LOG.debug("Create graph {} with copied config from '{}'",
name, clone);
graph = manager.cloneGraph(clone, name, configText);
} else {
LOG.debug("Create graph {} with config options '{}'", name,
configText);
graph = manager.createGraph(name, configText);
}
return ImmutableMap.of("name", graph.name(),
"backend", graph.backend());
}

@GET
@Timed
@Path("{name}/conf")
Expand Down Expand Up @@ -130,12 +173,9 @@ public void clear(@Context GraphManager manager,
@QueryParam("confirm_message") String message) {
LOG.debug("Clear graph by name '{}'", name);

E.checkArgument(CONFIRM_CLEAR.equals(message),
"Please take the message: %s", CONFIRM_CLEAR);
HugeGraph g = graph(manager, name);

if (!CONFIRM_CLEAR.equals(message)) {
throw new IllegalArgumentException(String.format(
"Please take the message: %s", CONFIRM_CLEAR));
}
g.truncateBackend();
}

Expand Down
72 changes: 69 additions & 3 deletions hugegraph-api/src/main/java/com/baidu/hugegraph/auth/ContextGremlinServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package com.baidu.hugegraph.auth;

import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;

Expand All @@ -28,23 +29,66 @@
import org.apache.tinkerpop.gremlin.server.Settings;
import org.apache.tinkerpop.gremlin.server.util.ThreadFactoryUtil;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.slf4j.Logger;

import com.baidu.hugegraph.HugeException;
import com.baidu.hugegraph.HugeGraph;
import com.baidu.hugegraph.auth.HugeGraphAuthProxy.Context;
import com.baidu.hugegraph.auth.HugeGraphAuthProxy.ContextThreadPoolExecutor;
import com.baidu.hugegraph.config.CoreOptions;
import com.baidu.hugegraph.event.EventHub;
import com.baidu.hugegraph.util.Events;
import com.baidu.hugegraph.util.Log;

/**
* GremlinServer with custom ServerGremlinExecutor, which can pass Context
*/
public class ContextGremlinServer extends GremlinServer {

public ContextGremlinServer(final Settings settings) {
private static final Logger LOG = Log.logger(GremlinServer.class);

private static final String G_PREFIX = "__g_";

private final EventHub eventHub;

public ContextGremlinServer(final Settings settings, EventHub eventHub) {
/*
* pass custom Executor https://github.com/apache/tinkerpop/pull/813
*/
super(settings, newGremlinExecutorService(settings));
this.eventHub = eventHub;
this.listenChanges();
}

private void listenChanges() {
this.eventHub.listen(Events.GRAPH_CREATE, event -> {
LOG.debug("GremlinServer accepts event 'graph.create'");
event.checkArgs(HugeGraph.class);
HugeGraph graph = (HugeGraph) event.args()[0];
this.injectGraph(graph);
return null;
});
this.eventHub.listen(Events.GRAPH_DROP, event -> {
LOG.debug("GremlinServer accepts event 'graph.drop'");
event.checkArgs(String.class);
String name = (String) event.args()[0];
this.removeGraph(name);
return null;
});
}

private void unlistenChanges() {
this.eventHub.unlisten(Events.GRAPH_CREATE);
this.eventHub.unlisten(Events.GRAPH_DROP);
}

@Override
public synchronized CompletableFuture<Void> stop() {
try {
return super.stop();
} finally {
this.unlistenChanges();
}
}

public void injectAuthGraph() {
Expand All @@ -59,12 +103,12 @@ public void injectAuthGraph() {
}
}

public void injectTraversalSource(String prefix) {
public void injectTraversalSource() {
GraphManager manager = this.getServerGremlinExecutor()
.getGraphManager();
for (String graph : manager.getGraphNames()) {
GraphTraversalSource g = manager.getGraph(graph).traversal();
String gName = prefix + graph;
String gName = G_PREFIX + graph;
if (manager.getTraversalSource(gName) != null) {
throw new HugeException(
"Found existing name '%s' in global bindings, " +
Expand All @@ -75,6 +119,28 @@ public void injectTraversalSource(String prefix) {
}
}

private void injectGraph(HugeGraph graph) {
String name = graph.name();
GraphManager manager = this.getServerGremlinExecutor()
.getGraphManager();
manager.putGraph(name, graph);

GraphTraversalSource g = manager.getGraph(name).traversal();
manager.putTraversalSource(G_PREFIX + name, g);
}

private void removeGraph(String name) {
GraphManager manager = this.getServerGremlinExecutor()
.getGraphManager();
try {
manager.removeGraph(name);
} catch (Exception e) {
throw new HugeException("Failed to remove graph '%s' from " +
"gremlin server context", e, name);
}
manager.removeTraversalSource(G_PREFIX + name);
}

static ExecutorService newGremlinExecutorService(Settings settings) {
if (settings.gremlinPool == 0) {
settings.gremlinPool = CoreOptions.CPUS;
Expand Down
6 changes: 5 additions & 1 deletion hugegraph-api/src/main/java/com/baidu/hugegraph/auth/StandardAuthenticator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import java.io.Console;
import java.net.InetAddress;
import java.util.Map;
import java.util.Scanner;

import org.apache.commons.lang.NotImplementedException;
Expand All @@ -32,6 +33,7 @@
import com.baidu.hugegraph.config.HugeConfig;
import com.baidu.hugegraph.config.ServerOptions;
import com.baidu.hugegraph.rpc.RpcClientProviderWithAuth;
import com.baidu.hugegraph.util.ConfigUtil;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.StringEncoding;

Expand Down Expand Up @@ -100,7 +102,9 @@ private String inputPassword() {
@Override
public void setup(HugeConfig config) {
String graphName = config.get(ServerOptions.AUTH_GRAPH_STORE);
String graphPath = config.getMap(ServerOptions.GRAPHS).get(graphName);
Map<String, String> graphConfs = ConfigUtil.scanGraphsDir(
config.get(ServerOptions.GRAPHS));
String graphPath = graphConfs.get(graphName);
E.checkArgument(graphPath != null,
"Can't find graph name '%s' in config '%s' at " +
"'rest-server.properties' to store auth information, " +
Expand Down
8 changes: 4 additions & 4 deletions hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,12 +135,12 @@ public static synchronized ServerOptions instance() {
2 * CoreOptions.CPUS
);

public static final ConfigListOption<String> GRAPHS =
new ConfigListOption<>(
public static final ConfigOption<String> GRAPHS =
new ConfigOption<>(
"graphs",
"The map of graphs' name and config file.",
"The directory store graphs' config file.",
disallowEmpty(),
"hugegraph:conf/hugegraph.properties"
"./conf/graphs"
);

public static final ConfigOption<Integer> MAX_VERTICES_PER_BATCH =
Expand Down
Loading

0 comments on commit d213166

Please sign in to comment.