diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72e562ee6d..c1ed82f18e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ jobs: runs-on: ubuntu-20.04 env: TRAVIS_DIR: hugegraph-dist/src/assembly/travis + REPORT_DIR: target/site/jacoco BACKEND: ${{ matrix.BACKEND }} TRIGGER_BRANCH_NAME: ${{ github.ref_name }} HEAD_BRANCH_NAME: ${{ github.head_ref }} @@ -72,9 +73,14 @@ jobs: - name: Run test run: | $TRAVIS_DIR/run-core-test.sh $BACKEND - $TRAVIS_DIR/run-api-test.sh $BACKEND + $TRAVIS_DIR/run-api-test.sh $BACKEND $REPORT_DIR $TRAVIS_DIR/run-unit-test.sh $BACKEND + - name: Run Raft test + if: ${{ env.BACKEND == 'rocksdb' }} + run: | + $TRAVIS_DIR/run-api-test-for-raft.sh $BACKEND $REPORT_DIR + - name: Run TinkerPop test if: ${{ env.RELEASE_BRANCH == 'true' }} run: | @@ -83,4 +89,4 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: - file: target/site/jacoco/jacoco.xml + file: ${{ env.REPORT_DIR }}/*.xml diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/TaskAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/TaskAPI.java index 90c34b02e5..f124362cdf 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/TaskAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/TaskAPI.java @@ -163,7 +163,7 @@ public Map update(@Context GraphManager manager, HugeTask task = scheduler.task(IdGenerator.of(id)); if (!task.completed() && !task.cancelling()) { scheduler.cancel(task); - if (task.cancelling()) { + if (task.cancelling() || task.cancelled()) { return task.asMap(); } } diff --git a/hugegraph-core/pom.xml b/hugegraph-core/pom.xml index ca06fc3120..1ca2e0413c 100644 --- a/hugegraph-core/pom.xml +++ b/hugegraph-core/pom.xml @@ -54,7 +54,7 @@ com.alipay.sofa jraft-core - 1.3.5 + 1.3.9 org.slf4j diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java index f94d52d78d..0e7a55328c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java @@ -111,20 +111,7 @@ private void listenChanges() { event.checkArgs(String.class, HugeType.class, Id.class); HugeType type = (HugeType) args[1]; Id id = (Id) args[2]; - this.arrayCaches.remove(type, id); - - id = generateId(type, id); - Object value = this.idCache.get(id); - if (value != null) { - // Invalidate id cache - this.idCache.invalidate(id); - - // Invalidate name cache - SchemaElement schema = (SchemaElement) value; - Id prefixedName = generateId(schema.type(), - schema.name()); - this.nameCache.invalidate(prefixedName); - } + this.invalidateCache(type, id); this.resetCachedAll(type); return true; } else if (Cache.ACTION_CLEAR.equals(args[0])) { @@ -140,21 +127,6 @@ private void listenChanges() { } } - private final void resetCachedAll(HugeType type) { - // Set the cache all flag of the schema type to false - this.cachedTypes().put(type, false); - } - - private void clearCache(boolean notify) { - this.idCache.clear(); - this.nameCache.clear(); - this.arrayCaches.clear(); - - if (notify) { - this.notifyChanges(Cache.ACTION_CLEARED, null, null); - } - } - private void unlistenChanges() { // Unlisten store event this.store().provider().unlisten(this.storeEventListener); @@ -164,11 +136,16 @@ private void unlistenChanges() { schemaEventHub.unlisten(Events.CACHE, this.cacheEventListener); } - private void notifyChanges(String action, HugeType type, Id id) { + private final void notifyChanges(String action, HugeType type, Id id) { EventHub graphEventHub = this.params().schemaEventHub(); graphEventHub.notify(Events.CACHE, action, type, id); } + private final void resetCachedAll(HugeType type) { + // Set the cache all flag of the schema type to false + this.cachedTypes().put(type, false); + } + private final void resetCachedAllIfReachedCapacity() { if (this.idCache.size() >= this.idCache.capacity()) { LOG.warn("Schema cache reached capacity({}): {}", @@ -181,20 +158,17 @@ private final CachedTypes cachedTypes() { return this.arrayCaches.cachedTypes(); } - private static Id generateId(HugeType type, Id id) { - // NOTE: it's slower performance to use: - // String.format("%x-%s", type.code(), name) - return IdGenerator.of(type.string() + "-" + id.asString()); - } + private final void clearCache(boolean notify) { + this.idCache.clear(); + this.nameCache.clear(); + this.arrayCaches.clear(); - private static Id generateId(HugeType type, String name) { - return IdGenerator.of(type.string() + "-" + name); + if (notify) { + this.notifyChanges(Cache.ACTION_CLEARED, null, null); + } } - @Override - protected void addSchema(SchemaElement schema) { - super.addSchema(schema); - + private final void updateCache(SchemaElement schema) { this.resetCachedAllIfReachedCapacity(); // update id cache @@ -207,6 +181,39 @@ protected void addSchema(SchemaElement schema) { // update optimized array cache this.arrayCaches.updateIfNeeded(schema); + } + + private final void invalidateCache(HugeType type, Id id) { + // remove from id cache and name cache + Id prefixedId = generateId(type, id); + Object value = this.idCache.get(prefixedId); + if (value != null) { + this.idCache.invalidate(prefixedId); + + SchemaElement schema = (SchemaElement) value; + Id prefixedName = generateId(schema.type(), schema.name()); + this.nameCache.invalidate(prefixedName); + } + + // remove from optimized array cache + this.arrayCaches.remove(type, id); + } + + private static Id generateId(HugeType type, Id id) { + // NOTE: it's slower performance to use: + // String.format("%x-%s", type.code(), name) + return IdGenerator.of(type.string() + "-" + id.asString()); + } + + private static Id generateId(HugeType type, String name) { + return IdGenerator.of(type.string() + "-" + name); + } + + @Override + protected void addSchema(SchemaElement schema) { + super.addSchema(schema); + + this.updateCache(schema); this.notifyChanges(Cache.ACTION_INVALIDED, schema.type(), schema.id()); } @@ -227,19 +234,15 @@ protected T getSchema(HugeType type, Id id) { if (value == null) { value = super.getSchema(type, id); if (value != null) { - this.resetCachedAllIfReachedCapacity(); - - this.idCache.update(prefixedId, value); - SchemaElement schema = (SchemaElement) value; - Id prefixedName = generateId(schema.type(), schema.name()); - this.nameCache.update(prefixedName, schema); + // update id cache, name cache and optimized array cache + this.updateCache(schema); } + } else { + // update optimized array cache for the result from id cache + this.arrayCaches.updateIfNeeded((SchemaElement) value); } - // update optimized array cache - this.arrayCaches.updateIfNeeded((SchemaElement) value); - return (T) value; } @@ -252,13 +255,8 @@ protected T getSchema(HugeType type, if (value == null) { value = super.getSchema(type, name); if (value != null) { - this.resetCachedAllIfReachedCapacity(); - - this.nameCache.update(prefixedName, value); - SchemaElement schema = (SchemaElement) value; - Id prefixedId = generateId(schema.type(), schema.id()); - this.idCache.update(prefixedId, schema); + this.updateCache(schema); } } return (T) value; @@ -268,18 +266,7 @@ protected T getSchema(HugeType type, protected void removeSchema(SchemaElement schema) { super.removeSchema(schema); - Id prefixedId = generateId(schema.type(), schema.id()); - Object value = this.idCache.get(prefixedId); - if (value != null) { - this.idCache.invalidate(prefixedId); - - schema = (SchemaElement) value; - Id prefixedName = generateId(schema.type(), schema.name()); - this.nameCache.invalidate(prefixedName); - } - - // remove from optimized array cache - this.arrayCaches.remove(schema.type(), schema.id()); + this.invalidateCache(schema.type(), schema.id()); this.notifyChanges(Cache.ACTION_INVALIDED, schema.type(), schema.id()); } @@ -299,16 +286,13 @@ protected List getAllSchema(HugeType type) { }); return results; } else { + this.cachedTypes().remove(type); List results = super.getAllSchema(type); long free = this.idCache.capacity() - this.idCache.size(); if (results.size() <= free) { // Update cache for (T schema : results) { - Id prefixedId = generateId(schema.type(), schema.id()); - this.idCache.update(prefixedId, schema); - - Id prefixedName = generateId(schema.type(), schema.name()); - this.nameCache.update(prefixedName, schema); + this.updateCache(schema); } this.cachedTypes().putIfAbsent(type, true); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStore.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStore.java index 17f55b192e..5f9dfb2d8a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStore.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStore.java @@ -50,11 +50,13 @@ public class RaftBackendStore implements BackendStore { private final BackendStore store; private final RaftSharedContext context; private final ThreadLocal mutationBatch; + private final boolean isSafeRead; public RaftBackendStore(BackendStore store, RaftSharedContext context) { this.store = store; this.context = context; this.mutationBatch = new ThreadLocal<>(); + this.isSafeRead = this.context.isSafeRead(); } public BackendStore originStore() { @@ -143,7 +145,8 @@ public Iterator query(Query query) { @Override public Number queryNumber(Query query) { - return (Number) this.queryByRaft(query, o -> this.store.queryNumber(query)); + return (Number) + this.queryByRaft(query, o -> this.store.queryNumber(query)); } @Override @@ -186,7 +189,10 @@ public void increaseCounter(HugeType type, long increment) { @Override public long getCounter(HugeType type) { - return (Long) this.queryByRaft(type, o -> this.store.getCounter(type)); + Object counter = this.queryByRaft(type, true, + o -> this.store.getCounter(type)); + assert counter instanceof Long; + return (Long) counter; } private Object submitAndWait(StoreAction action, byte[] data) { @@ -200,7 +206,12 @@ private Object submitAndWait(StoreCommand command) { } private Object queryByRaft(Object query, Function func) { - if (!this.context.isSafeRead()) { + return this.queryByRaft(query, this.isSafeRead, func); + } + + private Object queryByRaft(Object query, boolean safeRead, + Function func) { + if (!safeRead) { return func.apply(query); } @@ -212,8 +223,8 @@ public void run(Status status, long index, byte[] reqCtx) { future.complete(status, () -> func.apply(query)); } else { future.failure(status, new BackendException( - "Failed to execute query '%s' with read-index: %s", - query, status)); + "Failed to do raft read-index: %s", + status)); } } }; @@ -221,8 +232,8 @@ public void run(Status status, long index, byte[] reqCtx) { try { return future.waitFinished(); } catch (Throwable e) { - LOG.warn("Failed to execute query '{}' with read-index: {}", - query, future.status()); + LOG.warn("Failed to execute query '{}': {}", + query, future.status(), e); throw new BackendException("Failed to execute query: %s", e, query); } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java index 6c20db44f8..46e24f6731 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java @@ -158,7 +158,7 @@ public void init() { for (RaftBackendStore store : this.stores()) { store.init(); } - this.notifyAndWaitEvent(Events.STORE_INITED); + this.notifyAndWaitEvent(Events.STORE_INIT); LOG.debug("Graph '{}' store has been initialized", this.graph()); } @@ -214,7 +214,8 @@ public void initSystemInfo(HugeGraph graph) { @Override public void createSnapshot() { - StoreCommand command = new StoreCommand(StoreType.ALL, + // TODO: snapshot for StoreType.ALL instead of StoreType.GRAPH + StoreCommand command = new StoreCommand(StoreType.GRAPH, StoreAction.SNAPSHOT, null); RaftStoreClosure closure = new RaftStoreClosure(command); this.context.node().submitAndWait(command, closure); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java index 74f8e14db3..d3dd65b0ab 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java @@ -28,7 +28,7 @@ import org.slf4j.Logger; import com.alipay.sofa.jraft.Node; -import com.alipay.sofa.jraft.RaftGroupService; +import com.alipay.sofa.jraft.RaftServiceFactory; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.closure.ReadIndexClosure; import com.alipay.sofa.jraft.core.Replicator.ReplicatorStateListener; @@ -36,7 +36,6 @@ import com.alipay.sofa.jraft.entity.Task; import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.option.NodeOptions; -import com.alipay.sofa.jraft.rpc.RpcServer; import com.alipay.sofa.jraft.util.BytesUtil; import com.baidu.hugegraph.backend.BackendException; import com.baidu.hugegraph.util.LZ4Util; @@ -58,6 +57,7 @@ public RaftNode(RaftSharedContext context) { this.stateMachine = new StoreStateMachine(context); try { this.node = this.initRaftNode(); + LOG.info("Start raft node: {}", this); } catch (IOException e) { throw new BackendException("Failed to init raft node", e); } @@ -94,6 +94,7 @@ public void onLeaderInfoChange(PeerId leaderId, boolean selfIsLeader) { } public void shutdown() { + LOG.info("Shutdown raft node: {}", this); this.node.shutdown(); } @@ -116,13 +117,14 @@ private Node initRaftNode() throws IOException { // TODO: When support sharding, groupId needs to be bound to shard Id String groupId = this.context.group(); PeerId endpoint = this.context.endpoint(); - RpcServer rpcServer = this.context.rpcServer(); - RaftGroupService raftGroupService; - // Shared rpc server - raftGroupService = new RaftGroupService(groupId, endpoint, nodeOptions, - rpcServer, true); - // Start node - return raftGroupService.start(false); + /* + * Start raft node with shared rpc server: + * return new RaftGroupService(groupId, endpoint, nodeOptions, + * this.context.rpcServer(), true) + * .start(false) + */ + return RaftServiceFactory.createAndInitRaftNode(groupId, endpoint, + nodeOptions); } private void submitCommand(StoreCommand command, RaftStoreClosure closure) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java index 75a512da41..8342d08a0e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -30,14 +31,15 @@ import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.ThreadPoolExecutor; -import com.alipay.sofa.jraft.option.ReadOnlyOption; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; +import com.alipay.sofa.jraft.NodeManager; import com.alipay.sofa.jraft.conf.Configuration; import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.option.NodeOptions; import com.alipay.sofa.jraft.option.RaftOptions; +import com.alipay.sofa.jraft.option.ReadOnlyOption; import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory; import com.alipay.sofa.jraft.rpc.RpcServer; import com.alipay.sofa.jraft.util.NamedThreadFactory; @@ -46,6 +48,9 @@ import com.baidu.hugegraph.HugeGraphParams; import com.baidu.hugegraph.backend.cache.Cache; import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.query.Query; +import com.baidu.hugegraph.backend.store.BackendAction; +import com.baidu.hugegraph.backend.store.BackendMutation; import com.baidu.hugegraph.backend.store.BackendStore; import com.baidu.hugegraph.backend.store.raft.rpc.ListPeersProcessor; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreType; @@ -101,7 +106,7 @@ public final class RaftSharedContext { public RaftSharedContext(HugeGraphParams params) { this.params = params; - HugeConfig config = params.configuration(); + HugeConfig config = this.config(); this.schemaStoreName = config.get(CoreOptions.STORE_SCHEMA); this.graphStoreName = config.get(CoreOptions.STORE_GRAPH); @@ -126,12 +131,7 @@ public RaftSharedContext(HugeGraphParams params) { this.raftGroupManager = null; this.rpcForwarder = null; this.registerRpcRequestProcessors(); - } - - private void registerRpcRequestProcessors() { - this.rpcServer.registerProcessor(new StoreCommandProcessor(this)); - this.rpcServer.registerProcessor(new SetLeaderProcessor(this)); - this.rpcServer.registerProcessor(new ListPeersProcessor(this)); + LOG.info("Start raft server successfully: {}", this.endpoint()); } public void initRaftNode() { @@ -149,8 +149,14 @@ public void waitRaftNodeStarted() { } public void close() { - LOG.info("Stopping raft nodes"); - this.rpcServer.shutdown(); + LOG.info("Stop raft server: {}", this.endpoint()); + + RaftNode node = this.node(); + if (node != null) { + node.shutdown(); + } + + this.shutdownRpcServer(); } public RaftNode node() { @@ -168,10 +174,6 @@ public RaftGroupManager raftNodeManager(String group) { return this.raftGroupManager; } - public RpcServer rpcServer() { - return this.rpcServer; - } - public String group() { return DEFAULT_GROUP; } @@ -266,12 +268,41 @@ public NodeOptions nodeOptions() throws IOException { return nodeOptions; } - public void clearCache() { + protected void clearCache() { // Just choose two representatives used to represent schema and graph this.notifyCache(Cache.ACTION_CLEAR, HugeType.VERTEX_LABEL, null); this.notifyCache(Cache.ACTION_CLEAR, HugeType.VERTEX, null); } + protected void updateCacheIfNeeded(BackendMutation mutation, + boolean forwarded) { + // Update cache only when graph run in general mode + if (this.graphMode() != GraphMode.NONE) { + return; + } + /* + * 1. If Follower, need to update cache from store to tx + * 3. If Leader, request is forwarded by follower, need to update cache + * 2. If Leader, request comes from leader, don't need to update cache, + * because the cache will be updated by upper layer + */ + if (!forwarded && this.node().selfIsLeader()) { + return; + } + for (HugeType type : mutation.types()) { + List ids = new ArrayList<>((int) Query.COMMIT_BATCH); + if (type.isSchema() || type.isGraph()) { + java.util.Iterator it = mutation.mutation(type); + while (it.hasNext()) { + ids.add(it.next().entry().originId()); + } + this.notifyCache(Cache.ACTION_INVALID, type, ids); + } else { + // Ignore other types due to not cached them + } + } + } + protected void notifyCache(String action, HugeType type, List ids) { EventHub eventHub; if (type.isGraph()) { @@ -340,14 +371,25 @@ private RpcServer initAndStartRpcServer() { System.setProperty("bolt.channel_write_buf_high_water_mark", String.valueOf(highWaterMark)); - PeerId serverId = new PeerId(); - serverId.parse(this.config().get(CoreOptions.RAFT_ENDPOINT)); + PeerId endpoint = this.endpoint(); + NodeManager.getInstance().addAddress(endpoint.getEndpoint()); RpcServer rpcServer = RaftRpcServerFactory.createAndStartRaftRpcServer( - serverId.getEndpoint()); - LOG.info("RPC server is started successfully"); + endpoint.getEndpoint()); return rpcServer; } + private void shutdownRpcServer() { + this.rpcServer.shutdown(); + PeerId endpoint = this.endpoint(); + NodeManager.getInstance().removeAddress(endpoint.getEndpoint()); + } + + private void registerRpcRequestProcessors() { + this.rpcServer.registerProcessor(new StoreCommandProcessor(this)); + this.rpcServer.registerProcessor(new SetLeaderProcessor(this)); + this.rpcServer.registerProcessor(new ListPeersProcessor(this)); + } + private ExecutorService createReadIndexExecutor(int coreThreads) { int maxThreads = coreThreads << 2; String name = "store-read-index-callback"; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java index 3506b6ca18..0a7647cf3e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java @@ -19,8 +19,10 @@ package com.baidu.hugegraph.backend.store.raft; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; import org.slf4j.Logger; @@ -36,18 +38,12 @@ import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; import com.baidu.hugegraph.backend.BackendException; -import com.baidu.hugegraph.backend.cache.Cache; -import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.serializer.BytesBuffer; -import com.baidu.hugegraph.backend.store.BackendAction; import com.baidu.hugegraph.backend.store.BackendMutation; import com.baidu.hugegraph.backend.store.BackendStore; import com.baidu.hugegraph.backend.store.raft.RaftBackendStore.IncrCounter; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreAction; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreType; -import com.baidu.hugegraph.type.HugeType; -import com.baidu.hugegraph.type.define.GraphMode; import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.LZ4Util; import com.baidu.hugegraph.util.Log; @@ -72,78 +68,24 @@ private RaftNode node() { return this.context.node(); } - private void updateCacheIfNeeded(BackendMutation mutation, - boolean forwarded) { - // Update cache only when graph run in general mode - if (this.context.graphMode() != GraphMode.NONE) { - return; - } - /* - * 1. Follower need to update cache from store to tx - * 2. If request come from leader, cache will be updated by upper layer - * 3. If request is forwarded by follower, need to update cache - */ - if (!forwarded && this.node().selfIsLeader()) { - return; - } - for (HugeType type : mutation.types()) { - List ids = new ArrayList<>((int) Query.COMMIT_BATCH); - if (type.isSchema() || type.isGraph()) { - java.util.Iterator it = mutation.mutation(type); - while (it.hasNext()) { - ids.add(it.next().entry().originId()); - } - this.context.notifyCache(Cache.ACTION_INVALID, type, ids); - } else { - // Ignore other types due to not cached them - } - } - } - @Override public void onApply(Iterator iter) { LOG.debug("Node role: {}", this.node().selfIsLeader() ? "leader" : "follower"); - RaftStoreClosure closure = null; List> futures = new ArrayList<>(); try { + // Apply all the logs while (iter.hasNext()) { - closure = (RaftStoreClosure) iter.done(); + RaftStoreClosure closure = (RaftStoreClosure) iter.done(); if (closure != null) { - // Leader just take it out from the closure - StoreCommand command = closure.command(); - BytesBuffer buffer = BytesBuffer.wrap(command.data()); - // The first two bytes are StoreType and StoreAction - StoreType type = StoreType.valueOf(buffer.read()); - StoreAction action = StoreAction.valueOf(buffer.read()); - boolean forwarded = command.forwarded(); - // Let the producer thread to handle it - closure.complete(Status.OK(), () -> { - this.applyCommand(type, action, buffer, forwarded); - return null; - }); + futures.add(this.onApplyLeader(closure)); } else { - // Follower need readMutation data - byte[] bytes = iter.getData().array(); - // Let the backend thread do it directly - futures.add(this.context.backendExecutor().submit(() -> { - BytesBuffer buffer = LZ4Util.decompress(bytes, - RaftSharedContext.BLOCK_SIZE); - buffer.forReadWritten(); - StoreType type = StoreType.valueOf(buffer.read()); - StoreAction action = StoreAction.valueOf(buffer.read()); - try { - this.applyCommand(type, action, buffer, false); - } catch (Throwable e) { - String title = "Failed to execute backend command"; - LOG.error("{}: {}", title, action, e); - throw new BackendException(title, e); - } - })); + futures.add(this.onApplyFollower(iter.getData())); } iter.next(); } - // Follower wait tasks finished + + // Wait for all tasks finished for (Future future : futures) { future.get(); } @@ -152,17 +94,58 @@ public void onApply(Iterator iter) { LOG.error("{}", title, e); Status status = new Status(RaftError.ESTATEMACHINE, "%s: %s", title, e.getMessage()); - if (closure != null) { - closure.failure(status, e); - } // Will cause current node inactive // TODO: rollback to correct index iter.setErrorAndRollback(1L, status); } } - private void applyCommand(StoreType type, StoreAction action, - BytesBuffer buffer, boolean forwarded) { + private Future onApplyLeader(RaftStoreClosure closure) { + // Leader just take the command out from the closure + StoreCommand command = closure.command(); + BytesBuffer buffer = BytesBuffer.wrap(command.data()); + // The first two bytes are StoreType and StoreAction + StoreType type = StoreType.valueOf(buffer.read()); + StoreAction action = StoreAction.valueOf(buffer.read()); + boolean forwarded = command.forwarded(); + // Let the producer thread to handle it, and wait for it + CompletableFuture future = new CompletableFuture<>(); + closure.complete(Status.OK(), () -> { + Object result; + try { + result = this.applyCommand(type, action, buffer, forwarded); + } catch (Throwable e) { + future.completeExceptionally(e); + throw e; + } + future.complete(result); + return result; + }); + return future; + } + + private Future onApplyFollower(ByteBuffer data) { + // Follower need to read mutation data + byte[] bytes = data.array(); + // Let the backend thread do it directly + return this.context.backendExecutor().submit(() -> { + BytesBuffer buffer = LZ4Util.decompress(bytes, + RaftSharedContext.BLOCK_SIZE); + buffer.forReadWritten(); + StoreType type = StoreType.valueOf(buffer.read()); + StoreAction action = StoreAction.valueOf(buffer.read()); + try { + return this.applyCommand(type, action, buffer, false); + } catch (Throwable e) { + String title = "Failed to execute backend command"; + LOG.error("{}: {}", title, action, e); + throw new BackendException(title, e); + } + }); + } + + private Object applyCommand(StoreType type, StoreAction action, + BytesBuffer buffer, boolean forwarded) { E.checkState(type != StoreType.ALL, "Can't apply command for all store at one time"); BackendStore store = this.store(type); @@ -190,7 +173,7 @@ private void applyCommand(StoreType type, StoreAction action, store.beginTx(); for (BackendMutation mutation : mutations) { store.mutate(mutation); - this.updateCacheIfNeeded(mutation, forwarded); + this.context.updateCacheIfNeeded(mutation, forwarded); } store.commitTx(); break; @@ -205,6 +188,7 @@ private void applyCommand(StoreType type, StoreAction action, default: throw new IllegalArgumentException("Invalid action " + action); } + return null; } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/StoreCommandProcessor.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/StoreCommandProcessor.java index b9f6b80e6d..13ce2cfb8e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/StoreCommandProcessor.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/StoreCommandProcessor.java @@ -50,16 +50,20 @@ public StoreCommandProcessor(RaftSharedContext context) { @Override public Message processRequest(StoreCommandRequest request, RpcRequestClosure done) { - LOG.debug("Processing StoreCommandRequest"); + LOG.debug("Processing StoreCommandRequest: {}", request.getAction()); RaftNode node = this.context.node(); try { StoreCommand command = this.parseStoreCommand(request); RaftStoreClosure closure = new RaftStoreClosure(command); node.submitAndWait(command, closure); + // TODO: return the submitAndWait() result to rpc client return StoreCommandResponse.newBuilder().setStatus(true).build(); } catch (Throwable e) { - StoreCommandResponse.Builder builder; - builder = StoreCommandResponse.newBuilder().setStatus(false); + LOG.warn("Failed to process StoreCommandRequest: {}", + request.getAction(), e); + StoreCommandResponse.Builder builder = StoreCommandResponse + .newBuilder() + .setStatus(false); if (e.getMessage() != null) { builder.setMessage(e.getMessage()); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java index 584dccaba0..af567b384f 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java @@ -325,6 +325,10 @@ public Id removeOlapPk(PropertyKey propertyKey) { @Watched(prefix = "schema") public void updateSchemaStatus(SchemaElement schema, SchemaStatus status) { + if (!this.existsSchemaId(schema.type(), schema.id())) { + LOG.warn("Can't update schema '{}', it may be deleted", schema); + return; + } schema.status(status); this.updateSchema(schema); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java index a95707dc54..390abe39e5 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java @@ -50,7 +50,8 @@ public final class TaskManager { "server-info-db-worker-%d"; public static final String TASK_SCHEDULER = "task-scheduler-%d"; - protected static final int SCHEDULE_PERIOD = 3; // Unit second + protected static final int SCHEDULE_PERIOD = 1; // Unit second + private static final int THREADS = 4; private static final TaskManager MANAGER = new TaskManager(THREADS); diff --git a/hugegraph-dist/src/assembly/static/bin/raft-tools.sh b/hugegraph-dist/src/assembly/static/bin/raft-tools.sh index 4f0e6fc57d..34a7582dc8 100755 --- a/hugegraph-dist/src/assembly/static/bin/raft-tools.sh +++ b/hugegraph-dist/src/assembly/static/bin/raft-tools.sh @@ -14,6 +14,11 @@ LOG_PATH=${HOME_PATH}/logs . ${BIN_PATH}/util.sh +#export HUGEGRAPH_URL= +#export HUGEGRAPH_GRAPH= +#export HUGEGRAPH_USERNAME= +#export HUGEGRAPH_PASSWORD= + function print_usage() { echo "usage: raft-tools.sh [options]" echo "options: " @@ -26,9 +31,6 @@ function print_usage() { echo " -h,--help display help information" } -GRAPH="hugegraph" -ENDPOINT="" - if [[ $# -lt 2 ]]; then print_usage exit 0 @@ -36,56 +38,58 @@ fi function list_peers() { local graph=$1 - local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url` - local url=${rest_server_url}/graphs/${graph}/raft/list_peers + local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/list_peers - curl ${url} + curl ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD} } function get_leader() { local graph=$1 - local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url` - local url=${rest_server_url}/graphs/${graph}/raft/get_leader + local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/get_leader - curl ${url} + curl ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD} } function set_leader() { local graph=$1 local endpoint=$2 - local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url` - local url=${rest_server_url}/graphs/${graph}/raft/set_leader?endpoint=${endpoint} + local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/set_leader?endpoint=${endpoint} - curl -X POST ${url} + curl -X POST ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD} } function transfer_leader() { local graph=$1 local endpoint=$2 - local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url` - local url=${rest_server_url}/graphs/${graph}/raft/transfer_leader?endpoint=${endpoint} + local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/transfer_leader?endpoint=${endpoint} - curl -X POST ${url} + curl -X POST ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD} } function add_peer() { local graph=$1 local endpoint=$2 - local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url` - local url=${rest_server_url}/graphs/${graph}/raft/add_peer?endpoint=${endpoint} + local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/add_peer?endpoint=${endpoint} - curl -X POST ${url} + curl -X POST ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD} } function remove_peer() { local graph=$1 local endpoint=$2 - local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url` - local url=${rest_server_url}/graphs/${graph}/raft/remove_peer?endpoint=${endpoint} + local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/remove_peer?endpoint=${endpoint} - curl -X POST ${url} + curl -X POST ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD} } +if [ "${HUGEGRAPH_URL}" = "" ]; then + HUGEGRAPH_URL=`read_property ${CONF_PATH}/rest-server.properties restserver.url` +fi + +if [ "${HUGEGRAPH_GRAPH}" = "" ]; then + HUGEGRAPH_GRAPH="hugegraph" +fi + case $1 in # help --help|-h) diff --git a/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh index f2c2ce4728..91836eab22 100644 --- a/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh +++ b/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh @@ -7,14 +7,15 @@ GC_OPTION="" USER_OPTION="" SERVER_STARTUP_TIMEOUT_S=30 -while getopts "g:m:s:j:v" arg; do +while getopts "g:m:s:j:t:v" arg; do case ${arg} in g) GC_OPTION="$OPTARG" ;; m) OPEN_MONITOR="$OPTARG" ;; s) OPEN_SECURITY_CHECK="$OPTARG" ;; j) USER_OPTION="$OPTARG" ;; + t) SERVER_STARTUP_TIMEOUT_S="$OPTARG" ;; v) VERBOSE="verbose" ;; - ?) echo "USAGE: $0 [-g g1] [-m true|false] [-s true|false] [-j xxx] [-v]" && exit 1 ;; + ?) echo "USAGE: $0 [-g g1] [-m true|false] [-s true|false] [-j java_options] [-t timeout] [-v]" && exit 1 ;; esac done diff --git a/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-dist/src/assembly/static/bin/util.sh index 188d4bb545..178d4ddbbf 100755 --- a/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-dist/src/assembly/static/bin/util.sh @@ -136,7 +136,7 @@ function wait_for_startup() { now_s=`date '+%s'` done - echo "The operation timed out when attempting to connect to $server_url" >&2 + echo "The operation timed out(${timeout_s}s) when attempting to connect to $server_url" >&2 return 1 } diff --git a/hugegraph-dist/src/assembly/travis/build-report.sh b/hugegraph-dist/src/assembly/travis/build-report.sh index 06720debbe..fcf3fabcd5 100755 --- a/hugegraph-dist/src/assembly/travis/build-report.sh +++ b/hugegraph-dist/src/assembly/travis/build-report.sh @@ -3,6 +3,8 @@ set -ev BACKEND=$1 +JACOCO_PORT=$2 +JACOCO_REPORT_FILE=$3 OPTION_CLASS_FILES_BACKEND="--classfiles hugegraph-$BACKEND/target/classes/com/baidu/hugegraph" if [ "$BACKEND" == "memory" ]; then @@ -11,10 +13,10 @@ if [ "$BACKEND" == "memory" ]; then fi cd hugegraph-test -mvn jacoco:dump@pull-test-data -Dapp.host=localhost -Dapp.port=36320 -Dskip.dump=false +mvn jacoco:dump@pull-test-data -Dapp.host=localhost -Dapp.port=$JACOCO_PORT -Dskip.dump=false cd ../ java -jar $TRAVIS_DIR/jacococli.jar report hugegraph-test/target/jacoco-it.exec \ --classfiles hugegraph-dist/target/classes/com/baidu/hugegraph \ --classfiles hugegraph-api/target/classes/com/baidu/hugegraph \ --classfiles hugegraph-core/target/classes/com/baidu/hugegraph \ - $OPTION_CLASS_FILES_BACKEND --xml report.xml + ${OPTION_CLASS_FILES_BACKEND} --xml "${JACOCO_REPORT_FILE}" diff --git a/hugegraph-dist/src/assembly/travis/conf-raft1/graphs/hugegraph.properties b/hugegraph-dist/src/assembly/travis/conf-raft1/graphs/hugegraph.properties new file mode 100644 index 0000000000..0f043b44dc --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft1/graphs/hugegraph.properties @@ -0,0 +1,29 @@ +gremlin.graph=com.baidu.hugegraph.auth.HugeFactoryAuthProxy +#gremlin.graph=com.baidu.hugegraph.HugeFactory + +store=hugegraph + +backend=rocksdb +serializer=binary + +rocksdb.data_path=rocksdb-data-raft1 +rocksdb.wal_path=rocksdb-data-raft1 + +raft.mode=true +raft.safe_read=true +raft.use_snapshot=false +raft.endpoint=127.0.0.1:8281 +raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283 +raft.path=rocksdb-raftlog1 +raft.use_replicator_pipeline=true +raft.election_timeout=10000 +raft.snapshot_interval=3600 +raft.backend_threads=48 +raft.read_index_threads=8 +raft.read_strategy=ReadOnlyLeaseBased +raft.queue_size=16384 +raft.queue_publish_timeout=60 +raft.apply_batch=1 +raft.rpc_threads=8 +raft.rpc_connect_timeout=5000 +raft.rpc_timeout=60000 diff --git a/hugegraph-dist/src/assembly/travis/conf-raft1/gremlin-server.yaml b/hugegraph-dist/src/assembly/travis/conf-raft1/gremlin-server.yaml new file mode 100644 index 0000000000..29242f954e --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft1/gremlin-server.yaml @@ -0,0 +1,105 @@ +# host and port of gremlin server, need to be consistent with host and port in rest-server.properties +#host: 127.0.0.1 +port: 8181 + +# timeout in ms of gremlin query +scriptEvaluationTimeout: 30000 + +channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer +# don't set graph at here, this happens after support for dynamically adding graph +graphs: { +} +scriptEngines: { + gremlin-groovy: { + plugins: { + com.baidu.hugegraph.plugin.HugeGraphGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: { + classImports: [ + java.lang.Math, + com.baidu.hugegraph.backend.id.IdGenerator, + com.baidu.hugegraph.type.define.Directions, + com.baidu.hugegraph.type.define.NodeRole, + com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser, + com.baidu.hugegraph.traversal.algorithm.CountTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser, + com.baidu.hugegraph.traversal.algorithm.HugeTraverser, + com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser, + com.baidu.hugegraph.traversal.algorithm.KneighborTraverser, + com.baidu.hugegraph.traversal.algorithm.KoutTraverser, + com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser, + com.baidu.hugegraph.traversal.algorithm.PathsTraverser, + com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser, + com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser, + com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser, + com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep, + com.baidu.hugegraph.traversal.optimize.Text, + com.baidu.hugegraph.traversal.optimize.TraversalUtil, + com.baidu.hugegraph.util.DateUtil + ], + methodImports: [java.lang.Math#*] + }, + org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: { + files: [scripts/empty-sample.groovy] + } + } + } +} +serializers: + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } +metrics: { + consoleReporter: {enabled: false, interval: 180000}, + csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv}, + jmxReporter: {enabled: false}, + slf4jReporter: {enabled: false, interval: 180000}, + gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST}, + graphiteReporter: {enabled: false, interval: 180000} +} +maxInitialLineLength: 4096 +maxHeaderSize: 8192 +maxChunkSize: 8192 +maxContentLength: 65536 +maxAccumulationBufferComponents: 1024 +resultIterationBatchSize: 64 +writeBufferLowWaterMark: 32768 +writeBufferHighWaterMark: 65536 +ssl: { + enabled: false +} +authentication: { + authenticator: com.baidu.hugegraph.auth.StandardAuthenticator, + #authenticationHandler: org.apache.tinkerpop.gremlin.server.handler.SaslAndHttpBasicAuthenticationHandler, + authenticationHandler: com.baidu.hugegraph.auth.WsAndHttpBasicAuthHandler, + config: {tokens: conf/rest-server.properties} +} diff --git a/hugegraph-dist/src/assembly/travis/conf-raft1/rest-server.properties b/hugegraph-dist/src/assembly/travis/conf-raft1/rest-server.properties new file mode 100644 index 0000000000..2611b95a2f --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft1/rest-server.properties @@ -0,0 +1,11 @@ +restserver.url=http://127.0.0.1:8080 +gremlinserver.url=http://127.0.0.1:8181 +graphs=conf/graphs +auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator + +rpc.server_host=127.0.0.1 +rpc.server_port=8091 +rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 + +server.id=server1 +server.role=master diff --git a/hugegraph-dist/src/assembly/travis/conf-raft2/graphs/hugegraph.properties b/hugegraph-dist/src/assembly/travis/conf-raft2/graphs/hugegraph.properties new file mode 100644 index 0000000000..9aaf5fdfc4 --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft2/graphs/hugegraph.properties @@ -0,0 +1,29 @@ +gremlin.graph=com.baidu.hugegraph.auth.HugeFactoryAuthProxy +#gremlin.graph=com.baidu.hugegraph.HugeFactory + +store=hugegraph + +backend=rocksdb +serializer=binary + +rocksdb.data_path=rocksdb-data-raft2 +rocksdb.wal_path=rocksdb-data-raft2 + +raft.mode=true +raft.safe_read=true +raft.use_snapshot=false +raft.endpoint=127.0.0.1:8282 +raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283 +raft.path=rocksdb-raftlog2 +raft.use_replicator_pipeline=true +raft.election_timeout=10000 +raft.snapshot_interval=3600 +raft.backend_threads=48 +raft.read_index_threads=8 +raft.read_strategy=ReadOnlyLeaseBased +raft.queue_size=16384 +raft.queue_publish_timeout=60 +raft.apply_batch=1 +raft.rpc_threads=8 +raft.rpc_connect_timeout=5000 +raft.rpc_timeout=60000 diff --git a/hugegraph-dist/src/assembly/travis/conf-raft2/gremlin-server.yaml b/hugegraph-dist/src/assembly/travis/conf-raft2/gremlin-server.yaml new file mode 100644 index 0000000000..fcc0c351e4 --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft2/gremlin-server.yaml @@ -0,0 +1,105 @@ +# host and port of gremlin server +#host: 127.0.0.1 +port: 8182 + +# timeout in ms of gremlin query +scriptEvaluationTimeout: 30000 + +channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer +# don't set graph at here, this happens after support for dynamically adding graph +graphs: { +} +scriptEngines: { + gremlin-groovy: { + plugins: { + com.baidu.hugegraph.plugin.HugeGraphGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: { + classImports: [ + java.lang.Math, + com.baidu.hugegraph.backend.id.IdGenerator, + com.baidu.hugegraph.type.define.Directions, + com.baidu.hugegraph.type.define.NodeRole, + com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser, + com.baidu.hugegraph.traversal.algorithm.CountTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser, + com.baidu.hugegraph.traversal.algorithm.HugeTraverser, + com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser, + com.baidu.hugegraph.traversal.algorithm.KneighborTraverser, + com.baidu.hugegraph.traversal.algorithm.KoutTraverser, + com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser, + com.baidu.hugegraph.traversal.algorithm.PathsTraverser, + com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser, + com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser, + com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser, + com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep, + com.baidu.hugegraph.traversal.optimize.Text, + com.baidu.hugegraph.traversal.optimize.TraversalUtil, + com.baidu.hugegraph.util.DateUtil + ], + methodImports: [java.lang.Math#*] + }, + org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: { + files: [scripts/empty-sample.groovy] + } + } + } +} +serializers: + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } +metrics: { + consoleReporter: {enabled: false, interval: 180000}, + csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv}, + jmxReporter: {enabled: false}, + slf4jReporter: {enabled: false, interval: 180000}, + gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST}, + graphiteReporter: {enabled: false, interval: 180000} +} +maxInitialLineLength: 4096 +maxHeaderSize: 8192 +maxChunkSize: 8192 +maxContentLength: 65536 +maxAccumulationBufferComponents: 1024 +resultIterationBatchSize: 64 +writeBufferLowWaterMark: 32768 +writeBufferHighWaterMark: 65536 +ssl: { + enabled: false +} +authentication: { + authenticator: com.baidu.hugegraph.auth.StandardAuthenticator, + #authenticationHandler: org.apache.tinkerpop.gremlin.server.handler.SaslAndHttpBasicAuthenticationHandler, + authenticationHandler: com.baidu.hugegraph.auth.WsAndHttpBasicAuthHandler, + config: {tokens: conf/rest-server.properties} +} diff --git a/hugegraph-dist/src/assembly/travis/conf-raft2/rest-server.properties b/hugegraph-dist/src/assembly/travis/conf-raft2/rest-server.properties new file mode 100644 index 0000000000..cde90ac068 --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft2/rest-server.properties @@ -0,0 +1,11 @@ +restserver.url=http://127.0.0.1:8082 +gremlinserver.url=http://127.0.0.1:8182 +graphs=conf/graphs +auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator + +rpc.server_host=127.0.0.1 +rpc.server_port=8092 +rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 + +server.id=server2 +server.role=worker diff --git a/hugegraph-dist/src/assembly/travis/conf-raft3/graphs/hugegraph.properties b/hugegraph-dist/src/assembly/travis/conf-raft3/graphs/hugegraph.properties new file mode 100644 index 0000000000..0fb5a203cd --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft3/graphs/hugegraph.properties @@ -0,0 +1,29 @@ +gremlin.graph=com.baidu.hugegraph.auth.HugeFactoryAuthProxy +#gremlin.graph=com.baidu.hugegraph.HugeFactory + +store=hugegraph + +backend=rocksdb +serializer=binary + +rocksdb.data_path=rocksdb-data-raft3 +rocksdb.wal_path=rocksdb-data-raft3 + +raft.mode=true +raft.safe_read=true +raft.use_snapshot=false +raft.endpoint=127.0.0.1:8283 +raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283 +raft.path=rocksdb-raftlog3 +raft.use_replicator_pipeline=true +raft.election_timeout=10000 +raft.snapshot_interval=3600 +raft.backend_threads=48 +raft.read_index_threads=8 +raft.read_strategy=ReadOnlyLeaseBased +raft.queue_size=16384 +raft.queue_publish_timeout=60 +raft.apply_batch=1 +raft.rpc_threads=8 +raft.rpc_connect_timeout=5000 +raft.rpc_timeout=60000 diff --git a/hugegraph-dist/src/assembly/travis/conf-raft3/gremlin-server.yaml b/hugegraph-dist/src/assembly/travis/conf-raft3/gremlin-server.yaml new file mode 100644 index 0000000000..218f49964d --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft3/gremlin-server.yaml @@ -0,0 +1,105 @@ +# host and port of gremlin server +#host: 127.0.0.1 +port: 8183 + +# timeout in ms of gremlin query +scriptEvaluationTimeout: 30000 + +channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer +# don't set graph at here, this happens after support for dynamically adding graph +graphs: { +} +scriptEngines: { + gremlin-groovy: { + plugins: { + com.baidu.hugegraph.plugin.HugeGraphGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: { + classImports: [ + java.lang.Math, + com.baidu.hugegraph.backend.id.IdGenerator, + com.baidu.hugegraph.type.define.Directions, + com.baidu.hugegraph.type.define.NodeRole, + com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser, + com.baidu.hugegraph.traversal.algorithm.CountTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser, + com.baidu.hugegraph.traversal.algorithm.HugeTraverser, + com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser, + com.baidu.hugegraph.traversal.algorithm.KneighborTraverser, + com.baidu.hugegraph.traversal.algorithm.KoutTraverser, + com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser, + com.baidu.hugegraph.traversal.algorithm.PathsTraverser, + com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser, + com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser, + com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser, + com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep, + com.baidu.hugegraph.traversal.optimize.Text, + com.baidu.hugegraph.traversal.optimize.TraversalUtil, + com.baidu.hugegraph.util.DateUtil + ], + methodImports: [java.lang.Math#*] + }, + org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: { + files: [scripts/empty-sample.groovy] + } + } + } +} +serializers: + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } +metrics: { + consoleReporter: {enabled: false, interval: 180000}, + csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv}, + jmxReporter: {enabled: false}, + slf4jReporter: {enabled: false, interval: 180000}, + gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST}, + graphiteReporter: {enabled: false, interval: 180000} +} +maxInitialLineLength: 4096 +maxHeaderSize: 8192 +maxChunkSize: 8192 +maxContentLength: 65536 +maxAccumulationBufferComponents: 1024 +resultIterationBatchSize: 64 +writeBufferLowWaterMark: 32768 +writeBufferHighWaterMark: 65536 +ssl: { + enabled: false +} +authentication: { + authenticator: com.baidu.hugegraph.auth.StandardAuthenticator, + #authenticationHandler: org.apache.tinkerpop.gremlin.server.handler.SaslAndHttpBasicAuthenticationHandler, + authenticationHandler: com.baidu.hugegraph.auth.WsAndHttpBasicAuthHandler, + config: {tokens: conf/rest-server.properties} +} diff --git a/hugegraph-dist/src/assembly/travis/conf-raft3/rest-server.properties b/hugegraph-dist/src/assembly/travis/conf-raft3/rest-server.properties new file mode 100644 index 0000000000..e8e72a3fd3 --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft3/rest-server.properties @@ -0,0 +1,11 @@ +restserver.url=http://127.0.0.1:8083 +gremlinserver.url=http://127.0.0.1:8183 +graphs=conf/graphs +auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator + +rpc.server_host=127.0.0.1 +rpc.server_port=8093 +rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 + +server.id=server3 +server.role=worker diff --git a/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh b/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh new file mode 100755 index 0000000000..9c58e7170b --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -ev + +BACKEND=$1 +REPORT_DIR=$2 +REPORT_FILE=$REPORT_DIR/jacoco-api-test.xml + +TRAVIS_DIR=`dirname $0` +VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout` +SERVER_DIR=hugegraph-$VERSION +RAFT1_DIR=hugegraph-raft1 +RAFT2_DIR=hugegraph-raft2 +RAFT3_DIR=hugegraph-raft3 +CONF=$SERVER_DIR/conf/graphs/hugegraph.properties +REST_SERVER_CONF=$SERVER_DIR/conf/rest-server.properties +GREMLIN_SERVER_CONF=$SERVER_DIR/conf/gremlin-server.yaml + +JACOCO_PORT=36320 +RAFT_TOOLS=$RAFT1_DIR/bin/raft-tools.sh +RAFT_LEADER="127.0.0.1:8281" + +mvn package -DskipTests + +# mkdir for each raft-server +cp -r $SERVER_DIR $RAFT1_DIR +cp -r $SERVER_DIR $RAFT2_DIR +cp -r $SERVER_DIR $RAFT3_DIR + +# config raft-server (must keep '/.') +cp -rf $TRAVIS_DIR/conf-raft1/. $RAFT1_DIR/conf/ +cp -rf $TRAVIS_DIR/conf-raft2/. $RAFT2_DIR/conf/ +cp -rf $TRAVIS_DIR/conf-raft3/. $RAFT3_DIR/conf/ + +# start server +$TRAVIS_DIR/start-server.sh $RAFT1_DIR $BACKEND $JACOCO_PORT || (cat $RAFT1_DIR/logs/hugegraph-server.log && exit 1) & +$TRAVIS_DIR/start-server.sh $RAFT2_DIR $BACKEND || (cat $RAFT2_DIR/logs/hugegraph-server.log && exit 1) & +$TRAVIS_DIR/start-server.sh $RAFT3_DIR $BACKEND || (cat $RAFT3_DIR/logs/hugegraph-server.log && exit 1) + +export HUGEGRAPH_USERNAME=admin +export HUGEGRAPH_PASSWORD=pa +$RAFT_TOOLS --set-leader "hugegraph" "$RAFT_LEADER" + +# run api-test +mvn test -P api-test,$BACKEND || (cat $RAFT1_DIR/logs/hugegraph-server.log && exit 1) + +$TRAVIS_DIR/build-report.sh $BACKEND $JACOCO_PORT $REPORT_FILE + +# stop server +$TRAVIS_DIR/stop-server.sh $RAFT1_DIR +$TRAVIS_DIR/stop-server.sh $RAFT2_DIR +$TRAVIS_DIR/stop-server.sh $RAFT3_DIR diff --git a/hugegraph-dist/src/assembly/travis/run-api-test.sh b/hugegraph-dist/src/assembly/travis/run-api-test.sh index 15573dc03f..4de8d61898 100755 --- a/hugegraph-dist/src/assembly/travis/run-api-test.sh +++ b/hugegraph-dist/src/assembly/travis/run-api-test.sh @@ -3,6 +3,8 @@ set -ev BACKEND=$1 +REPORT_DIR=$2 +REPORT_FILE=$REPORT_DIR/jacoco-api-test-for-raft.xml TRAVIS_DIR=`dirname $0` VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout` @@ -10,6 +12,7 @@ SERVER_DIR=hugegraph-$VERSION CONF=$SERVER_DIR/conf/graphs/hugegraph.properties REST_SERVER_CONF=$SERVER_DIR/conf/rest-server.properties GREMLIN_SERVER_CONF=$SERVER_DIR/conf/gremlin-server.yaml +JACOCO_PORT=36320 mvn package -DskipTests @@ -28,9 +31,13 @@ authentication: { config: {tokens: conf/rest-server.properties} }" >> $GREMLIN_SERVER_CONF -$TRAVIS_DIR/start-server.sh $SERVER_DIR $BACKEND || (cat $SERVER_DIR/logs/hugegraph-server.log && exit 1) +# start server +$TRAVIS_DIR/start-server.sh $SERVER_DIR $BACKEND $JACOCO_PORT || (cat $SERVER_DIR/logs/hugegraph-server.log && exit 1) # run api-test mvn test -P api-test,$BACKEND || (cat $SERVER_DIR/logs/hugegraph-server.log && exit 1) -$TRAVIS_DIR/build-report.sh $BACKEND -$TRAVIS_DIR/stop-server.sh + +$TRAVIS_DIR/build-report.sh $BACKEND $JACOCO_PORT $REPORT_FILE + +# stop server +$TRAVIS_DIR/stop-server.sh $SERVER_DIR diff --git a/hugegraph-dist/src/assembly/travis/start-server.sh b/hugegraph-dist/src/assembly/travis/start-server.sh index 616dcd0649..3a94a0dae0 100755 --- a/hugegraph-dist/src/assembly/travis/start-server.sh +++ b/hugegraph-dist/src/assembly/travis/start-server.sh @@ -6,14 +6,21 @@ HOME_DIR=$(pwd) TRAVIS_DIR=$(dirname $0) BASE_DIR=$1 BACKEND=$2 +JACOCO_PORT=$3 + +JACOCO_JAR=${HOME_DIR}/${TRAVIS_DIR}/jacocoagent.jar + BIN=$BASE_DIR/bin CONF=$BASE_DIR/conf/graphs/hugegraph.properties REST_CONF=$BASE_DIR/conf/rest-server.properties GREMLIN_CONF=$BASE_DIR/conf/gremlin-server.yaml -declare -A backend_serializer_map=(["memory"]="text" ["cassandra"]="cassandra" \ - ["scylladb"]="scylladb" ["mysql"]="mysql" \ - ["hbase"]="hbase" ["rocksdb"]="binary" \ +declare -A backend_serializer_map=(["memory"]="text" \ + ["cassandra"]="cassandra" \ + ["scylladb"]="scylladb" \ + ["mysql"]="mysql" \ + ["hbase"]="hbase" \ + ["rocksdb"]="binary" \ ["postgresql"]="postgresql") SERIALIZER=${backend_serializer_map[$BACKEND]} @@ -37,5 +44,9 @@ fi # Append schema.sync_deletion=true to config file echo "schema.sync_deletion=true" >> $CONF -AGENT_JAR=${HOME_DIR}/${TRAVIS_DIR}/jacocoagent.jar -echo -e "pa" | $BIN/init-store.sh && $BIN/start-hugegraph.sh -j "-javaagent:${AGENT_JAR}=includes=*,port=36320,destfile=jacoco-it.exec,output=tcpserver" -v +JACOCO_OPTION="" +if [ -n "$JACOCO_PORT" ]; then + JACOCO_OPTION="-javaagent:${JACOCO_JAR}=includes=*,port=${JACOCO_PORT},destfile=jacoco-it.exec,output=tcpserver" +fi + +echo -e "pa" | $BIN/init-store.sh && $BIN/start-hugegraph.sh -j "$JACOCO_OPTION" -t 60 -v diff --git a/hugegraph-dist/src/assembly/travis/stop-server.sh b/hugegraph-dist/src/assembly/travis/stop-server.sh index a966c88c78..584ef27c5d 100755 --- a/hugegraph-dist/src/assembly/travis/stop-server.sh +++ b/hugegraph-dist/src/assembly/travis/stop-server.sh @@ -2,8 +2,8 @@ set -ev -VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout` -BASE_DIR=hugegraph-$VERSION +BASE_DIR=$1 BIN=$BASE_DIR/bin +VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout` $BIN/stop-hugegraph.sh diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/GremlinApiTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/GremlinApiTest.java index c983ca1134..cff29cedb1 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/GremlinApiTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/GremlinApiTest.java @@ -24,6 +24,7 @@ import javax.ws.rs.core.Response; +import org.junit.Assume; import org.junit.Test; import com.baidu.hugegraph.testutil.Assert; @@ -86,12 +87,29 @@ public void testScript() { @Test public void testClearAndInit() { String body = "{" + + "\"gremlin\":\"hugegraph.backendStoreFeatures()" + + " .supportsSharedStorage();\"," + + "\"bindings\":{}," + + "\"language\":\"gremlin-groovy\"," + + "\"aliases\":{\"g\":\"__g_hugegraph\"}}"; + String content = assertResponseStatus(200, client().post(path, body)); + Map result = assertJsonContains(content, "result"); + @SuppressWarnings({ "unchecked" }) + Object data = ((List) assertMapContains(result, "data")).get(0); + boolean supportsSharedStorage = (boolean) data; + Assume.assumeTrue("Can't clear non-shared-storage backend", + supportsSharedStorage); + + body = "{" + "\"gremlin\":\"" - + "def auth = hugegraph.hugegraph().authManager();" - + "def admin = auth.findUser('admin');" - + "hugegraph.clearBackend();" - + "hugegraph.initBackend();" - + "auth.createUser(admin);\"," + + " if (!hugegraph.backendStoreFeatures()" + + " .supportsSharedStorage())" + + " return;" + + " def auth = hugegraph.hugegraph().authManager();" + + " def admin = auth.findUser('admin');" + + " hugegraph.clearBackend();" + + " hugegraph.initBackend();" + + " auth.createUser(admin);\"," + "\"bindings\":{}," + "\"language\":\"gremlin-groovy\"," + "\"aliases\":{\"g\":\"__g_hugegraph\"}}"; @@ -99,7 +117,7 @@ public void testClearAndInit() { body = "{" + "\"gremlin\":\"hugegraph.serverStarted(" - + "IdGenerator.of('server1'), NodeRole.MASTER)\"," + + " IdGenerator.of('server1'), NodeRole.MASTER)\"," + "\"bindings\":{}," + "\"language\":\"gremlin-groovy\"," + "\"aliases\":{\"g\":\"__g_hugegraph\"}}"; diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/ProjectApiTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/ProjectApiTest.java index 7c020bad13..b0ca27c972 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/ProjectApiTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/ProjectApiTest.java @@ -27,17 +27,48 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; +import org.junit.After; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; +import com.baidu.hugegraph.util.JsonUtil; +import com.google.common.collect.ImmutableMap; + public class ProjectApiTest extends BaseApiTest { private final static String path = "graphs/hugegraph/auth/projects"; - @Before - public void setup() { - BaseApiTest.truncate(); + @Override + @After + public void teardown() throws Exception { + Response resp = client().get(path); + String respBody = assertResponseStatus(200, resp); + List projects = readList(respBody, "projects", Map.class); + for (Object project : projects) { + @SuppressWarnings("unchecked") + Map projectMap = ((Map) project); + String projectId = (String) projectMap.get("id"); + // remove graphs from project if needed + List projectGraphs = (List) projectMap.get("project_graphs"); + if (projectGraphs != null && projectGraphs.size() > 0) { + Map graphs = ImmutableMap.of("project_graphs", + projectGraphs); + resp = client().target() + .path(path) + .path(projectId) + .queryParam("action", "remove_graph") + .request() + .put(Entity.json(JsonUtil.toJson(graphs))); + assertResponseStatus(200, resp); + } + // delete project + resp = client().target() + .path(path) + .path(projectId) + .request() + .delete(); + assertResponseStatus(204, resp); + } } @Test @@ -141,32 +172,32 @@ public void testAddGraphs() { } @Test - public void testremoveGraphs() { + public void testRemoveGraphs() { String projectId = this.createProjectAndAddGraph("project_test", "graph_test"); - String graph = "{\"project_graphs\":[\"graph_test\"]}"; + String graphs = "{\"project_graphs\":[\"graph_test\"]}"; Response resp = client().target() .path(path) .path(projectId) .queryParam("action", "remove_graph") .request() - .put(Entity.json(graph)); + .put(Entity.json(graphs)); assertResponseStatus(200, resp); String project = this.getProject(projectId); Assert.assertFalse(project.contains("project_graphs")); - this.addGraphs(projectId, "graph_test1", "graph_test2"); + this.addGraphsToProject(projectId, "graph_test1", "graph_test2"); - graph = "{\"project_graphs\":[\"graph_test1\"]}"; + graphs = "{\"project_graphs\":[\"graph_test1\"]}"; resp = client().target() .path(path) .path(projectId) .queryParam("action", "remove_graph") .request() - .put(Entity.json(graph)); - + .put(Entity.json(graphs)); assertResponseStatus(200, resp); + project = this.getProject(projectId); List graphs1 = assertJsonContains(project, "project_graphs"); Assert.assertEquals(1, graphs1.size()); @@ -199,19 +230,18 @@ private String createProjectAndAddGraph(String projectName, String graph) { String projectId = assertJsonContains(createProject(projectName, null), "id"); - this.addGraphs(projectId, graph); + this.addGraphsToProject(projectId, graph); return projectId; } - private void addGraphs(String projectId, String... graphNames) { + private void addGraphsToProject(String projectId, String... graphNames) { Assert.assertFalse(ArrayUtils.isEmpty(graphNames)); StringBuilder graphNamesBuilder = new StringBuilder(); for (int i = 0; i < graphNames.length - 1; i++) { graphNamesBuilder.append(String.format("\"%s\",", graphNames[i])); } - graphNamesBuilder.append( - String.format("\"%s\"", - graphNames[graphNames.length - 1])); + graphNamesBuilder.append(String.format("\"%s\"", + graphNames[graphNames.length - 1])); String graphs = String.format("{\"project_graphs\":[%s]}", graphNamesBuilder); Response resp = client().target() diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/TaskApiTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/TaskApiTest.java index 2a73e1c969..8884e5b242 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/TaskApiTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/TaskApiTest.java @@ -79,7 +79,8 @@ public void testCancel() { r.getStatus() == 202 || r.getStatus() == 400); if (r.getStatus() == 202) { String status = assertJsonContains(content, "task_status"); - Assert.assertEquals("cancelling", status); + Assert.assertTrue(status, status.equals("cancelling") || + status.equals("cancelled")); } else { assert r.getStatus() == 400; String error = String.format(