From 6e2c2b57b9742d1233aa7f91a414fbd2cb3af71c Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Tue, 4 Jan 2022 12:06:48 +0800 Subject: [PATCH] add raft-server test Change-Id: I44e0e3c3cb44b806bd87b2216a339673f26d3166 --- .github/workflows/ci.yml | 5 + .../assembly/static/bin/start-hugegraph.sh | 5 +- .../src/assembly/static/bin/util.sh | 2 +- .../src/assembly/travis/build-report.sh | 3 +- .../conf-raft1/graphs/hugegraph.properties | 29 +++++ .../travis/conf-raft1/gremlin-server.yaml | 105 ++++++++++++++++++ .../travis/conf-raft1/rest-server.properties | 11 ++ .../conf-raft2/graphs/hugegraph.properties | 29 +++++ .../travis/conf-raft2/gremlin-server.yaml | 105 ++++++++++++++++++ .../travis/conf-raft2/rest-server.properties | 11 ++ .../conf-raft3/graphs/hugegraph.properties | 29 +++++ .../travis/conf-raft3/gremlin-server.yaml | 105 ++++++++++++++++++ .../travis/conf-raft3/rest-server.properties | 11 ++ .../assembly/travis/run-api-test-for-raft.sh | 43 +++++++ .../src/assembly/travis/run-api-test.sh | 11 +- .../src/assembly/travis/start-server.sh | 21 +++- .../src/assembly/travis/stop-server.sh | 4 +- 17 files changed, 515 insertions(+), 14 deletions(-) create mode 100644 hugegraph-dist/src/assembly/travis/conf-raft1/graphs/hugegraph.properties create mode 100644 hugegraph-dist/src/assembly/travis/conf-raft1/gremlin-server.yaml create mode 100644 hugegraph-dist/src/assembly/travis/conf-raft1/rest-server.properties create mode 100644 hugegraph-dist/src/assembly/travis/conf-raft2/graphs/hugegraph.properties create mode 100644 hugegraph-dist/src/assembly/travis/conf-raft2/gremlin-server.yaml create mode 100644 hugegraph-dist/src/assembly/travis/conf-raft2/rest-server.properties create mode 100644 hugegraph-dist/src/assembly/travis/conf-raft3/graphs/hugegraph.properties create mode 100644 hugegraph-dist/src/assembly/travis/conf-raft3/gremlin-server.yaml create mode 100644 hugegraph-dist/src/assembly/travis/conf-raft3/rest-server.properties create mode 100755 hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72e562ee6d..36c28bca25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,6 +75,11 @@ jobs: $TRAVIS_DIR/run-api-test.sh $BACKEND $TRAVIS_DIR/run-unit-test.sh $BACKEND + - name: Run Raft test + if: ${{ env.BACKEND == 'rocksdb' }} + run: | + $TRAVIS_DIR/run-api-test-for-raft.sh $BACKEND + - name: Run TinkerPop test if: ${{ env.RELEASE_BRANCH == 'true' }} run: | diff --git a/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh index f2c2ce4728..91836eab22 100644 --- a/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh +++ b/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh @@ -7,14 +7,15 @@ GC_OPTION="" USER_OPTION="" SERVER_STARTUP_TIMEOUT_S=30 -while getopts "g:m:s:j:v" arg; do +while getopts "g:m:s:j:t:v" arg; do case ${arg} in g) GC_OPTION="$OPTARG" ;; m) OPEN_MONITOR="$OPTARG" ;; s) OPEN_SECURITY_CHECK="$OPTARG" ;; j) USER_OPTION="$OPTARG" ;; + t) SERVER_STARTUP_TIMEOUT_S="$OPTARG" ;; v) VERBOSE="verbose" ;; - ?) echo "USAGE: $0 [-g g1] [-m true|false] [-s true|false] [-j xxx] [-v]" && exit 1 ;; + ?) echo "USAGE: $0 [-g g1] [-m true|false] [-s true|false] [-j java_options] [-t timeout] [-v]" && exit 1 ;; esac done diff --git a/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-dist/src/assembly/static/bin/util.sh index 188d4bb545..178d4ddbbf 100755 --- a/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-dist/src/assembly/static/bin/util.sh @@ -136,7 +136,7 @@ function wait_for_startup() { now_s=`date '+%s'` done - echo "The operation timed out when attempting to connect to $server_url" >&2 + echo "The operation timed out(${timeout_s}s) when attempting to connect to $server_url" >&2 return 1 } diff --git a/hugegraph-dist/src/assembly/travis/build-report.sh b/hugegraph-dist/src/assembly/travis/build-report.sh index 06720debbe..8c8026171e 100755 --- a/hugegraph-dist/src/assembly/travis/build-report.sh +++ b/hugegraph-dist/src/assembly/travis/build-report.sh @@ -3,6 +3,7 @@ set -ev BACKEND=$1 +JACOCO_PORT=$2 OPTION_CLASS_FILES_BACKEND="--classfiles hugegraph-$BACKEND/target/classes/com/baidu/hugegraph" if [ "$BACKEND" == "memory" ]; then @@ -11,7 +12,7 @@ if [ "$BACKEND" == "memory" ]; then fi cd hugegraph-test -mvn jacoco:dump@pull-test-data -Dapp.host=localhost -Dapp.port=36320 -Dskip.dump=false +mvn jacoco:dump@pull-test-data -Dapp.host=localhost -Dapp.port=$JACOCO_PORT -Dskip.dump=false cd ../ java -jar $TRAVIS_DIR/jacococli.jar report hugegraph-test/target/jacoco-it.exec \ --classfiles hugegraph-dist/target/classes/com/baidu/hugegraph \ diff --git a/hugegraph-dist/src/assembly/travis/conf-raft1/graphs/hugegraph.properties b/hugegraph-dist/src/assembly/travis/conf-raft1/graphs/hugegraph.properties new file mode 100644 index 0000000000..769e64f977 --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft1/graphs/hugegraph.properties @@ -0,0 +1,29 @@ +gremlin.graph=com.baidu.hugegraph.auth.HugeFactoryAuthProxy +#gremlin.graph=com.baidu.hugegraph.HugeFactory + +store=hugegraph + +backend=rocksdb +serializer=binary + +rocksdb.data_path=rocksdb-data-raft1 +rocksdb.wal_path=rocksdb-data-raft1 + +raft.mode=true +raft.safe_read=false +raft.use_snapshot=false +raft.endpoint=127.0.0.1:8281 +raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283 +raft.path=rocksdb-raftlog1 +raft.use_replicator_pipeline=true +raft.election_timeout=10000 +raft.snapshot_interval=3600 +raft.backend_threads=48 +raft.read_index_threads=8 +raft.read_strategy=ReadOnlyLeaseBased +raft.queue_size=16384 +raft.queue_publish_timeout=60 +raft.apply_batch=1 +raft.rpc_threads=80 +raft.rpc_connect_timeout=5000 +raft.rpc_timeout=60000 diff --git a/hugegraph-dist/src/assembly/travis/conf-raft1/gremlin-server.yaml b/hugegraph-dist/src/assembly/travis/conf-raft1/gremlin-server.yaml new file mode 100644 index 0000000000..29242f954e --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft1/gremlin-server.yaml @@ -0,0 +1,105 @@ +# host and port of gremlin server, need to be consistent with host and port in rest-server.properties +#host: 127.0.0.1 +port: 8181 + +# timeout in ms of gremlin query +scriptEvaluationTimeout: 30000 + +channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer +# don't set graph at here, this happens after support for dynamically adding graph +graphs: { +} +scriptEngines: { + gremlin-groovy: { + plugins: { + com.baidu.hugegraph.plugin.HugeGraphGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: { + classImports: [ + java.lang.Math, + com.baidu.hugegraph.backend.id.IdGenerator, + com.baidu.hugegraph.type.define.Directions, + com.baidu.hugegraph.type.define.NodeRole, + com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser, + com.baidu.hugegraph.traversal.algorithm.CountTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser, + com.baidu.hugegraph.traversal.algorithm.HugeTraverser, + com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser, + com.baidu.hugegraph.traversal.algorithm.KneighborTraverser, + com.baidu.hugegraph.traversal.algorithm.KoutTraverser, + com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser, + com.baidu.hugegraph.traversal.algorithm.PathsTraverser, + com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser, + com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser, + com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser, + com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep, + com.baidu.hugegraph.traversal.optimize.Text, + com.baidu.hugegraph.traversal.optimize.TraversalUtil, + com.baidu.hugegraph.util.DateUtil + ], + methodImports: [java.lang.Math#*] + }, + org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: { + files: [scripts/empty-sample.groovy] + } + } + } +} +serializers: + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } +metrics: { + consoleReporter: {enabled: false, interval: 180000}, + csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv}, + jmxReporter: {enabled: false}, + slf4jReporter: {enabled: false, interval: 180000}, + gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST}, + graphiteReporter: {enabled: false, interval: 180000} +} +maxInitialLineLength: 4096 +maxHeaderSize: 8192 +maxChunkSize: 8192 +maxContentLength: 65536 +maxAccumulationBufferComponents: 1024 +resultIterationBatchSize: 64 +writeBufferLowWaterMark: 32768 +writeBufferHighWaterMark: 65536 +ssl: { + enabled: false +} +authentication: { + authenticator: com.baidu.hugegraph.auth.StandardAuthenticator, + #authenticationHandler: org.apache.tinkerpop.gremlin.server.handler.SaslAndHttpBasicAuthenticationHandler, + authenticationHandler: com.baidu.hugegraph.auth.WsAndHttpBasicAuthHandler, + config: {tokens: conf/rest-server.properties} +} diff --git a/hugegraph-dist/src/assembly/travis/conf-raft1/rest-server.properties b/hugegraph-dist/src/assembly/travis/conf-raft1/rest-server.properties new file mode 100644 index 0000000000..2611b95a2f --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft1/rest-server.properties @@ -0,0 +1,11 @@ +restserver.url=http://127.0.0.1:8080 +gremlinserver.url=http://127.0.0.1:8181 +graphs=conf/graphs +auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator + +rpc.server_host=127.0.0.1 +rpc.server_port=8091 +rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 + +server.id=server1 +server.role=master diff --git a/hugegraph-dist/src/assembly/travis/conf-raft2/graphs/hugegraph.properties b/hugegraph-dist/src/assembly/travis/conf-raft2/graphs/hugegraph.properties new file mode 100644 index 0000000000..ed38a6e9e2 --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft2/graphs/hugegraph.properties @@ -0,0 +1,29 @@ +gremlin.graph=com.baidu.hugegraph.auth.HugeFactoryAuthProxy +#gremlin.graph=com.baidu.hugegraph.HugeFactory + +store=hugegraph + +backend=rocksdb +serializer=binary + +rocksdb.data_path=rocksdb-data-raft2 +rocksdb.wal_path=rocksdb-data-raft2 + +raft.mode=true +raft.safe_read=false +raft.use_snapshot=false +raft.endpoint=127.0.0.1:8282 +raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283 +raft.path=rocksdb-raftlog2 +raft.use_replicator_pipeline=true +raft.election_timeout=10000 +raft.snapshot_interval=3600 +raft.backend_threads=48 +raft.read_index_threads=8 +raft.read_strategy=ReadOnlyLeaseBased +raft.queue_size=16384 +raft.queue_publish_timeout=60 +raft.apply_batch=1 +raft.rpc_threads=80 +raft.rpc_connect_timeout=5000 +raft.rpc_timeout=60000 diff --git a/hugegraph-dist/src/assembly/travis/conf-raft2/gremlin-server.yaml b/hugegraph-dist/src/assembly/travis/conf-raft2/gremlin-server.yaml new file mode 100644 index 0000000000..fcc0c351e4 --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft2/gremlin-server.yaml @@ -0,0 +1,105 @@ +# host and port of gremlin server +#host: 127.0.0.1 +port: 8182 + +# timeout in ms of gremlin query +scriptEvaluationTimeout: 30000 + +channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer +# don't set graph at here, this happens after support for dynamically adding graph +graphs: { +} +scriptEngines: { + gremlin-groovy: { + plugins: { + com.baidu.hugegraph.plugin.HugeGraphGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: { + classImports: [ + java.lang.Math, + com.baidu.hugegraph.backend.id.IdGenerator, + com.baidu.hugegraph.type.define.Directions, + com.baidu.hugegraph.type.define.NodeRole, + com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser, + com.baidu.hugegraph.traversal.algorithm.CountTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser, + com.baidu.hugegraph.traversal.algorithm.HugeTraverser, + com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser, + com.baidu.hugegraph.traversal.algorithm.KneighborTraverser, + com.baidu.hugegraph.traversal.algorithm.KoutTraverser, + com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser, + com.baidu.hugegraph.traversal.algorithm.PathsTraverser, + com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser, + com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser, + com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser, + com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep, + com.baidu.hugegraph.traversal.optimize.Text, + com.baidu.hugegraph.traversal.optimize.TraversalUtil, + com.baidu.hugegraph.util.DateUtil + ], + methodImports: [java.lang.Math#*] + }, + org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: { + files: [scripts/empty-sample.groovy] + } + } + } +} +serializers: + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } +metrics: { + consoleReporter: {enabled: false, interval: 180000}, + csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv}, + jmxReporter: {enabled: false}, + slf4jReporter: {enabled: false, interval: 180000}, + gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST}, + graphiteReporter: {enabled: false, interval: 180000} +} +maxInitialLineLength: 4096 +maxHeaderSize: 8192 +maxChunkSize: 8192 +maxContentLength: 65536 +maxAccumulationBufferComponents: 1024 +resultIterationBatchSize: 64 +writeBufferLowWaterMark: 32768 +writeBufferHighWaterMark: 65536 +ssl: { + enabled: false +} +authentication: { + authenticator: com.baidu.hugegraph.auth.StandardAuthenticator, + #authenticationHandler: org.apache.tinkerpop.gremlin.server.handler.SaslAndHttpBasicAuthenticationHandler, + authenticationHandler: com.baidu.hugegraph.auth.WsAndHttpBasicAuthHandler, + config: {tokens: conf/rest-server.properties} +} diff --git a/hugegraph-dist/src/assembly/travis/conf-raft2/rest-server.properties b/hugegraph-dist/src/assembly/travis/conf-raft2/rest-server.properties new file mode 100644 index 0000000000..cde90ac068 --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft2/rest-server.properties @@ -0,0 +1,11 @@ +restserver.url=http://127.0.0.1:8082 +gremlinserver.url=http://127.0.0.1:8182 +graphs=conf/graphs +auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator + +rpc.server_host=127.0.0.1 +rpc.server_port=8092 +rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 + +server.id=server2 +server.role=worker diff --git a/hugegraph-dist/src/assembly/travis/conf-raft3/graphs/hugegraph.properties b/hugegraph-dist/src/assembly/travis/conf-raft3/graphs/hugegraph.properties new file mode 100644 index 0000000000..71d15201bb --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft3/graphs/hugegraph.properties @@ -0,0 +1,29 @@ +gremlin.graph=com.baidu.hugegraph.auth.HugeFactoryAuthProxy +#gremlin.graph=com.baidu.hugegraph.HugeFactory + +store=hugegraph + +backend=rocksdb +serializer=binary + +rocksdb.data_path=rocksdb-data-raft3 +rocksdb.wal_path=rocksdb-data-raft3 + +raft.mode=true +raft.safe_read=false +raft.use_snapshot=false +raft.endpoint=127.0.0.1:8283 +raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283 +raft.path=rocksdb-raftlog3 +raft.use_replicator_pipeline=true +raft.election_timeout=10000 +raft.snapshot_interval=3600 +raft.backend_threads=48 +raft.read_index_threads=8 +raft.read_strategy=ReadOnlyLeaseBased +raft.queue_size=16384 +raft.queue_publish_timeout=60 +raft.apply_batch=1 +raft.rpc_threads=80 +raft.rpc_connect_timeout=5000 +raft.rpc_timeout=60000 diff --git a/hugegraph-dist/src/assembly/travis/conf-raft3/gremlin-server.yaml b/hugegraph-dist/src/assembly/travis/conf-raft3/gremlin-server.yaml new file mode 100644 index 0000000000..218f49964d --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft3/gremlin-server.yaml @@ -0,0 +1,105 @@ +# host and port of gremlin server +#host: 127.0.0.1 +port: 8183 + +# timeout in ms of gremlin query +scriptEvaluationTimeout: 30000 + +channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer +# don't set graph at here, this happens after support for dynamically adding graph +graphs: { +} +scriptEngines: { + gremlin-groovy: { + plugins: { + com.baidu.hugegraph.plugin.HugeGraphGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: { + classImports: [ + java.lang.Math, + com.baidu.hugegraph.backend.id.IdGenerator, + com.baidu.hugegraph.type.define.Directions, + com.baidu.hugegraph.type.define.NodeRole, + com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser, + com.baidu.hugegraph.traversal.algorithm.CountTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser, + com.baidu.hugegraph.traversal.algorithm.HugeTraverser, + com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser, + com.baidu.hugegraph.traversal.algorithm.KneighborTraverser, + com.baidu.hugegraph.traversal.algorithm.KoutTraverser, + com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser, + com.baidu.hugegraph.traversal.algorithm.PathsTraverser, + com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser, + com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser, + com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser, + com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep, + com.baidu.hugegraph.traversal.optimize.Text, + com.baidu.hugegraph.traversal.optimize.TraversalUtil, + com.baidu.hugegraph.util.DateUtil + ], + methodImports: [java.lang.Math#*] + }, + org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: { + files: [scripts/empty-sample.groovy] + } + } + } +} +serializers: + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } +metrics: { + consoleReporter: {enabled: false, interval: 180000}, + csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv}, + jmxReporter: {enabled: false}, + slf4jReporter: {enabled: false, interval: 180000}, + gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST}, + graphiteReporter: {enabled: false, interval: 180000} +} +maxInitialLineLength: 4096 +maxHeaderSize: 8192 +maxChunkSize: 8192 +maxContentLength: 65536 +maxAccumulationBufferComponents: 1024 +resultIterationBatchSize: 64 +writeBufferLowWaterMark: 32768 +writeBufferHighWaterMark: 65536 +ssl: { + enabled: false +} +authentication: { + authenticator: com.baidu.hugegraph.auth.StandardAuthenticator, + #authenticationHandler: org.apache.tinkerpop.gremlin.server.handler.SaslAndHttpBasicAuthenticationHandler, + authenticationHandler: com.baidu.hugegraph.auth.WsAndHttpBasicAuthHandler, + config: {tokens: conf/rest-server.properties} +} diff --git a/hugegraph-dist/src/assembly/travis/conf-raft3/rest-server.properties b/hugegraph-dist/src/assembly/travis/conf-raft3/rest-server.properties new file mode 100644 index 0000000000..e8e72a3fd3 --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/conf-raft3/rest-server.properties @@ -0,0 +1,11 @@ +restserver.url=http://127.0.0.1:8083 +gremlinserver.url=http://127.0.0.1:8183 +graphs=conf/graphs +auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator + +rpc.server_host=127.0.0.1 +rpc.server_port=8093 +rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 + +server.id=server3 +server.role=worker diff --git a/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh b/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh new file mode 100755 index 0000000000..2880c8f880 --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -ev + +BACKEND=$1 + +TRAVIS_DIR=`dirname $0` +VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout` +SERVER_DIR=hugegraph-$VERSION +RAFT1_DIR=hugegraph-raft1 +RAFT2_DIR=hugegraph-raft2 +RAFT3_DIR=hugegraph-raft3 +CONF=$SERVER_DIR/conf/graphs/hugegraph.properties +REST_SERVER_CONF=$SERVER_DIR/conf/rest-server.properties +GREMLIN_SERVER_CONF=$SERVER_DIR/conf/gremlin-server.yaml +JACOCO_PORT=36320 + +mvn package -DskipTests + +# mkdir for each raft-server +cp -r $SERVER_DIR $RAFT1_DIR +cp -r $SERVER_DIR $RAFT2_DIR +cp -r $SERVER_DIR $RAFT3_DIR + +# config raft-server (must keep '/.') +cp -rf $TRAVIS_DIR/conf-raft1/. $RAFT1_DIR/conf/ +cp -rf $TRAVIS_DIR/conf-raft2/. $RAFT2_DIR/conf/ +cp -rf $TRAVIS_DIR/conf-raft3/. $RAFT3_DIR/conf/ + +# start server +$TRAVIS_DIR/start-server.sh $RAFT1_DIR $BACKEND $JACOCO_PORT || (cat $RAFT1_DIR/logs/hugegraph-server.log && exit 1) & +$TRAVIS_DIR/start-server.sh $RAFT2_DIR $BACKEND || (cat $RAFT2_DIR/logs/hugegraph-server.log && exit 1) & +$TRAVIS_DIR/start-server.sh $RAFT3_DIR $BACKEND || (cat $RAFT3_DIR/logs/hugegraph-server.log && exit 1) + +# run api-test +mvn test -P api-test,$BACKEND || (cat $SERVER_DIR/logs/hugegraph-server.log && exit 1) + +$TRAVIS_DIR/build-report.sh $BACKEND $JACOCO_PORT + +# stop server +$TRAVIS_DIR/stop-server.sh $RAFT1_DIR +$TRAVIS_DIR/stop-server.sh $RAFT2_DIR +$TRAVIS_DIR/stop-server.sh $RAFT3_DIR diff --git a/hugegraph-dist/src/assembly/travis/run-api-test.sh b/hugegraph-dist/src/assembly/travis/run-api-test.sh index 15573dc03f..d29a773ff1 100755 --- a/hugegraph-dist/src/assembly/travis/run-api-test.sh +++ b/hugegraph-dist/src/assembly/travis/run-api-test.sh @@ -10,6 +10,7 @@ SERVER_DIR=hugegraph-$VERSION CONF=$SERVER_DIR/conf/graphs/hugegraph.properties REST_SERVER_CONF=$SERVER_DIR/conf/rest-server.properties GREMLIN_SERVER_CONF=$SERVER_DIR/conf/gremlin-server.yaml +JACOCO_PORT=36320 mvn package -DskipTests @@ -28,9 +29,13 @@ authentication: { config: {tokens: conf/rest-server.properties} }" >> $GREMLIN_SERVER_CONF -$TRAVIS_DIR/start-server.sh $SERVER_DIR $BACKEND || (cat $SERVER_DIR/logs/hugegraph-server.log && exit 1) +# start server +$TRAVIS_DIR/start-server.sh $SERVER_DIR $BACKEND $JACOCO_PORT || (cat $SERVER_DIR/logs/hugegraph-server.log && exit 1) # run api-test mvn test -P api-test,$BACKEND || (cat $SERVER_DIR/logs/hugegraph-server.log && exit 1) -$TRAVIS_DIR/build-report.sh $BACKEND -$TRAVIS_DIR/stop-server.sh + +$TRAVIS_DIR/build-report.sh $BACKEND $JACOCO_PORT + +# stop server +$TRAVIS_DIR/stop-server.sh $SERVER_DIR diff --git a/hugegraph-dist/src/assembly/travis/start-server.sh b/hugegraph-dist/src/assembly/travis/start-server.sh index 616dcd0649..3a94a0dae0 100755 --- a/hugegraph-dist/src/assembly/travis/start-server.sh +++ b/hugegraph-dist/src/assembly/travis/start-server.sh @@ -6,14 +6,21 @@ HOME_DIR=$(pwd) TRAVIS_DIR=$(dirname $0) BASE_DIR=$1 BACKEND=$2 +JACOCO_PORT=$3 + +JACOCO_JAR=${HOME_DIR}/${TRAVIS_DIR}/jacocoagent.jar + BIN=$BASE_DIR/bin CONF=$BASE_DIR/conf/graphs/hugegraph.properties REST_CONF=$BASE_DIR/conf/rest-server.properties GREMLIN_CONF=$BASE_DIR/conf/gremlin-server.yaml -declare -A backend_serializer_map=(["memory"]="text" ["cassandra"]="cassandra" \ - ["scylladb"]="scylladb" ["mysql"]="mysql" \ - ["hbase"]="hbase" ["rocksdb"]="binary" \ +declare -A backend_serializer_map=(["memory"]="text" \ + ["cassandra"]="cassandra" \ + ["scylladb"]="scylladb" \ + ["mysql"]="mysql" \ + ["hbase"]="hbase" \ + ["rocksdb"]="binary" \ ["postgresql"]="postgresql") SERIALIZER=${backend_serializer_map[$BACKEND]} @@ -37,5 +44,9 @@ fi # Append schema.sync_deletion=true to config file echo "schema.sync_deletion=true" >> $CONF -AGENT_JAR=${HOME_DIR}/${TRAVIS_DIR}/jacocoagent.jar -echo -e "pa" | $BIN/init-store.sh && $BIN/start-hugegraph.sh -j "-javaagent:${AGENT_JAR}=includes=*,port=36320,destfile=jacoco-it.exec,output=tcpserver" -v +JACOCO_OPTION="" +if [ -n "$JACOCO_PORT" ]; then + JACOCO_OPTION="-javaagent:${JACOCO_JAR}=includes=*,port=${JACOCO_PORT},destfile=jacoco-it.exec,output=tcpserver" +fi + +echo -e "pa" | $BIN/init-store.sh && $BIN/start-hugegraph.sh -j "$JACOCO_OPTION" -t 60 -v diff --git a/hugegraph-dist/src/assembly/travis/stop-server.sh b/hugegraph-dist/src/assembly/travis/stop-server.sh index a966c88c78..584ef27c5d 100755 --- a/hugegraph-dist/src/assembly/travis/stop-server.sh +++ b/hugegraph-dist/src/assembly/travis/stop-server.sh @@ -2,8 +2,8 @@ set -ev -VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout` -BASE_DIR=hugegraph-$VERSION +BASE_DIR=$1 BIN=$BASE_DIR/bin +VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout` $BIN/stop-hugegraph.sh