diff --git a/.travis.yml b/.travis.yml index 0f6951c044..4b5c57ad72 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ language: java jdk: - - oraclejdk8 + - openjdk8 -dist: trusty +dist: xenial sudo: required diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java index 11aa6a0361..cfa1aec501 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java @@ -249,7 +249,8 @@ public BackendFeatures backendStoreFeatures() { @Override public void serverStarted(Id serverId, NodeRole serverRole) { - LOG.info("Init server info for graph '{}'...", this.name); + LOG.info("Init server info [{}-{}] for graph '{}'...", + serverId, serverRole, this.name); this.serverInfoManager().initServerInfo(serverId, serverRole); LOG.info("Restoring incomplete tasks for graph '{}'...", this.name); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/ServerInfoManager.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/ServerInfoManager.java index 17aa4fabb5..807673eae2 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/ServerInfoManager.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/ServerInfoManager.java @@ -163,6 +163,8 @@ public synchronized void initServerInfo(Id server, NodeRole role) { HugeServerInfo serverInfo = new HugeServerInfo(server, role); serverInfo.maxLoad(this.calcMaxLoad()); this.save(serverInfo); + + LOG.info("Init server info: {}", serverInfo); } public Id selfServerId() { @@ -191,14 +193,11 @@ public void heartbeat() { this.save(serverInfo); } - public void decreaseLoad(int load) { - try { - HugeServerInfo serverInfo = this.selfServerInfo(); - serverInfo.increaseLoad(-load); - this.save(serverInfo); - } catch (Throwable t) { - LOG.error("Exception occurred when decrease server load", t); - } + public synchronized void decreaseLoad(int load) { + assert load > 0 : load; + HugeServerInfo serverInfo = this.selfServerInfo(); + serverInfo.increaseLoad(-load); + this.save(serverInfo); } public int calcMaxLoad() { @@ -345,6 +344,7 @@ private HugeServerInfo removeServerInfo(Id server) { if (server == null) { return null; } + LOG.info("Remove server info: {}", server); return this.call(() -> { Iterator vertices = this.tx().queryVertices(server); Vertex vertex = QueryResults.one(vertices); @@ -356,11 +356,11 @@ private HugeServerInfo removeServerInfo(Id server) { }); } - public void updateServerInfos(Collection serverInfos) { + protected void updateServerInfos(Collection serverInfos) { this.save(serverInfos); } - public Collection allServerInfos() { + protected Collection allServerInfos() { Iterator infos = this.serverInfos(NO_LIMIT, null); try (ListIterator iter = new ListIterator<>( MAX_SERVERS, infos)) { @@ -370,11 +370,11 @@ public Collection allServerInfos() { } } - public Iterator serverInfos(String page) { + protected Iterator serverInfos(String page) { return this.serverInfos(ImmutableMap.of(), PAGE_SIZE, page); } - public Iterator serverInfos(long limit, String page) { + protected Iterator serverInfos(long limit, String page) { return this.serverInfos(ImmutableMap.of(), limit, page); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java index 3bbf723328..0db34f6fdc 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java @@ -444,10 +444,14 @@ protected void cancelTasksOnWorker(Id server) { protected void taskDone(HugeTask task) { this.remove(task); - this.serverManager().decreaseLoad(task.load()); - - LOG.debug("Task '{}' done on server '{}'", - task.id(), this.serverManager().selfServerId()); + Id selfServerId = this.serverManager().selfServerId(); + try { + this.serverManager().decreaseLoad(task.load()); + } catch (Throwable e) { + LOG.error("Failed to decrease load for task '{}' on server '{}'", + task.id(), selfServerId, e); + } + LOG.debug("Task '{}' done on server '{}'", task.id(), selfServerId); } protected void remove(HugeTask task) { diff --git a/hugegraph-dist/src/assembly/travis/install-backend.sh b/hugegraph-dist/src/assembly/travis/install-backend.sh index 7d12ac8757..75a53e612c 100755 --- a/hugegraph-dist/src/assembly/travis/install-backend.sh +++ b/hugegraph-dist/src/assembly/travis/install-backend.sh @@ -8,12 +8,23 @@ if [ ! -d $HOME/downloads ]; then mkdir $HOME/downloads fi -if [[ "$BACKEND" == "cassandra" || "$BACKEND" == "scylladb" ]]; then - $TRAVIS_DIR/install-cassandra.sh -elif [[ "$BACKEND" == "hbase" ]]; then - $TRAVIS_DIR/install-hbase.sh -elif [[ "$BACKEND" == "mysql" ]]; then - $TRAVIS_DIR/install-mysql.sh -elif [[ "$BACKEND" == "postgresql" ]]; then - $TRAVIS_DIR/install-postgresql.sh -fi +case $BACKEND in + cassandra) + $TRAVIS_DIR/install-cassandra.sh + ;; + scylladb) + $TRAVIS_DIR/install-scylladb.sh + ;; + hbase) + $TRAVIS_DIR/install-hbase.sh + ;; + mysql) + $TRAVIS_DIR/install-mysql.sh + ;; + postgresql) + $TRAVIS_DIR/install-postgresql.sh + ;; + *) + # don't need to install for other backends + ;; +esac diff --git a/hugegraph-dist/src/assembly/travis/install-cassandra.sh b/hugegraph-dist/src/assembly/travis/install-cassandra.sh index 9e49d27999..7734e78277 100755 --- a/hugegraph-dist/src/assembly/travis/install-cassandra.sh +++ b/hugegraph-dist/src/assembly/travis/install-cassandra.sh @@ -3,27 +3,28 @@ set -ev TRAVIS_DIR=`dirname $0` -CASSA_DOWNLOAD_ADDRESS="http://archive.apache.org/dist/cassandra" -CASSA_VERSION="3.10" -CASSA_PACKAGE="apache-cassandra-${CASSA_VERSION}" -CASSA_TAR="${CASSA_PACKAGE}-bin.tar.gz" +CASS_DOWNLOAD_ADDRESS="http://archive.apache.org/dist/cassandra" +CASS_VERSION="3.10" +CASS_PACKAGE="apache-cassandra-${CASS_VERSION}" +CASS_TAR="${CASS_PACKAGE}-bin.tar.gz" +CASS_CONF="${CASS_PACKAGE}/conf/cassandra.yaml" # download cassandra -if [ ! -f $HOME/downloads/${CASSA_TAR} ]; then - wget -q -O $HOME/downloads/${CASSA_TAR} ${CASSA_DOWNLOAD_ADDRESS}/${CASSA_VERSION}/${CASSA_TAR} +if [ ! -f $HOME/downloads/${CASS_TAR} ]; then + wget -q -O $HOME/downloads/${CASS_TAR} ${CASS_DOWNLOAD_ADDRESS}/${CASS_VERSION}/${CASS_TAR} fi # decompress cassandra -cp $HOME/downloads/${CASSA_TAR} ${CASSA_TAR} && tar xzf ${CASSA_TAR} +cp $HOME/downloads/${CASS_TAR} ${CASS_TAR} && tar -xzf ${CASS_TAR} # using tmpfs for the Cassandra data directory reduces travis test runtime sudo mkdir /mnt/ramdisk sudo mount -t tmpfs -o size=1024m tmpfs /mnt/ramdisk -sudo ln -s /mnt/ramdisk $CASSA_PACKAGE/data +sudo ln -s /mnt/ramdisk $CASS_PACKAGE/data # config cassandra -sed -i "s/batch_size_warn_threshold_in_kb:.*/batch_size_warn_threshold_in_kb: 10240/g" ${CASSA_PACKAGE}/conf/cassandra.yaml -sed -i "s/batch_size_fail_threshold_in_kb:.*/batch_size_fail_threshold_in_kb: 10240/g" ${CASSA_PACKAGE}/conf/cassandra.yaml +sed -i "s/batch_size_warn_threshold_in_kb:.*/batch_size_warn_threshold_in_kb: 10240/g" $CASS_CONF +sed -i "s/batch_size_fail_threshold_in_kb:.*/batch_size_fail_threshold_in_kb: 10240/g" $CASS_CONF # start cassandra service -sh ${CASSA_PACKAGE}/bin/cassandra +sh ${CASS_PACKAGE}/bin/cassandra diff --git a/hugegraph-dist/src/assembly/travis/install-scylladb.sh b/hugegraph-dist/src/assembly/travis/install-scylladb.sh new file mode 100755 index 0000000000..1933baf5e6 --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/install-scylladb.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -ev + +TRAVIS_DIR=`dirname $0` +# reference: +# https://www.scylladb.com/download/?platform=ubuntu-16.04&version=scylla-4.4#open-source +# https://github.com/scylladb/gocqlx/commit/c7f0483dd30b1c7ad1972ea135528dc95cc4ce32 +DOWNLOAD_ADDRESS="http://downloads.scylladb.com/deb/ubuntu/scylla-4.4-$(lsb_release -s -c).list" +SCYLLA_PORT=9042 +SCYLLA_CONF="/etc/scylla/scylla.yaml" +SCYLLA_OPTS="--network-stack posix --enable-in-memory-data-store 1 --developer-mode 1" +SCYLLA_OPTS_LOG="--log-to-stdout 1 --default-log-level info" + +# download and install scylladb +sudo apt-get install apt-transport-https +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 5e08fbd8b5d6ec9c +sudo curl -L --output /etc/apt/sources.list.d/scylla.list $DOWNLOAD_ADDRESS + +sudo apt-get update +sudo apt-get install scylla + +# config scylladb +sudo sed -i "s/batch_size_warn_threshold_in_kb:.*/batch_size_warn_threshold_in_kb: 10240/g" $SCYLLA_CONF +sudo sed -i "s/batch_size_fail_threshold_in_kb:.*/batch_size_fail_threshold_in_kb: 10240/g" $SCYLLA_CONF +cat $SCYLLA_CONF + +sudo scylla_dev_mode_setup --developer-mode 1 +cat /etc/scylla.d/dev-mode.conf + +# setup scylladb with scylla_setup by expect +sudo apt-get install expect +sudo expect < graphs = openGraphs("schema_g1", "schema_g2"); for (HugeGraph graph : graphs) { graph.initBackend(); - graph.serverStarted(IdGenerator.of("server1"), NodeRole.MASTER); } HugeGraph g1 = graphs.get(0); HugeGraph g2 = graphs.get(1); + g1.serverStarted(IdGenerator.of("server-g1c"), NodeRole.MASTER); + g2.serverStarted(IdGenerator.of("server-g2c"), NodeRole.MASTER); g1.schema().propertyKey("id").asInt().create(); g2.schema().propertyKey("id").asText().create(); @@ -392,6 +393,7 @@ private static Configuration buildConfig(String graphName) { String walPath = config.getString(RocksDBOptions.WAL_PATH.name()); config.setProperty(RocksDBOptions.WAL_PATH.name(), Paths.get(walPath, graphName).toString()); + return config; } }