diff --git a/.gitignore b/.gitignore index 0a782a4..f3d9c0d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,17 @@ */.DStore docker/data/ docker/logs/ -nebula/.hgignore +docker/secret/* nebula/store/ pom.xml pom.xml.asc *.class /classes/ -nebula/target +/target/ /checkouts/ .lein-deps-sum .lein-repl-history .lein-plugins/ .lein-failures .nrepl-port -.cpcache/ +.cpcache/ \ No newline at end of file diff --git a/README.md b/README.md index 9a8e437..20a0345 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,13 @@ Jepsen testing framework for Nebula Graph ## Get started -- Run dockers -change directory to ```nebula-jepsen/``` -run ```./start.sh``` -wait until all nodes started and you are now in the container of control node. -follow the instructions showed on welcome page like below: -- Run the test -```cd nebula``` -```lein run test``` +- Run docker containers + - Change directory to ```nebula-jepsen/```, run ```./up.sh``` to start + - A space named 'test' will be automatically created, with default setting of ```partition_num=5, replica_factor=3```, can be changed by editting ```nebula-jepsen/docker/create_space.txt``` + - Wait until all nodes started and you are now in the container of control node. + + +- Run the test + ```cd nebula``` + ```lein run test ``` + diff --git a/docker/Dockerfile.node b/docker/Dockerfile.node new file mode 100644 index 0000000..f11869f --- /dev/null +++ b/docker/Dockerfile.node @@ -0,0 +1,12 @@ +FROM vesoft/nebula-storaged:nightly + +COPY run.sh / +COPY start-stop-daemon /usr/bin/ + +RUN yum install -y openssh* sudo net-tools wget curl vim iptables initscripts make gcc gcc-c++ ncurses-devel + +COPY ./conf/sshd_config /etc/ssh/sshd_config + +RUN service sshd restart + +ENTRYPOINT ["/run.sh"] \ No newline at end of file diff --git a/docker/conf/1/nebula-storaged.conf b/docker/conf/1/nebula-storaged.conf new file mode 100644 index 0000000..43b9465 --- /dev/null +++ b/docker/conf/1/nebula-storaged.conf @@ -0,0 +1,58 @@ +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=/root/pids/nebula-storaged.pid + +########## logging ########## +# The directory to host logging files, which must already exists +--log_dir=/logs +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 + +########## networking ########## +# Meta server address +--meta_server_addrs=172.28.1.1:45500 +# Local ip +--local_ip=172.28.2.1 +# Storage daemon listening port +--port=44500 +# HTTP service ip +--ws_ip=172.28.2.1 +# HTTP service port +--ws_http_port=12000 +# HTTP2 service port +--ws_h2_port=12002 + +########## storage ########## +# Root data path, multiple paths should be splitted by comma. +# One path per instance, if --engine_type is `rocksdb' +--data_path=/data/storage +# The type of part manager, [memory | meta] +--part_man_type=memory +# The default reserved bytes for one batch operation +--rocksdb_batch_size=4096 +# The default block cache size used in BlockBasedTable. +# The unit is MB. +--rocksdb_block_cache=4 +# The type of storage engine, `rocksdb', `memory', etc. +--engine_type=rocksdb +# The type of part, `simple', `consensus'... +--part_type=simple + +############## rocksdb Options ############## +--rocksdb_disable_wal=true +# rocksdb DBOptions in json, each name and value of option is a string, given as "option_name":"option_value" separated by comma +--rocksdb_db_options={} +# rocksdb ColumnFamilyOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_column_family_options={} +# rocksdb BlockBasedTableOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_block_based_table_options={} + +--storage_kv_mode=true +--raft_heartbeat_interval_secs=1 +--load_data_interval_secs=5 diff --git a/docker/conf/2/nebula-storaged.conf b/docker/conf/2/nebula-storaged.conf new file mode 100644 index 0000000..fb6fb67 --- /dev/null +++ b/docker/conf/2/nebula-storaged.conf @@ -0,0 +1,58 @@ +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=/root/pids/nebula-storaged.pid + +########## logging ########## +# The directory to host logging files, which must already exists +--log_dir=/logs +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 + +########## networking ########## +# Meta server address +--meta_server_addrs=172.28.1.1:45500 +# Local ip +--local_ip=172.28.2.2 +# Storage daemon listening port +--port=44500 +# HTTP service ip +--ws_ip=172.28.2.2 +# HTTP service port +--ws_http_port=12000 +# HTTP2 service port +--ws_h2_port=12002 + +########## storage ########## +# Root data path, multiple paths should be splitted by comma. +# One path per instance, if --engine_type is `rocksdb' +--data_path=/data/storage +# The type of part manager, [memory | meta] +--part_man_type=memory +# The default reserved bytes for one batch operation +--rocksdb_batch_size=4096 +# The default block cache size used in BlockBasedTable. +# The unit is MB. +--rocksdb_block_cache=4 +# The type of storage engine, `rocksdb', `memory', etc. +--engine_type=rocksdb +# The type of part, `simple', `consensus'... +--part_type=simple + +############## rocksdb Options ############## +--rocksdb_disable_wal=true +# rocksdb DBOptions in json, each name and value of option is a string, given as "option_name":"option_value" separated by comma +--rocksdb_db_options={} +# rocksdb ColumnFamilyOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_column_family_options={} +# rocksdb BlockBasedTableOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_block_based_table_options={} + +--storage_kv_mode=true +--raft_heartbeat_interval_secs=1 +--load_data_interval_secs=5 \ No newline at end of file diff --git a/docker/conf/3/nebula-storaged.conf b/docker/conf/3/nebula-storaged.conf new file mode 100644 index 0000000..388ae20 --- /dev/null +++ b/docker/conf/3/nebula-storaged.conf @@ -0,0 +1,58 @@ +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=/root/pids/nebula-storaged.pid + +########## logging ########## +# The directory to host logging files, which must already exists +--log_dir=/logs +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 + +########## networking ########## +# Meta server address +--meta_server_addrs=172.28.1.1:45500 +# Local ip +--local_ip=172.28.2.3 +# Storage daemon listening port +--port=44500 +# HTTP service ip +--ws_ip=172.28.2.3 +# HTTP service port +--ws_http_port=12000 +# HTTP2 service port +--ws_h2_port=12002 + +########## storage ########## +# Root data path, multiple paths should be splitted by comma. +# One path per instance, if --engine_type is `rocksdb' +--data_path=/data/storage +# The type of part manager, [memory | meta] +--part_man_type=memory +# The default reserved bytes for one batch operation +--rocksdb_batch_size=4096 +# The default block cache size used in BlockBasedTable. +# The unit is MB. +--rocksdb_block_cache=4 +# The type of storage engine, `rocksdb', `memory', etc. +--engine_type=rocksdb +# The type of part, `simple', `consensus'... +--part_type=simple + +############## rocksdb Options ############## +--rocksdb_disable_wal=true +# rocksdb DBOptions in json, each name and value of option is a string, given as "option_name":"option_value" separated by comma +--rocksdb_db_options={} +# rocksdb ColumnFamilyOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_column_family_options={} +# rocksdb BlockBasedTableOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_block_based_table_options={} + +--storage_kv_mode=true +--raft_heartbeat_interval_secs=1 +--load_data_interval_secs=5 \ No newline at end of file diff --git a/docker/conf/4/nebula-storaged.conf b/docker/conf/4/nebula-storaged.conf new file mode 100644 index 0000000..7c1b6d7 --- /dev/null +++ b/docker/conf/4/nebula-storaged.conf @@ -0,0 +1,58 @@ +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=/root/pids/nebula-storaged.pid + +########## logging ########## +# The directory to host logging files, which must already exists +--log_dir=/logs +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 + +########## networking ########## +# Meta server address +--meta_server_addrs=172.28.1.1:45500 +# Local ip +--local_ip=172.28.2.4 +# Storage daemon listening port +--port=44500 +# HTTP service ip +--ws_ip=172.28.2.4 +# HTTP service port +--ws_http_port=12000 +# HTTP2 service port +--ws_h2_port=12002 + +########## storage ########## +# Root data path, multiple paths should be splitted by comma. +# One path per instance, if --engine_type is `rocksdb' +--data_path=/data/storage +# The type of part manager, [memory | meta] +--part_man_type=memory +# The default reserved bytes for one batch operation +--rocksdb_batch_size=4096 +# The default block cache size used in BlockBasedTable. +# The unit is MB. +--rocksdb_block_cache=4 +# The type of storage engine, `rocksdb', `memory', etc. +--engine_type=rocksdb +# The type of part, `simple', `consensus'... +--part_type=simple + +############## rocksdb Options ############## +--rocksdb_disable_wal=true +# rocksdb DBOptions in json, each name and value of option is a string, given as "option_name":"option_value" separated by comma +--rocksdb_db_options={} +# rocksdb ColumnFamilyOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_column_family_options={} +# rocksdb BlockBasedTableOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_block_based_table_options={} + +--storage_kv_mode=true +--raft_heartbeat_interval_secs=1 +--load_data_interval_secs=5 \ No newline at end of file diff --git a/docker/conf/5/nebula-storaged.conf b/docker/conf/5/nebula-storaged.conf new file mode 100644 index 0000000..a9cbaa4 --- /dev/null +++ b/docker/conf/5/nebula-storaged.conf @@ -0,0 +1,58 @@ +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=/root/pids/nebula-storaged.pid + +########## logging ########## +# The directory to host logging files, which must already exists +--log_dir=/logs +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 + +########## networking ########## +# Meta server address +--meta_server_addrs=172.28.1.1:45500 +# Local ip +--local_ip=172.28.2.5 +# Storage daemon listening port +--port=44500 +# HTTP service ip +--ws_ip=172.28.2.5 +# HTTP service port +--ws_http_port=12000 +# HTTP2 service port +--ws_h2_port=12002 + +########## storage ########## +# Root data path, multiple paths should be splitted by comma. +# One path per instance, if --engine_type is `rocksdb' +--data_path=/data/storage +# The type of part manager, [memory | meta] +--part_man_type=memory +# The default reserved bytes for one batch operation +--rocksdb_batch_size=4096 +# The default block cache size used in BlockBasedTable. +# The unit is MB. +--rocksdb_block_cache=4 +# The type of storage engine, `rocksdb', `memory', etc. +--engine_type=rocksdb +# The type of part, `simple', `consensus'... +--part_type=simple + +############## rocksdb Options ############## +--rocksdb_disable_wal=true +# rocksdb DBOptions in json, each name and value of option is a string, given as "option_name":"option_value" separated by comma +--rocksdb_db_options={} +# rocksdb ColumnFamilyOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_column_family_options={} +# rocksdb BlockBasedTableOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_block_based_table_options={} + +--storage_kv_mode=true +--raft_heartbeat_interval_secs=1 +--load_data_interval_secs=5 \ No newline at end of file diff --git a/docker/conf/sshd_config b/docker/conf/sshd_config new file mode 100644 index 0000000..50cb02c --- /dev/null +++ b/docker/conf/sshd_config @@ -0,0 +1,139 @@ +# $OpenBSD: sshd_config,v 1.100 2016/08/15 12:32:04 naddy Exp $ + +# This is the sshd server system-wide configuration file. See +# sshd_config(5) for more information. + +# This sshd was compiled with PATH=/usr/local/bin:/usr/bin + +# The strategy used for options in the default sshd_config shipped with +# OpenSSH is to specify options with their default value where +# possible, but leave them commented. Uncommented options override the +# default value. + +# If you want to change the port on a SELinux system, you have to tell +# SELinux about this change. +# semanage port -a -t ssh_port_t -p tcp #PORTNUMBER +# +Port 22 +#AddressFamily any +#ListenAddress 0.0.0.0 +#ListenAddress :: + +HostKey /etc/ssh/ssh_host_rsa_key +#HostKey /etc/ssh/ssh_host_dsa_key +HostKey /etc/ssh/ssh_host_ecdsa_key +HostKey /etc/ssh/ssh_host_ed25519_key + +# Ciphers and keying +#RekeyLimit default none + +# Logging +#SyslogFacility AUTH +SyslogFacility AUTHPRIV +#LogLevel INFO + +# Authentication: + +#LoginGraceTime 2m +PermitRootLogin yes +#StrictModes yes +#MaxAuthTries 6 +#MaxSessions 10 + +PubkeyAuthentication yes + +# The default is to check both .ssh/authorized_keys and .ssh/authorized_keys2 +# but this is overridden so installations will only check .ssh/authorized_keys +AuthorizedKeysFile .ssh/authorized_keys + +#AuthorizedPrincipalsFile none + +#AuthorizedKeysCommand none +#AuthorizedKeysCommandUser nobody + +# For this to work you will also need host keys in /etc/ssh/ssh_known_hosts +#HostbasedAuthentication no +# Change to yes if you don't trust ~/.ssh/known_hosts for +# HostbasedAuthentication +#IgnoreUserKnownHosts no +# Don't read the user's ~/.rhosts and ~/.shosts files +#IgnoreRhosts yes + +# To disable tunneled clear text passwords, change to no here! +PasswordAuthentication yes +#PermitEmptyPasswords no +PasswordAuthentication yes + +# Change to no to disable s/key passwords +#ChallengeResponseAuthentication yes +ChallengeResponseAuthentication no + +# Kerberos options +#KerberosAuthentication no +#KerberosOrLocalPasswd yes +#KerberosTicketCleanup yes +#KerberosGetAFSToken no +#KerberosUseKuserok yes + +# GSSAPI options +GSSAPIAuthentication yes +GSSAPICleanupCredentials no +#GSSAPIStrictAcceptorCheck yes +#GSSAPIKeyExchange no +#GSSAPIEnablek5users no + +# Set this to 'yes' to enable PAM authentication, account processing, +# and session processing. If this is enabled, PAM authentication will +# be allowed through the ChallengeResponseAuthentication and +# PasswordAuthentication. Depending on your PAM configuration, +# PAM authentication via ChallengeResponseAuthentication may bypass +# the setting of "PermitRootLogin without-password". +# If you just want the PAM account and session checks to run without +# PAM authentication, then enable this but set PasswordAuthentication +# and ChallengeResponseAuthentication to 'no'. +# WARNING: 'UsePAM no' is not supported in Red Hat Enterprise Linux and may cause several +# problems. +UsePAM yes + +#AllowAgentForwarding yes +#AllowTcpForwarding yes +#GatewayPorts no +X11Forwarding yes +#X11DisplayOffset 10 +#X11UseLocalhost yes +#PermitTTY yes +#PrintMotd yes +#PrintLastLog yes +#TCPKeepAlive yes +#UseLogin no +#UsePrivilegeSeparation sandbox +#PermitUserEnvironment no +#Compression delayed +#ClientAliveInterval 0 +#ClientAliveCountMax 3 +#ShowPatchLevel no +#UseDNS yes +#PidFile /var/run/sshd.pid +#MaxStartups 10:30:100 +#PermitTunnel no +#ChrootDirectory none +#VersionAddendum none + +# no default banner path +#Banner none + +# Accept locale-related environment variables +AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES +AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT +AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE +AcceptEnv XMODIFIERS + +# override default of no subsystems +Subsystem sftp /usr/libexec/openssh/sftp-server + +# Example of overriding settings on a per-user basis +#Match User anoncvs +# X11Forwarding no +# AllowTcpForwarding no +# PermitTTY no +# ForceCommand cvs server diff --git a/create_space.txt b/docker/create_space.txt similarity index 100% rename from create_space.txt rename to docker/create_space.txt diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index c01f970..e08956b 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -33,170 +33,115 @@ services: restart: on-failure n1: - image: wfystx/nebula-node:v4 + image: wfystx/nebula-node:latest container_name: nebula-n1 hostname: n1 + env_file: ./secret/node.env privileged: true environment: USER: root - command: - - --meta_server_addrs=172.28.1.1:45500 - - --local_ip=172.28.2.1 - - --ws_ip=172.28.2.1 - - --port=44500 - - --data_path=/data/storage - - --log_dir=/logs - - --storage_kv_mode=true depends_on: - metad - healthcheck: - test: ["CMD", "curl", "-f", "http://172.28.2.1:12000/status"] - interval: 1m30s - timeout: 10s - retries: 3 - start_period: 20s ports: - 12000 - 12002 volumes: - ./data/n1:/data/storage - ./logs/n1:/logs + - ./run.sh:/run.sh + - ./conf/1/nebula-storaged.conf:/usr/local/nebula/etc/nebula-storaged.conf networks: nebula-net: ipv4_address: 172.28.2.1 restart: on-failure n2: - image: wfystx/nebula-node:v4 + image: wfystx/nebula-node:latest container_name: nebula-n2 hostname: n2 + env_file: ./secret/node.env privileged: true environment: USER: root - command: - - --meta_server_addrs=172.28.1.1:45500 - - --local_ip=172.28.2.2 - - --ws_ip=172.28.2.2 - - --port=44500 - - --data_path=/data/storage - - --log_dir=/logs - - --storage_kv_mode=true depends_on: - metad - healthcheck: - test: ["CMD", "curl", "-f", "http://172.28.2.2:12000/status"] - interval: 1m30s - timeout: 10s - retries: 3 - start_period: 20s ports: - 12000 - 12002 volumes: - ./data/n2:/data/storage - ./logs/n2:/logs + - ./run.sh:/run.sh + - ./conf/2/nebula-storaged.conf:/usr/local/nebula/etc/nebula-storaged.conf networks: nebula-net: ipv4_address: 172.28.2.2 restart: on-failure n3: - image: wfystx/nebula-node:v4 + image: wfystx/nebula-node:latest container_name: nebula-n3 hostname: n3 + env_file: ./secret/node.env privileged: true environment: USER: root - command: - - --meta_server_addrs=172.28.1.1:45500 - - --local_ip=172.28.2.3 - - --ws_ip=172.28.2.3 - - --port=44500 - - --data_path=/data/storage - - --log_dir=/logs - - --storage_kv_mode=true depends_on: - metad - healthcheck: - test: ["CMD", "curl", "-f", "http://172.28.2.3:12000/status"] - interval: 1m30s - timeout: 10s - retries: 3 - start_period: 20s ports: - 12000 - 12002 volumes: - ./data/n3:/data/storage - ./logs/n3:/logs + - ./run.sh:/run.sh + - ./conf/3/nebula-storaged.conf:/usr/local/nebula/etc/nebula-storaged.conf networks: nebula-net: ipv4_address: 172.28.2.3 restart: on-failure n4: - image: wfystx/nebula-node:v4 + image: wfystx/nebula-node:latest container_name: nebula-n4 hostname: n4 + env_file: ./secret/node.env privileged: true environment: USER: root - command: - - --meta_server_addrs=172.28.1.1:45500 - - --local_ip=172.28.2.4 - - --ws_ip=172.28.2.4 - - --port=44500 - - --data_path=/data/storage - - --log_dir=/logs - - --storage_kv_mode=true depends_on: - metad - healthcheck: - test: ["CMD", "curl", "-f", "http://172.28.2.4:12000/status"] - interval: 1m30s - timeout: 10s - retries: 3 - start_period: 20s ports: - 12000 - 12002 volumes: - ./data/n4:/data/storage - ./logs/n4:/logs + - ./run.sh:/run.sh + - ./conf/4/nebula-storaged.conf:/usr/local/nebula/etc/nebula-storaged.conf networks: nebula-net: ipv4_address: 172.28.2.4 restart: on-failure n5: - image: wfystx/nebula-node:v4 + image: wfystx/nebula-node:latest container_name: nebula-n5 hostname: n5 + env_file: ./secret/node.env privileged: true environment: USER: root - command: - - --meta_server_addrs=172.28.1.1:45500 - - --local_ip=172.28.2.5 - - --ws_ip=172.28.2.5 - - --port=44500 - - --data_path=/data/storage - - --log_dir=/logs - - --storage_kv_mode=true depends_on: - metad - healthcheck: - test: ["CMD", "curl", "-f", "http://172.28.2.5:12000/status"] - interval: 1m30s - timeout: 10s - retries: 3 - start_period: 20s ports: - 12000 - 12002 volumes: - ./data/n5:/data/storage - ./logs/n5:/logs + - ./run.sh:/run.sh + - ./conf/5/nebula-storaged.conf:/usr/local/nebula/etc/nebula-storaged.conf networks: nebula-net: ipv4_address: 172.28.2.5 diff --git a/docker/init.sh b/docker/init.sh index 8e4bd46..90cce78 100755 --- a/docker/init.sh +++ b/docker/init.sh @@ -1,20 +1,28 @@ -#!/bin/bash -password=root -echo 'y' | ssh-keygen -f $HOME/.ssh/id_rsa -t rsa -N '' +#!/bin/sh -echo "" > ~/.ssh/known_hosts +: "${SSH_PRIVATE_KEY?SSH_PRIVATE_KEY is empty, please use up.sh}" +: "${SSH_PUBLIC_KEY?SSH_PUBLIC_KEY is empty, please use up.sh}" -for f in $(seq 1 5) - do - ssh-keyscan -t rsa n$f >> ~/.ssh/known_hosts - expect <<-EOF - set timeout 5 - spawn ssh-copy-id -i n$f - expect { - "yes/no" { send "yes\n";exp_continue } - "password:" { send "$password\n" } - } - interact - expect eof -EOF -done \ No newline at end of file +if [ ! -f ~/.ssh/known_hosts ]; then + mkdir -m 700 ~/.ssh + echo $SSH_PRIVATE_KEY | perl -p -e 's/↩/\n/g' > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + echo $SSH_PUBLIC_KEY > ~/.ssh/id_rsa.pub + echo > ~/.ssh/known_hosts + for f in $(seq 1 5);do + ssh-keyscan -t rsa n$f >> ~/.ssh/known_hosts + done +fi + +# TODO: assert that SSH_PRIVATE_KEY==~/.ssh/id_rsa + +cat < Found authorized keys" + mkdir -p /root/.ssh + chmod 700 /root/.ssh + touch /root/.ssh/authorized_keys + chmod 600 /root/.ssh/authorized_keys + IFS=$'\n' + arr=$(echo ${AUTHORIZED_KEYS} | tr "," "\n") + for x in $arr + do + x=$(echo $x |sed -e 's/^ *//' -e 's/ *$//') + cat /root/.ssh/authorized_keys | grep "$x" >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "=> Adding public key to /root/.ssh/authorized_keys: $x" + echo "$x" >> /root/.ssh/authorized_keys + fi + done fi -service sshd restart \ No newline at end of file + +echo 'root' | passwd root --stdin + +/usr/local/nebula/scripts/nebula-storaged.service -c /usr/local/nebula/etc/nebula-storaged.conf start + +exec /usr/sbin/sshd -D + +tail -f /dev/null \ No newline at end of file diff --git a/docker/start-stop-daemon b/docker/start-stop-daemon new file mode 100755 index 0000000..02f4fdc Binary files /dev/null and b/docker/start-stop-daemon differ diff --git a/docker/up.sh b/docker/up.sh new file mode 100755 index 0000000..bd84ca8 --- /dev/null +++ b/docker/up.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +set -e # exit on an error + +exists() { + type $1 > /dev/null 2>&1 +} + +WARNING(){ + /bin/echo -e "\e[101m\e[97m[WARNING]\e[49m\e[39m $@" +} + +INFO(){ + /bin/echo -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@" +} + +ERROR(){ + /bin/echo -e "\e[101m\e[97m[ERROR]\e[49m\e[39m $@" +} + +function init() { + if [ -d "data/" ];then + rm -rf data/* + fi + if [ -d "logs/" ];then + rm -rf logs/* + fi + + docker-compose -f docker-compose.yaml up -d +} + +if [ ! -d "secret" ]; then + mkdir secret +fi + +# Generate SSH keys for the control node +if [ ! -f ./secret/node.env ]; then + INFO "Generating key pair" + ssh-keygen -t rsa -N "" -f ./secret/id_rsa + + INFO "Generating ./secret/control.env" + echo "# generated by nebula-jepsen/docker/up.sh" > ./secret/control.env + echo "# NOTE: \\n is expressed as ↩" >> ./secret/control.env + echo SSH_PRIVATE_KEY="$(cat ./secret/id_rsa | perl -p -e "s/\n/↩/g")" >> ./secret/control.env + echo SSH_PUBLIC_KEY=$(cat ./secret/id_rsa.pub) >> ./secret/control.env + + INFO "Generating ./secret/node.env" + echo "# generated by nebula-jepsen/docker/up.sh" > ./secret/node.env + echo ROOT_PASS=root >> ./secret/node.env + echo AUTHORIZED_KEYS=$(cat ./secret/id_rsa.pub) >> ./secret/node.env +else + INFO "No need to generate key pair" +fi + +exists docker || { ERROR "Please install docker (https://docs.docker.com/engine/installation/)"; exit 1; } +exists docker-compose || { ERROR "Please install docker-compose (https://docs.docker.com/compose/install/)"; exit 1; } + +init + +sleep 2s + +docker run -idt --name nebula-console --network=host vesoft/nebula-console:nightly --addr=127.0.0.1 --port=3699 +cat create_space.txt | docker exec -i nebula-console /bin/bash +docker stop nebula-console +docker rm nebula-console + +docker run --env-file ./secret/control.env --name nebula-control --privileged=true -it --net docker_nebula-net \ + --link nebula-n1:n1 --link nebula-n2:n2 --link nebula-n3:n3 --link nebula-n4:n4 \ + --link nebula-n5:n5 -v $(pwd)/../:/jepsen_dev wfystx/nebula-control:latest /bin/bash + diff --git a/nebula/.gitignore b/nebula/.gitignore new file mode 100644 index 0000000..a1f33f6 --- /dev/null +++ b/nebula/.gitignore @@ -0,0 +1,11 @@ +/target +/classes +/checkouts +profiles.clj +pom.xml +pom.xml.asc +*.class +/.lein-* +/.nrepl-port +.hgignore +.hg/ diff --git a/nebula/lib/client-1.0.0-beta.jar b/nebula/lib/client-1.0.0-beta.jar index f38fef9..e5374a2 100644 Binary files a/nebula/lib/client-1.0.0-beta.jar and b/nebula/lib/client-1.0.0-beta.jar differ diff --git a/nebula/lib/commons-codec-1.13.jar b/nebula/lib/commons-codec-1.13.jar new file mode 100644 index 0000000..bf6ccb3 Binary files /dev/null and b/nebula/lib/commons-codec-1.13.jar differ diff --git a/nebula/lib/commons-lang-2.6.jar b/nebula/lib/commons-lang-2.6.jar new file mode 100644 index 0000000..98467d3 Binary files /dev/null and b/nebula/lib/commons-lang-2.6.jar differ diff --git a/nebula/lib/commons-lang3-3.8.jar b/nebula/lib/commons-lang3-3.8.jar new file mode 100644 index 0000000..b234b3f Binary files /dev/null and b/nebula/lib/commons-lang3-3.8.jar differ diff --git a/nebula/src/nebula/client.clj b/nebula/src/nebula/client.clj index 39bd032..b90b0a8 100644 --- a/nebula/src/nebula/client.clj +++ b/nebula/src/nebula/client.clj @@ -20,7 +20,6 @@ [metaHost metaPort] (let [metaClient (MetaClientImpl. metaHost metaPort)] (def storageClient (StorageClientImpl. metaClient)) - (.switchSpace storageClient default-space) storageClient)) (defn connect @@ -34,11 +33,9 @@ (defn put ([storageClient k v] - (let [part (+ (mod (.hash storageClient k) 5) 1)] - (.put storageClient part k v)))) + (.put storageClient default-space k v))) (defn get ([storageClient k] - (let [part (+ (mod (.hash storageClient k) 5) 1)] - (.get storageClient part k)))) + (.get storageClient default-space k))) diff --git a/nebula/src/nebula/core.clj b/nebula/src/nebula/core.clj index 56d0f2b..484c70c 100644 --- a/nebula/src/nebula/core.clj +++ b/nebula/src/nebula/core.clj @@ -19,10 +19,11 @@ (:import [com.vesoft.nebula.storage.client StorageClientImpl] [com.vesoft.nebula.graph.client GraphClientImpl])) -(def dir "/opt/nebula") -(def binary "nebula") -(def logfile (str dir "/nebula.log")) -(def pidfile (str dir "/nebula.pid")) +(def dir "/usr/local/nebula/") +(def binary (str dir "bin/nebula-storaged")) +(def logfile "/root/nebula.log") +(def pidfile "/root/pids/nebula-storaged.pid") +(def datafile "/data/storage/nebula/*") (def default-port 44500) (def default-space 1) @@ -36,8 +37,47 @@ (defn r [_ _] {:type :invoke, :f :read, :value nil}) (defn w [_ _] {:type :invoke, :f :write, :value (rand-int 5)}) +(defn running? + "Is the service running?" + [binary pidfile] + (try + (c/exec :start-stop-daemon :--status + :--pidfile pidfile + :--exec binary) + true + (catch RuntimeException _ false))) + +(defn start-nebula! + "Starts nebula storage service." + [node] + (info "starting storage node" node) + (c/su + (assert (not (running? binary pidfile))) + (c/exec :start-stop-daemon :--start + :--background + :--make-pidfile + :--pidfile pidfile + :--chdir dir + :--exec binary + :-- + "--flagfile" + "/usr/local/nebula/etc/nebula-storaged.conf") + (info node "started"))) + +(defn stop-nebula! + "Stops nebula storage service." + [node] + (info "stopping storage node" node) + (c/su + (cu/grepkill! :nebula-storaged) + (c/exec :rm :-rf pidfile))) + +(defn flag-file + [] + (str dir "etc/nebula-storaged.conf")) + (defn create-space - "Not using yet. Test space will be created by ./start script" + "Not using yet. Test space will be created by ./up.sh script" [graphHost graphPort] (let [graphClient (GraphClientImpl. graphHost graphPort)] (.connect graphClient default-username default-password) @@ -53,16 +93,19 @@ [version] (reify db/DB (setup! [_ test node] - ;(info node "Depolying Nebula KV Test Environment") - ) + (c/su + (start-nebula! node) ;start nebula storage in each node + (Thread/sleep 3000))) (teardown! [_ test node] - (info "tearing down Nebula" node) - (c/su (c/exec :rm :-rf dir))) - + (stop-nebula! node) ;stop + (Thread/sleep 3000) + (c/su + (c/exec :rm :-rf (c/lit "/data/storage/nebula/*")))) ;clean data stored in storage node + db/LogFiles - (log-files [_ test node] - [logfile]))) + (log-files [_ test node] + [logfile]))) (defn parse-long "Parses a string to a Long. Passes through `nil`." @@ -75,19 +118,17 @@ (open! [this test node] this) (setup! [this test node] - (assoc this :conn (nclient/init default-meta-host default-meta-port))) + (assoc this :conn (nclient/init default-meta-host default-meta-port))) ; :conn is a object of StorageClient (invoke! [this test op] - (let [k (get-random-key) - crash (if (= :read (:f op)) :fail :info)] - (try - (case (:f op) - :read (assoc op :type :ok, :value (-> conn - (nclient/get k))) - :write (do (nclient/put conn k (:value op)) - (assoc op :type :ok))) - (catch java.lang.NullPointerException e - (assoc op :type :fail, :error :nullpointer_exception))))) + (try + (case (:f op) + :read (assoc op :type :ok, :value (-> conn + (nclient/get "f"))) ; get the value of a specific key from storage + :write (do (nclient/put conn "f" (:value op)) ; put a key : value pair to storage + (assoc op :type :ok))) + (catch java.lang.NullPointerException e + (assoc op :type :fail, :error :nullpointer_exception)))) ; basically this will happen when there is no that key (teardown! [this test]) @@ -103,14 +144,14 @@ :os centos/os :db (db "v1.0.0") :client (Client. nil) - :nemesis (nemesis/partition-random-halves) + :nemesis nemesis/noop :checker (checker/compose {:perf (checker/perf) :timeline (timeline/html) - :linear (checker/linearizable {:model (model/register) - :algorithm :linear})}) + :linear (checker/linearizable {:model (model/register) + :algorithm :linear})}) :generator (->> (gen/mix [r w]) - (gen/stagger 1/2) + (gen/stagger 3) (gen/nemesis (gen/seq (cycle [(gen/sleep 5) {:type :info, :f :start} diff --git a/start.sh b/start.sh deleted file mode 100755 index 7f79baa..0000000 --- a/start.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash - -function start() { - cnt=0 - for f in $(seq 1 5);do - if docker ps -a | grep -qw nebula-n$f; then - docker ps | grep -qw nebula-n$f || docker start nebula-n$f - cnt=$((${cnt} + 1)) - fi - done - if docker ps -a | grep -qw nebula-metad;then - docker ps | grep -qw nebula-metad || docker start nebula-metad - cnt=$((${cnt} + 1)) - fi - if docker ps -a | grep -qw nebula-graphd;then - docker ps | grep -qw nebula-graphd || docker start nebula-graphd - cnt=$((${cnt} + 1)) - fi - - if [ "$cnt" -ne 7 ];then - if [ "$cnt" -eq 0 ];then - init - else - stop - remove - init - fi - fi - - if ! docker ps -a | grep -qw nebula-control; then - docker run --name nebula-control --privileged=true -it --net docker_nebula-net \ - --link nebula-n1:n1 --link nebula-n2:n2 --link nebula-n3:n3 --link nebula-n4:n4 \ - --link nebula-n5:n5 -v $(pwd)/:/jepsen_dev wfystx/nebula-control:latest /bin/bash - else - docker ps | grep -qw nebula-control || docker start nebula-control - docker exec -t -i nebula-control /bin/bash - fi -} - -function remove() { - for f in $(seq 1 5);do - echo "remove nebula-jepsen storage node $f" - docker rm -f nebula-n$f - done - echo "remove nebula-jepsen meta node" - docker rm -f nebula-metad - echo "remove nebula-jepsen graph node" - docker rm -f nebula-graphd - echo "remove nebula-jepsen control node" - docker rm -f nebula-control -} - -function stop() { - for f in $(seq 1 5);do - docker stop nebula-n$f - done - docker stop nebula-metad - docker stop nebula-graphd - docker stop nebula-control -} - -function init() { - if [ -d "docker/data/" ];then - rm -rf docker/data/ - fi - if [ -d "docker/logs/" ];then - rm -rf docker/data/ - fi - docker-compose -f docker/docker-compose.yaml up -d - for f in $(seq 1 5);do - docker exec nebula-n$f /run.sh - done - docker run -idt --name nebula-console --network=host vesoft/nebula-console:nightly --addr=127.0.0.1 --port=3699 - cat create_space.txt | docker exec -i nebula-console /bin/bash - docker stop nebula-console - docker rm nebula-console -} - -function restart() { - stop - remove - init - docker run --name nebula-control --privileged=true -it --net docker_nebula-net \ - --link nebula-n1:n1 --link nebula-n2:n2 --link nebula-n3:n3 --link nebula-n4:n4 \ - --link nebula-n5:n5 -v $(pwd)/:/jepsen_dev wfystx/nebula-control:latest /bin/bash -} - -case $1 in - "start") - start - ;; - "stop") - stop - ;; - "remove") - remove - ;; - "restart") - restart - ;; - "help") - echo "start.sh [start|restart|stop|remove]" - ;; - *) - echo "start.sh [start|restart|stop|remove]" - ;; -esac \ No newline at end of file