From 25da6430a32f15518c078c5482963708bb9ee470 Mon Sep 17 00:00:00 2001 From: linning Date: Sat, 12 Oct 2019 00:22:12 +0800 Subject: [PATCH 1/3] add integration tests Signed-off-by: linning --- Makefile | 10 +++ tests/README.md | 49 ++++++++++++++ tests/_utils/run_sql | 19 ++++++ tests/br_full/run.sh | 58 ++++++++++++++++ tests/br_full/workload | 12 ++++ tests/br_shuffle_leader/run.sh | 49 ++++++++++++++ tests/br_shuffle_leader/workload | 12 ++++ tests/br_shuffle_region/run.sh | 49 ++++++++++++++ tests/br_shuffle_region/workload | 12 ++++ tests/br_single_table/run.sh | 43 ++++++++++++ tests/br_single_table/workload | 12 ++++ tests/config/tidb.toml | 3 + tests/config/tikv.toml | 1 + tests/run.sh | 112 +++++++++++++++++++++++++++++++ 14 files changed, 441 insertions(+) create mode 100644 tests/README.md create mode 100755 tests/_utils/run_sql create mode 100755 tests/br_full/run.sh create mode 100644 tests/br_full/workload create mode 100755 tests/br_shuffle_leader/run.sh create mode 100644 tests/br_shuffle_leader/workload create mode 100755 tests/br_shuffle_region/run.sh create mode 100644 tests/br_shuffle_region/workload create mode 100755 tests/br_single_table/run.sh create mode 100644 tests/br_single_table/workload create mode 100644 tests/config/tidb.toml create mode 100644 tests/config/tikv.toml create mode 100755 tests/run.sh diff --git a/Makefile b/Makefile index cfac026ef828a..1c8fe427dd060 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,16 @@ build: test: GO111MODULE=on go test -race ./... +integration_test: + @which bin/tidb-server + @which bin/tikv-server + @which bin/pd-server + @which bin/pd-ctl + @which bin/tikv-importer + @which bin/go-ycsb + @which bin/br + tests/run.sh + tools: @echo "install tools..." @GO111MODULE=off go get github.com/twitchtv/retool diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000000000..57684b5cf2ca8 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,49 @@ +This folder contains all tests which relies on external processes such as TiDB. + +Unit tests (the `*_test.go` files inside the source directory) should *never* rely on external +programs. + +## Preparations + +1. The following 7 executables must be copied or linked into these locations: + * `bin/tidb-server` + * `bin/tikv-server` + * `bin/pd-server` + * `bin/pd-ctl` + * `bin/tikv-importer` + * `bin/go-ycsb` + * `bin/br` + + The versions must be ≥2.1.0 as usual. + +2. The following programs must be installed: + + * `mysql` (the CLI client) + * `curl` + +3. The user executing the tests must have permission to create the folder + `/tmp/backup_restore_test`. All test artifacts will be written into this folder. + +## Running + +Make sure the path is `br/` + +Run `make integration_test` to execute the integration tests. This command will + +1. Check that all 7 executables exist. +3. Execute `tests/run.sh` + +If the first steps are done before, you could also run `tests/run.sh` directly. +This script will + +1. Start PD, TiKV, TiDB and TiKV-Importer in background with local storage +2. Find out all `tests/*/run.sh` and run it + +## Writing new tests + +New integration tests can be written as shell scripts in `tests/TEST_NAME/run.sh`. +The script should exit with a nonzero error code on failure. + +Several convenient commands are provided: + +* `run_sql ` — Executes an SQL query on the TiDB database \ No newline at end of file diff --git a/tests/_utils/run_sql b/tests/_utils/run_sql new file mode 100755 index 0000000000000..906af942d06c6 --- /dev/null +++ b/tests/_utils/run_sql @@ -0,0 +1,19 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +echo "[$(date)] Executing SQL: $1" > "$TEST_DIR/sql_res.$TEST_NAME.txt" +mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -E -e "$1" diff --git a/tests/br_full/run.sh b/tests/br_full/run.sh new file mode 100755 index 0000000000000..de90adec90305 --- /dev/null +++ b/tests/br_full/run.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu +DB="$TEST_NAME" +TABLE="usertable" +DB_COUNT=3 + +for i in $(seq $DB_COUNT); do + run_sql "CREATE DATABASE $DB${i};" + go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i} +done + +for i in $(seq $DB_COUNT); do + row_count_ori[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}') +done + +# backup full +br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB/backupdata" --ratelimit 100 --concurrency 4 + +for i in $(seq $DB_COUNT); do + run_sql "DROP DATABASE $DB${i};" +done + +# restore full +br restore full --connect "root@tcp($TIDB_ADDR)/" --importer $IMPORTER_ADDR --meta backupmeta --status $TIDB_IP:10080 --pd $PD_ADDR + +for i in $(seq $DB_COUNT); do + row_count_new[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}') +done + +fail=false +for i in $(seq $DB_COUNT); do + if [ "${row_count_ori[i]}" != "${row_count_new[i]}" ];then + fail=true + echo "TEST: [$TEST_NAME] fail on database $DB${i}" + fi + echo "[original] row count: ${row_count_ori[i]}, [after br] row count: ${row_count_new[i]}" +done + +if $fail; then + echo "TEST: [$TEST_NAME] failed!" + exit 1 +else + echo "TEST: [$TEST_NAME] successed!" +fi \ No newline at end of file diff --git a/tests/br_full/workload b/tests/br_full/workload new file mode 100644 index 0000000000000..97456e817ec5d --- /dev/null +++ b/tests/br_full/workload @@ -0,0 +1,12 @@ +recordcount=2000 +operationcount=0 +workload=core + +readallfields=true + +readproportion=0 +updateproportion=0 +scanproportion=0 +insertproportion=0 + +requestdistribution=uniform \ No newline at end of file diff --git a/tests/br_shuffle_leader/run.sh b/tests/br_shuffle_leader/run.sh new file mode 100755 index 0000000000000..f3d06fad1f393 --- /dev/null +++ b/tests/br_shuffle_leader/run.sh @@ -0,0 +1,49 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu +DB="$TEST_NAME" +TABLE="usertable" + +run_sql "CREATE DATABASE $DB;" + +go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB + +row_count_ori=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}') + +# add shuffle leader scheduler +pd-ctl -u "http://$PD_ADDR" -d sched add shuffle-leader-scheduler + +# backup with shuffle leader +br --pd $PD_ADDR backup table -s "local://$TEST_DIR/$DB/backupdata" --db $DB -t $TABLE --ratelimit 100 --concurrency 4 + +run_sql "DELETE FROM $DB.$TABLE;" + +# restore with shuffle leader +br restore table --db $DB --table $TABLE --connect "root@tcp($TIDB_ADDR)/" --importer $IMPORTER_ADDR --meta backupmeta --status $TIDB_IP:10080 --pd $PD_ADDR + +# remove shuffle leader scheduler +pd-ctl -u "http://$PD_ADDR" -d sched remove shuffle-leader-scheduler + +row_count_new=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}') + +echo "[original] row count: $row_count_ori, [after br] row count: $row_count_new" + +if [ "$row_count_ori" -eq "$row_count_new" ];then + echo "TEST: [$TEST_NAME] successed!" +else + echo "TEST: [$TEST_NAME] failed!" + exit 1 +fi \ No newline at end of file diff --git a/tests/br_shuffle_leader/workload b/tests/br_shuffle_leader/workload new file mode 100644 index 0000000000000..97456e817ec5d --- /dev/null +++ b/tests/br_shuffle_leader/workload @@ -0,0 +1,12 @@ +recordcount=2000 +operationcount=0 +workload=core + +readallfields=true + +readproportion=0 +updateproportion=0 +scanproportion=0 +insertproportion=0 + +requestdistribution=uniform \ No newline at end of file diff --git a/tests/br_shuffle_region/run.sh b/tests/br_shuffle_region/run.sh new file mode 100755 index 0000000000000..a5b07f79a829e --- /dev/null +++ b/tests/br_shuffle_region/run.sh @@ -0,0 +1,49 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu +DB="$TEST_NAME" +TABLE="usertable" + +run_sql "CREATE DATABASE $DB;" + +go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB + +row_count_ori=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}') + +# add shuffle region scheduler +pd-ctl -u "http://$PD_ADDR" -d sched add shuffle-region-scheduler + +# backup with shuffle region +br --pd $PD_ADDR backup table -s "local://$TEST_DIR/$DB/backupdata" --db $DB -t $TABLE --ratelimit 100 --concurrency 4 + +run_sql "DELETE FROM $DB.$TABLE;" + +# restore with shuffle region +br restore table --db $DB --table $TABLE --connect "root@tcp($TIDB_ADDR)/" --importer $IMPORTER_ADDR --meta backupmeta --status $TIDB_IP:10080 --pd $PD_ADDR + +# remove shuffle region scheduler +pd-ctl -u "http://$PD_ADDR" -d sched remove shuffle-region-scheduler + +row_count_new=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}') + +echo "[original] row count: $row_count_ori, [after br] row count: $row_count_new" + +if [ "$row_count_ori" -eq "$row_count_new" ];then + echo "TEST: [$TEST_NAME] successed!" +else + echo "TEST: [$TEST_NAME] failed!" + exit 1 +fi \ No newline at end of file diff --git a/tests/br_shuffle_region/workload b/tests/br_shuffle_region/workload new file mode 100644 index 0000000000000..97456e817ec5d --- /dev/null +++ b/tests/br_shuffle_region/workload @@ -0,0 +1,12 @@ +recordcount=2000 +operationcount=0 +workload=core + +readallfields=true + +readproportion=0 +updateproportion=0 +scanproportion=0 +insertproportion=0 + +requestdistribution=uniform \ No newline at end of file diff --git a/tests/br_single_table/run.sh b/tests/br_single_table/run.sh new file mode 100755 index 0000000000000..cd6eb22f207a7 --- /dev/null +++ b/tests/br_single_table/run.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu +DB="$TEST_NAME" +TABLE="usertable" + +run_sql "CREATE DATABASE $DB;" + +go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB + +row_count_ori=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}') + +# backup table +br --pd $PD_ADDR backup table -s "local://$TEST_DIR/$DB/backupdata" --db $DB -t $TABLE --ratelimit 100 --concurrency 4 + +run_sql "DELETE FROM $DB.$TABLE;" + +# restore table +br restore table --db $DB --table $TABLE --connect "root@tcp($TIDB_ADDR)/" --importer $IMPORTER_ADDR --meta backupmeta --status $TIDB_IP:10080 --pd $PD_ADDR + +row_count_new=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}') + +echo "[original] row count: $row_count_ori, [after br] row count: $row_count_new" + +if [ "$row_count_ori" -eq "$row_count_new" ];then + echo "TEST: [$TEST_NAME] successed!" +else + echo "TEST: [$TEST_NAME] failed!" + exit 1 +fi \ No newline at end of file diff --git a/tests/br_single_table/workload b/tests/br_single_table/workload new file mode 100644 index 0000000000000..97456e817ec5d --- /dev/null +++ b/tests/br_single_table/workload @@ -0,0 +1,12 @@ +recordcount=2000 +operationcount=0 +workload=core + +readallfields=true + +readproportion=0 +updateproportion=0 +scanproportion=0 +insertproportion=0 + +requestdistribution=uniform \ No newline at end of file diff --git a/tests/config/tidb.toml b/tests/config/tidb.toml new file mode 100644 index 0000000000000..f6f252e01d09b --- /dev/null +++ b/tests/config/tidb.toml @@ -0,0 +1,3 @@ +# config of tidb + +lease = "0s" \ No newline at end of file diff --git a/tests/config/tikv.toml b/tests/config/tikv.toml new file mode 100644 index 0000000000000..71b69758dacaa --- /dev/null +++ b/tests/config/tikv.toml @@ -0,0 +1 @@ +# config of tikv \ No newline at end of file diff --git a/tests/run.sh b/tests/run.sh new file mode 100755 index 0000000000000..f260c3ce3422d --- /dev/null +++ b/tests/run.sh @@ -0,0 +1,112 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +TEST_DIR=/tmp/backup_restore_test + +PD_ADDR="127.0.0.1:2379" +IMPORTER_ADDR="127.0.0.1:8808" +TIDB_IP="127.0.0.1" +TIDB_PORT="4000" +TIDB_ADDR="127.0.0.1:4000" +# actaul tikv_addr are TIKV_ADDR${i} +TIKV_ADDR="127.0.0.1:2016" +TIKV_COUNT=2 + +stop_services() { + killall -9 tikv-server || true + killall -9 pd-server || true + killall -9 tidb-server || true + killall -9 tikv-importer || true + + find "$TEST_DIR" -maxdepth 1 -not -path "$TEST_DIR" -not -name "*.log" | xargs rm -r || true +} + +start_services() { + stop_services + + mkdir -p "$TEST_DIR" + rm -f "$TEST_DIR"/*.log + + echo "Starting PD..." + bin/pd-server \ + --client-urls "http://$PD_ADDR" \ + --log-file "$TEST_DIR/pd.log" \ + --data-dir "$TEST_DIR/pd" & + # wait until PD is online... + while ! curl -o /dev/null -sf "http://$PD_ADDR/pd/api/v1/version"; do + sleep 1 + done + + echo "Starting TiKV..." + for i in $(seq $TIKV_COUNT); do + bin/tikv-server \ + --pd "$PD_ADDR" \ + -A "$TIKV_ADDR$i" \ + --log-file "$TEST_DIR/tikv${i}.log" \ + -C "tests/config/tikv.toml" \ + -s "$TEST_DIR/tikv${i}" & + done + sleep 1 + + echo "Starting TiDB..." + bin/tidb-server \ + -P 4000 \ + --store tikv \ + --path "$PD_ADDR" \ + --config "tests/config/tidb.toml" \ + --log-file "$TEST_DIR/tidb.log" & + + echo "Starting Importer..." + bin/tikv-importer \ + -A "$IMPORTER_ADDR" \ + --log-file "$TEST_DIR/importer.log" \ + --import-dir "$TEST_DIR/importer" & + + echo "Verifying TiDB is started..." + i=0 + while ! curl -o /dev/null -sf "http://$TIDB_IP:10080/status"; do + i=$((i+1)) + if [ "$i" -gt 10 ]; then + echo 'Failed to start TiDB' + exit 1 + fi + sleep 3 + done +} + +trap stop_services EXIT +start_services + +if [ "${1-}" = '--debug' ]; then + echo 'You may now debug from another terminal. Press [ENTER] to continue.' + read line +fi + +for script in tests/*/run.sh; do + echo "*===== Running test $script... =====*" + TEST_DIR="$TEST_DIR" \ + PD_ADDR="$PD_ADDR" \ + IMPORTER_ADDR="$IMPORTER_ADDR" \ + TIDB_IP="$TIDB_IP" \ + TIDB_PORT="$TIDB_PORT" \ + TIDB_ADDR="$TIDB_ADDR" \ + TIKV_ADDR="$TIKV_ADDR" \ + PATH="tests/_utils:bin:$PATH" \ + TEST_NAME="$(basename "$(dirname "$script")")" \ + sh "$script" + rm backupmeta +done From e5e8c5bfdadeb4b38abb90c8a0d952de8495c6f2 Mon Sep 17 00:00:00 2001 From: linning Date: Sat, 12 Oct 2019 13:45:26 +0800 Subject: [PATCH 2/3] update test config Signed-off-by: linning --- tests/config/tidb.toml | 3 +++ tests/config/tikv.toml | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/config/tidb.toml b/tests/config/tidb.toml index f6f252e01d09b..24a982438ec54 100644 --- a/tests/config/tidb.toml +++ b/tests/config/tidb.toml @@ -1,3 +1,6 @@ # config of tidb +# Schema lease duration +# There are lot of ddl in the tests, setting this +# to 0s to accelerate the tests lease = "0s" \ No newline at end of file diff --git a/tests/config/tikv.toml b/tests/config/tikv.toml index 71b69758dacaa..56192a16cab76 100644 --- a/tests/config/tikv.toml +++ b/tests/config/tikv.toml @@ -1 +1,5 @@ -# config of tikv \ No newline at end of file +# config of tikv + +[coprocessor] +region-max-keys = 20 +region-split-keys = 12 \ No newline at end of file From 2250a8a65d9e668b32d38093dc36bab32f7fa23e Mon Sep 17 00:00:00 2001 From: linning Date: Sat, 12 Oct 2019 16:34:30 +0800 Subject: [PATCH 3/3] build br bin in integration_test Signed-off-by: linning --- Makefile | 4 ++-- tests/README.md | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 1c8fe427dd060..77c55caa1dba2 100644 --- a/Makefile +++ b/Makefile @@ -8,12 +8,12 @@ GOCHECKER := awk '{ print } END { if (NR > 0) { exit 1 } }' all: check build test build: - GO111MODULE=on go build -race + GO111MODULE=on go build -race -o bin/br test: GO111MODULE=on go test -race ./... -integration_test: +integration_test: build @which bin/tidb-server @which bin/tikv-server @which bin/pd-server diff --git a/tests/README.md b/tests/README.md index 57684b5cf2ca8..0644037c7cdc2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -5,14 +5,13 @@ programs. ## Preparations -1. The following 7 executables must be copied or linked into these locations: +1. The following 6 executables must be copied or linked into these locations: * `bin/tidb-server` * `bin/tikv-server` * `bin/pd-server` * `bin/pd-ctl` * `bin/tikv-importer` * `bin/go-ycsb` - * `bin/br` The versions must be ≥2.1.0 as usual. @@ -30,10 +29,11 @@ Make sure the path is `br/` Run `make integration_test` to execute the integration tests. This command will -1. Check that all 7 executables exist. +1. Build `br` +2. Check that all 6 required executables and `br` executable exist 3. Execute `tests/run.sh` -If the first steps are done before, you could also run `tests/run.sh` directly. +If the first tow steps are done before, you could also run `tests/run.sh` directly. This script will 1. Start PD, TiKV, TiDB and TiKV-Importer in background with local storage