tests: add kill lightning test in checkpoint_chunks case (pingcap#158)

use gofail to control lightning to kill itself after one chunk is imported

lightning/common/util.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/go-sql-driver/mysql"
@@ -245,3 +246,8 @@ func GetJSON(client *http.Client, url string, v interface{}) error {
 
 	return errors.Trace(json.NewDecoder(resp.Body).Decode(v))
 }
+
+// KillMySelf sends sigint to current process, used in integration test only
+func KillMySelf() error {
+	return errors.Trace(syscall.Kill(syscall.Getpid(), syscall.SIGINT))
+}

lightning/restore/restore.go

@@ -381,6 +381,14 @@ func (rc *RestoreController) listenCheckpointUpdates() {
 		// goto RETURN3
 
 		// gofail: RETURN3:
+
+		// gofail: var KillIfImportedChunk struct{}
+		// if _, ok := scp.merger.(*ChunkCheckpointMerger); ok {
+		// 	common.KillMySelf()
+		// }
+		// goto RETURN4
+
+		// gofail: RETURN4:
 	}
 	rc.checkpointsWg.Done()
 }

tests/checkpoint_chunks/run.sh

@@ -20,6 +20,18 @@ DBPATH="$TEST_DIR/cpch.mydump"
 CHUNK_COUNT=5
 ROW_COUNT=1000
 
+verify_checkpoint_noop() {
+    # After everything is done, there should be no longer new calls to WriteEngine/CloseAndRecv
+    # (and thus `kill_lightning_after_one_chunk` will spare this final check)
+    echo "******** Verify checkpoint no-op ********"
+    run_lightning
+    run_sql 'SELECT count(i), sum(i) FROM cpch_tsr.tbl;'
+    check_contains "count(i): $(($ROW_COUNT*$CHUNK_COUNT))"
+    check_contains "sum(i): $(( $ROW_COUNT*$CHUNK_COUNT*(($CHUNK_COUNT+2)*$ROW_COUNT + 1)/2 ))"
+    run_sql "SELECT count(*) FROM tidb_lightning_checkpoint_test_cpch.table_v4 WHERE status >= 200"
+    check_contains "count(*): 1"
+}
+
 mkdir -p $DBPATH
 echo 'CREATE DATABASE cpch_tsr;' > "$DBPATH/cpch_tsr-schema-create.sql"
 echo 'CREATE TABLE tbl(i BIGINT UNSIGNED PRIMARY KEY);' > "$DBPATH/cpch_tsr.tbl-schema.sql"
@@ -47,21 +59,49 @@ for i in $(seq "$CHUNK_COUNT"); do
 done
 set -e
 
-# After everything is done, there should be no longer new calls to WriteEngine/CloseAndRecv
-# (and thus `kill_lightning_after_one_chunk` will spare this final check)
-echo "******** Verify checkpoint no-op ********"
-run_lightning
-run_sql 'SELECT count(i), sum(i) FROM cpch_tsr.tbl;'
-check_contains "count(i): $(($ROW_COUNT*$CHUNK_COUNT))"
-check_contains "sum(i): $(( $ROW_COUNT*$CHUNK_COUNT*(($CHUNK_COUNT+2)*$ROW_COUNT + 1)/2 ))"
-run_sql "SELECT count(*) FROM tidb_lightning_checkpoint_test_cpch.table_v4 WHERE status >= 200"
-check_contains "count(*): 1"
+verify_checkpoint_noop
+
+# Next, test kill lightning via signal mechanism
+run_sql 'DROP DATABASE IF EXISTS cpch_tsr'
+run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cpch'
+
+# Set the failpoint to kill the lightning instance as soon as one chunk is imported, via signal mechanism
+# If checkpoint does work, this should only kill $CHUNK_COUNT instances of lightnings.
+export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/KillIfImportedChunk=return'
+
+for i in $(seq "$CHUNK_COUNT"); do
+    echo "******** Importing Chunk Now (step $i/$CHUNK_COUNT) ********"
+    run_lightning
+done
+
+set +e
+i=0
+wait_max_time=20
+while [ $i -lt $wait_max_time ]; do
+    lightning_proc=$(ps -ef|grep '[b]in/tidb-lightning.test')
+    ret="$?"
+    if [ "$ret" -eq 0 ]; then
+        sleep 1
+    else
+        break
+    fi
+done
+if [ $i -ge $wait_max_time ]; then
+    echo "wait lightning exit failed"
+    exit 1
+fi
+set -e
+
+verify_checkpoint_noop
 
 # Repeat, but using the file checkpoint
 run_sql 'DROP DATABASE IF EXISTS cpch_tsr'
 run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cpch'
 rm -f "$TEST_DIR/cpch.pb"
 
+# Set the failpoint to kill the lightning instance as soon as one chunk is imported
+# If checkpoint does work, this should only kill $CHUNK_COUNT instances of lightnings.
+export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/FailIfImportedChunk=return'
 set +e
 for i in $(seq "$CHUNK_COUNT"); do
     echo "******** Importing Chunk using File checkpoint Now (step $i/$CHUNK_COUNT) ********"