*: addressed comments

cmd/tidb-lightning-ctl/main.go

@@ -163,8 +163,12 @@ func checkpointErrorDestroy(ctx context.Context, cfg *config.Config, tableName s
 
 	for _, table := range targetTables {
 		for engineID := 0; engineID < table.EnginesCount; engineID++ {
-			if closedEngine, err := importer.UnsafeCloseEngine(ctx, table.TableName, engineID); err == nil {
-				fmt.Fprintln(os.Stderr, "Cleaning up engine:", table.TableName, engineID)
+			fmt.Fprintln(os.Stderr, "Closing and cleaning up engine:", table.TableName, engineID)
+			closedEngine, err := importer.UnsafeCloseEngine(ctx, table.TableName, engineID)
+			if err != nil {
+				fmt.Fprintln(os.Stderr, "* Encountered error while closing engine:", err)
+				lastErr = err
+			} else {
 				closedEngine.Cleanup(ctx)
 			}
 		}

lightning/kv/importer.go

@@ -31,19 +31,23 @@ Usual workflow:
 
 2. For each table,
 
-	a. Create an `OpenedEngine` via `importer.OpenEngine()`
+	i. Split into multiple "batches" consisting of data files with roughly equal total size.
 
-	b. For each chunk,
+	ii. For each batch,
 
-		i. Create a `WriteStream` via `engine.NewWriteStream()`
-		ii. Deliver data into the stream via `stream.Put()`
-		iii. Close the stream via `stream.Close()`
+		a. Create an `OpenedEngine` via `importer.OpenEngine()`
 
-	c. When all chunks are written, obtain a `ClosedEngine` via `engine.CloseEngine()`
+		b. For each chunk,
 
-	d. Import data via `engine.Import()`
+			i. Create a `WriteStream` via `engine.NewWriteStream()`
+			ii. Deliver data into the stream via `stream.Put()`
+			iii. Close the stream via `stream.Close()`
 
-	e. Cleanup via `engine.Cleanup()`
+		c. When all chunks are written, obtain a `ClosedEngine` via `engine.CloseEngine()`
+
+		d. Import data via `engine.Import()`
+
+		e. Cleanup via `engine.Cleanup()`
 
 3. Close the connection via `importer.Close()`
 

lightning/mydump/region.go

@@ -62,10 +62,9 @@ func MakeTableRegions(meta *MDTableMeta, columns int, batchSize int64) ([]*Table
 		rowIDMax := prevRowIDMax + dataFileSize/(int64(columns)+2)
 		filesRegions = append(filesRegions, &TableRegion{
 			EngineID: curEngineID,
-
-			DB:    meta.DB,
-			Table: meta.Name,
-			File:  dataFile,
+			DB:       meta.DB,
+			Table:    meta.Name,
+			File:     dataFile,
 			Chunk: Chunk{
 				Offset:       0,
 				EndOffset:    dataFileSize,

tests/checkpoint_engines/config.toml

@@ -6,6 +6,7 @@ level = "info"
 
 [checkpoint]
 enable = true
+driver = "file"
 
 [tikv-importer]
 addr = "127.0.0.1:8808"

tests/checkpoint_engines/mysql.toml

@@ -0,0 +1,29 @@
+[lightning]
+table-concurrency = 1
+check-requirements = false
+file = "/tmp/lightning_test_result/lightning-checkpoint-engines.log"
+level = "info"
+
+[checkpoint]
+enable = true
+driver = "mysql"
+
+[tikv-importer]
+addr = "127.0.0.1:8808"
+
+[mydumper]
+data-source-dir = "tests/checkpoint_engines/data"
+batch-size = 1 # force creating a new batch for every file.
+
+[tidb]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+status-port = 10080
+pd-addr = "127.0.0.1:2379"
+log-level = "error"
+
+[post-restore]
+checksum = true
+compact = false
+analyze = false

tests/checkpoint_engines/run.sh

@@ -46,3 +46,26 @@ check_contains 'sum(c): 10'
 run_sql 'SELECT count(*), sum(c) FROM cpeng.b'
 check_contains 'count(*): 4'
 check_contains 'sum(c): 46'
+
+# Now, try again with MySQL checkpoints
+
+run_sql 'DROP DATABASE cpeng;'
+
+set +e
+for i in $(seq 4); do
+    echo "******** Importing Table Now (step $i/4) ********"
+    run_lightning mysql 2> /dev/null
+    [ $? -ne 0 ] || exit 1
+done
+set -e
+
+echo "******** Verify checkpoint no-op ********"
+run_lightning mysql
+
+run_sql 'SELECT count(*), sum(c) FROM cpeng.a'
+check_contains 'count(*): 4'
+check_contains 'sum(c): 10'
+
+run_sql 'SELECT count(*), sum(c) FROM cpeng.b'
+check_contains 'count(*): 4'
+check_contains 'sum(c): 46'