lightning: make OpLevelOptional suppress the error of DoChecksum (#45486

) (#45866) close #45382
pingcap · Aug 15, 2023 · adf6de5 · adf6de5
1 parent 4973f87
commit adf6de5
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 11 deletions.
diff --git a/br/pkg/lightning/restore/BUILD.bazel b/br/pkg/lightning/restore/BUILD.bazel
@@ -82,7 +82,9 @@ go_library(
         "@com_github_tikv_pd_client//:client",
         "@io_etcd_go_etcd_client_v3//:client",
         "@org_golang_google_grpc//:grpc",
+        "@org_golang_google_grpc//codes",
         "@org_golang_google_grpc//keepalive",
+        "@org_golang_google_grpc//status",
         "@org_golang_x_exp//maps",
         "@org_golang_x_exp//slices",
         "@org_golang_x_sync//errgroup",

diff --git a/br/pkg/lightning/restore/table_restore.go b/br/pkg/lightning/restore/table_restore.go
@@ -44,6 +44,8 @@ import (
 	"go.uber.org/multierr"
 	"go.uber.org/zap"
 	"golang.org/x/exp/slices"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 type TableRestore struct {
@@ -843,15 +845,26 @@ func (tr *TableRestore) postProcess(
 
 			var remoteChecksum *RemoteChecksum
 			remoteChecksum, err = DoChecksum(ctx, tr.tableInfo)
+			failpoint.Inject("checksum-error", func() {
+				tr.logger.Info("failpoint checksum-error injected.")
+				remoteChecksum = nil
+				err = status.Error(codes.Unknown, "Checksum meets error.")
+			})
 			if err != nil {
-				return false, err
+				if rc.cfg.PostRestore.Checksum != config.OpLevelOptional {
+					return false, err
+				}
+				tr.logger.Warn("do checksum failed, will skip this error and go on", log.ShortError(err))
+				err = nil
 			}
-			err = tr.compareChecksum(remoteChecksum, localChecksum)
-			// with post restore level 'optional', we will skip checksum error
-			if rc.cfg.PostRestore.Checksum == config.OpLevelOptional {
-				if err != nil {
-					tr.logger.Warn("compare checksum failed, will skip this error and go on", log.ShortError(err))
-					err = nil
+			if remoteChecksum != nil {
+				err = tr.compareChecksum(remoteChecksum, localChecksum)
+				// with post restore level 'optional', we will skip checksum error
+				if rc.cfg.PostRestore.Checksum == config.OpLevelOptional {
+					if err != nil {
+						tr.logger.Warn("compare checksum failed, will skip this error and go on", log.ShortError(err))
+						err = nil
+					}
 				}
 			}
 		} else {
@@ -893,11 +906,12 @@ func (tr *TableRestore) postProcess(
 		case forcePostProcess || !rc.cfg.PostRestore.PostProcessAtLast:
 			err := tr.analyzeTable(ctx, rc.tidbGlue.GetSQLExecutor())
 			// witch post restore level 'optional', we will skip analyze error
-			if rc.cfg.PostRestore.Analyze == config.OpLevelOptional {
-				if err != nil {
-					tr.logger.Warn("analyze table failed, will skip this error and go on", log.ShortError(err))
-					err = nil
+			if err != nil {
+				if rc.cfg.PostRestore.Analyze != config.OpLevelOptional {
+					return false, err
 				}
+				tr.logger.Warn("analyze table failed, will skip this error and go on", log.ShortError(err))
+				err = nil
 			}
 			saveCpErr := rc.saveStatusCheckpoint(ctx, tr.tableName, checkpoints.WholeTableEngineID, err, checkpoints.CheckpointStatusAnalyzed)
 			if err = firstErr(err, saveCpErr); err != nil {

diff --git a/br/tests/lightning_routes/config.toml b/br/tests/lightning_routes/config.toml
@@ -8,3 +8,6 @@ schema-pattern = "routes_a*"
 table-pattern = "t*"
 target-schema = "routes_b"
 target-table = "u"
+
+[post-restore]
+checksum = "optional"
diff --git a/br/tests/lightning_routes/run.sh b/br/tests/lightning_routes/run.sh
@@ -4,12 +4,17 @@
 
 set -eux
 
+echo "testing checksum-error..."
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/lightning/restore/checksum-error=1*return()"
+
 run_sql 'DROP DATABASE IF EXISTS routes_a0;'
 run_sql 'DROP DATABASE IF EXISTS routes_a1;'
 run_sql 'DROP DATABASE IF EXISTS routes_b;'
 
 run_lightning
 
+echo "test checksum-error success!"
+
 run_sql 'SELECT count(1), sum(x) FROM routes_b.u;'
 check_contains 'count(1): 4'
 check_contains 'sum(x): 259'