diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index 25c5f91b1..18ce61ce3 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -74,6 +74,9 @@ type TableDiff struct { // set false if want to comapre the data directly UseChecksum bool + // set true if just want compare data by checksum, will skip select data when checksum is not equal + OnlyUseChecksum bool + // collation config in mysql/tidb, should corresponding to charset. Collation string @@ -284,6 +287,11 @@ func (t *TableDiff) checkChunkDataEqual(ctx context.Context, checkJobs []*CheckJ log.Warnf("table: %s, range: %s, args: %v, checksum is not equal, one is %d, another is %d", dbutil.TableName(job.Schema, job.Table), job.Where, job.Args, sourceChecksum, targetChecksum) } + if t.UseChecksum && t.OnlyUseChecksum { + equal = false + continue + } + // if checksum is not equal or don't need compare checksum, compare the data log.Infof("select data from %s for range (where: %s, args: %v) and then check data", dbutil.TableName(job.Schema, job.Table), job.Where, job.Args) sourceRows := make(map[string][]map[string]*dbutil.ColumnData) diff --git a/sync_diff_inspector/config.go b/sync_diff_inspector/config.go index 301e5561d..9622f654b 100644 --- a/sync_diff_inspector/config.go +++ b/sync_diff_inspector/config.go @@ -180,6 +180,9 @@ type Config struct { // set false if want to comapre the data directly UseChecksum bool `toml:"use-checksum" json:"use-checksum"` + // set true if just want compare data by checksum, will skip select data when checksum is not equal. + OnlyUseChecksum bool `toml:"only-use-checksum" json:"only-use-checksum"` + // the name of the file which saves sqls used to fix different data FixSQLFile string `toml:"fix-sql-file" json:"fix-sql-file"` @@ -312,5 +315,17 @@ func (c *Config) checkConfig() bool { } } + if c.OnlyUseChecksum { + if !c.UseChecksum { + log.Error("need set use-checksum = true") + return false + } + } else { + if len(c.FixSQLFile) == 0 { + log.Warn("fix-sql-file is invalid, will use default value 'fix.sql'") + c.FixSQLFile = "fix.sql" + } + } + return true } diff --git a/sync_diff_inspector/config.toml b/sync_diff_inspector/config.toml index 4cfb8a1d4..d175f2756 100644 --- a/sync_diff_inspector/config.toml +++ b/sync_diff_inspector/config.toml @@ -18,15 +18,19 @@ sample-percent = 100 use-rowid = false # calculate the data's checksum, and compare data by checksum. +# set false if want to comapre the data directly use-checksum = true +# set true if just want compare data by checksum, will skip select data when checksum is not equal. +only-use-checksum = false + # ignore check table's data ignore-data-check = false # ignore check table's struct ignore-struct-check = false -# the name of the file which saves sqls used to fix different data +# the name of the file which saves sqls used to fix different data. fix-sql-file = "fix.sql" # use this tidb's statistics information to split chunk diff --git a/sync_diff_inspector/diff.go b/sync_diff_inspector/diff.go index 037018387..30b5c7175 100644 --- a/sync_diff_inspector/diff.go +++ b/sync_diff_inspector/diff.go @@ -36,6 +36,7 @@ type Diff struct { checkThreadCount int useRowID bool useChecksum bool + onlyUseChecksum bool ignoreDataCheck bool ignoreStructCheck bool tables map[string]map[string]*TableConfig @@ -56,6 +57,7 @@ func NewDiff(ctx context.Context, cfg *Config) (diff *Diff, err error) { checkThreadCount: cfg.CheckThreadCount, useRowID: cfg.UseRowID, useChecksum: cfg.UseChecksum, + onlyUseChecksum: cfg.OnlyUseChecksum, ignoreDataCheck: cfg.IgnoreDataCheck, ignoreStructCheck: cfg.IgnoreStructCheck, tidbInstanceID: cfg.TiDBInstanceID, @@ -404,6 +406,7 @@ func (df *Diff) Equal() (err error) { CheckThreadCount: df.checkThreadCount, UseRowID: df.useRowID, UseChecksum: df.useChecksum, + OnlyUseChecksum: df.onlyUseChecksum, IgnoreStructCheck: df.ignoreStructCheck, IgnoreDataCheck: df.ignoreDataCheck, TiDBStatsSource: tidbStatsSource,