From 644c92e81b565bdb8473384942b9cc2037d4d442 Mon Sep 17 00:00:00 2001 From: csuzhangxc Date: Mon, 2 Dec 2019 11:51:34 +0800 Subject: [PATCH 1/2] *: update tidb-tools mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 132f486ab2..e7f87ebea2 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/pingcap/parser v0.0.0-20191112053614-3b43b46331d5 github.com/pingcap/pd v1.1.0-beta.0.20191114130008-2488cb978644 // indirect github.com/pingcap/tidb v1.1.0-beta.0.20191115021711-b274eb2079dc - github.com/pingcap/tidb-tools v3.0.7-0.20191129034125-a5a26316b874+incompatible + github.com/pingcap/tidb-tools v3.0.7-0.20191202034632-451c58d281c7+incompatible github.com/prometheus/client_golang v1.2.1 github.com/prometheus/procfs v0.0.7 // indirect github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect diff --git a/go.sum b/go.sum index 0a848127bf..aea22bf0fb 100644 --- a/go.sum +++ b/go.sum @@ -292,8 +292,8 @@ github.com/pingcap/pd v1.1.0-beta.0.20191114130008-2488cb978644/go.mod h1:Z/VMtX github.com/pingcap/tidb v1.1.0-beta.0.20191115021711-b274eb2079dc h1:RtwQ6Kb8QWkUcInh2vzSc8HtQ2VHNNBV6datScfyF3c= github.com/pingcap/tidb v1.1.0-beta.0.20191115021711-b274eb2079dc/go.mod h1:8DSQhELIQr/UqWaolPciDK1Nj6ARvD0NjBmQLCRJpxY= github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tidb-tools v3.0.7-0.20191129034125-a5a26316b874+incompatible h1:N5wERrnu/SPN8V8xAFNewbsGAtRZ3h3Ivpo9qF6SyuY= -github.com/pingcap/tidb-tools v3.0.7-0.20191129034125-a5a26316b874+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= +github.com/pingcap/tidb-tools v3.0.7-0.20191202034632-451c58d281c7+incompatible h1:3RgiaMx1vc6MGIiFlBvymDU+onQnnnXBLx0UTMZ944w= +github.com/pingcap/tidb-tools v3.0.7-0.20191202034632-451c58d281c7+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 h1:rRMLMjIMFulCX9sGKZ1hoov/iROMsKyC8Snc02nSukw= github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pingcap/tipb v0.0.0-20191112054303-0b0ad0d4a92e h1:TWwzCfLrj9GH5uaT0VcvdSnrHuwEntUfoHDTYdOzNNI= From 8737591dbfa27f147a882ee6ac662bb0a24cedf3 Mon Sep 17 00:00:00 2001 From: csuzhangxc Date: Mon, 2 Dec 2019 11:59:16 +0800 Subject: [PATCH 2/2] *: use `IsRetryableError` in tidb-tools --- loader/db.go | 5 +++-- pkg/retry/errors.go | 33 --------------------------------- pkg/retry/strategy_test.go | 3 ++- syncer/db.go | 5 +++-- syncer/error_test.go | 31 ------------------------------- 5 files changed, 8 insertions(+), 69 deletions(-) diff --git a/loader/db.go b/loader/db.go index 95ecca0037..3b61fac69c 100644 --- a/loader/db.go +++ b/loader/db.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" tmysql "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb-tools/pkg/dbutil" "go.uber.org/zap" "github.com/pingcap/dm/dm/config" @@ -65,7 +66,7 @@ func (conn *DBConn) querySQL(ctx *tcontext.Context, query string, args ...interf } return true } - if retry.IsRetryableError(err) { + if dbutil.IsRetryableError(err) { ctx.L().Warn("query statement", zap.Int("retry", retryTime), zap.String("query", utils.TruncateString(query, -1)), zap.String("argument", utils.TruncateInterface(args, -1)), @@ -130,7 +131,7 @@ func (conn *DBConn) executeSQL(ctx *tcontext.Context, queries []string, args ... } return true } - if retry.IsRetryableError(err) { + if dbutil.IsRetryableError(err) { ctx.L().Warn("execute statements", zap.Int("retry", retryTime), zap.String("queries", utils.TruncateInterface(queries, -1)), zap.String("arguments", utils.TruncateInterface(args, -1)), diff --git a/pkg/retry/errors.go b/pkg/retry/errors.go index 1b075efb27..680c55531a 100644 --- a/pkg/retry/errors.go +++ b/pkg/retry/errors.go @@ -15,12 +15,7 @@ package retry import ( "database/sql/driver" - "strings" - - "github.com/go-sql-driver/mysql" "github.com/pingcap/errors" - tmysql "github.com/pingcap/parser/mysql" - gmysql "github.com/siddontang/go-mysql/mysql" ) var ( @@ -45,36 +40,8 @@ var ( "binlog checksum mismatch, data may be corrupted", "get event err EOF", } - - // Retryable1105Msgs list the error messages of some retryable error with 1105 code. - Retryable1105Msgs = []string{ - "Information schema is out of date", - "Information schema is changed", - } ) -// IsRetryableError tells whether this error should retry -func IsRetryableError(err error) bool { - err = errors.Cause(err) // check the original error - mysqlErr, ok := err.(*mysql.MySQLError) - if ok { - switch mysqlErr.Number { - // ER_LOCK_DEADLOCK can retry to commit while meet deadlock - case gmysql.ER_LOCK_DEADLOCK, tmysql.ErrPDServerTimeout, tmysql.ErrTiKVServerTimeout, tmysql.ErrTiKVServerBusy, tmysql.ErrResolveLockTimeout, tmysql.ErrRegionUnavailable, tmysql.ErrQueryInterrupted, tmysql.ErrWriteConflictInTiDB, tmysql.ErrTableLocked, tmysql.ErrWriteConflict: - return true - case tmysql.ErrUnknown: - for _, msg := range Retryable1105Msgs { - if strings.Contains(mysqlErr.Message, msg) { - return true - } - } - default: - return false - } - } - return false -} - // IsConnectionError tells whether this error should reconnect to Database func IsConnectionError(err error) bool { err = errors.Cause(err) diff --git a/pkg/retry/strategy_test.go b/pkg/retry/strategy_test.go index 18ee8b43d2..1fa2d9c814 100644 --- a/pkg/retry/strategy_test.go +++ b/pkg/retry/strategy_test.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/dm/pkg/terror" . "github.com/pingcap/check" + "github.com/pingcap/tidb-tools/pkg/dbutil" ) func TestSuite(t *testing.T) { @@ -69,7 +70,7 @@ func (t *testStrategySuite) TestFiniteRetryStrategy(c *C) { // invalid connection will return ErrInvalidConn immediately no matter how many retries left params.IsRetryableFn = func(int, error) bool { - return IsRetryableError(err) + return dbutil.IsRetryableError(err) } operateFn = func(*tcontext.Context) (interface{}, error) { mysqlErr := driver.ErrBadConn diff --git a/syncer/db.go b/syncer/db.go index 8af616954e..12b7d22a0c 100644 --- a/syncer/db.go +++ b/syncer/db.go @@ -31,6 +31,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/parser" tmysql "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/filter" "github.com/siddontang/go-mysql/mysql" "go.uber.org/zap" @@ -183,7 +184,7 @@ func (conn *DBConn) querySQL(tctx *tcontext.Context, query string, args ...inter sqlRetriesTotal.WithLabelValues("query", conn.cfg.Name).Add(1) return true } - if retry.IsRetryableError(err) { + if dbutil.IsRetryableError(err) { tctx.L().Warn("query statement", zap.Int("retry", retryTime), zap.String("query", utils.TruncateString(query, -1)), zap.String("argument", utils.TruncateInterface(args, -1))) @@ -238,7 +239,7 @@ func (conn *DBConn) executeSQLWithIgnore(tctx *tcontext.Context, ignoreError fun sqlRetriesTotal.WithLabelValues("stmt_exec", conn.cfg.Name).Add(1) return true } - if retry.IsRetryableError(err) { + if dbutil.IsRetryableError(err) { tctx.L().Warn("execute statements", zap.Int("retry", retryTime), zap.String("queries", utils.TruncateInterface(queries, -1)), zap.String("arguments", utils.TruncateInterface(args, -1))) diff --git a/syncer/error_test.go b/syncer/error_test.go index b682cfddcb..6a4660a064 100644 --- a/syncer/error_test.go +++ b/syncer/error_test.go @@ -14,18 +14,14 @@ package syncer import ( - "database/sql/driver" - "github.com/go-sql-driver/mysql" . "github.com/pingcap/check" "github.com/pingcap/errors" tmysql "github.com/pingcap/parser/mysql" - gmysql "github.com/siddontang/go-mysql/mysql" "github.com/pingcap/dm/pkg/conn" "github.com/pingcap/dm/pkg/context" tcontext "github.com/pingcap/dm/pkg/context" - "github.com/pingcap/dm/pkg/retry" "github.com/pingcap/dm/pkg/utils" ) @@ -36,33 +32,6 @@ func newMysqlErr(number uint16, message string) *mysql.MySQLError { } } -func (s *testSyncerSuite) TestIsRetryableError(c *C) { - cases := []struct { - err error - isRetryable bool - }{ - {newMysqlErr(tmysql.ErrNoDB, "no baseConn error"), false}, - {errors.New("unknown error"), false}, - {newMysqlErr(tmysql.ErrDBCreateExists, "baseConn already exists"), false}, - {driver.ErrBadConn, false}, - {newMysqlErr(gmysql.ER_LOCK_DEADLOCK, "Deadlock found when trying to get lock; try restarting transaction"), true}, - {newMysqlErr(tmysql.ErrPDServerTimeout, "pd server timeout"), true}, - {newMysqlErr(tmysql.ErrTiKVServerTimeout, "tikv server timeout"), true}, - {newMysqlErr(tmysql.ErrTiKVServerBusy, "tikv server busy"), true}, - {newMysqlErr(tmysql.ErrResolveLockTimeout, "resolve lock timeout"), true}, - {newMysqlErr(tmysql.ErrRegionUnavailable, "region unavailable"), true}, - {newMysqlErr(tmysql.ErrUnknown, "i/o timeout"), false}, - {newMysqlErr(tmysql.ErrUnknown, "can't drop column with index"), false}, - {newMysqlErr(tmysql.ErrUnknown, "Information schema is out of date"), true}, - {newMysqlErr(tmysql.ErrUnknown, "Information schema is changed"), true}, - } - - for _, t := range cases { - c.Logf("err %v, expected %v", t.err, t.isRetryable) - c.Assert(retry.IsRetryableError(t.err), Equals, t.isRetryable) - } -} - func (s *testSyncerSuite) TestSpecificError(c *C) { err := newMysqlErr(tmysql.ErrNoSuchThread, "Unknown thread id: 111") c.Assert(utils.IsNoSuchThreadError(err), Equals, true)