From 1a2745e79b90e7181b851091dc0e846892f9be9c Mon Sep 17 00:00:00 2001 From: lance6716 Date: Mon, 22 Aug 2022 10:37:05 +0800 Subject: [PATCH 1/2] syncer(dm): fix collation compatibility behaviour Signed-off-by: lance6716 --- dm/syncer/ddl.go | 3 ++- dm/syncer/ddl_test.go | 14 +++++++++----- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/dm/syncer/ddl.go b/dm/syncer/ddl.go index 0713c7e6c1c..716a82d1443 100644 --- a/dm/syncer/ddl.go +++ b/dm/syncer/ddl.go @@ -275,11 +275,12 @@ func adjustCollation(tctx *tcontext.Context, ddlInfo *ddlInfo, statusVars []byte // adjustColumnsCollation adds column's collation. func adjustColumnsCollation(tctx *tcontext.Context, createStmt *ast.CreateTableStmt, charsetAndDefaultCollationMap map[string]string) { +ColumnLoop: for _, col := range createStmt.Cols { for _, options := range col.Options { // already have 'Collation' if options.Tp == ast.ColumnOptionCollate { - continue + continue ColumnLoop } } fieldType := col.Tp diff --git a/dm/syncer/ddl_test.go b/dm/syncer/ddl_test.go index f5e6cbf3b0c..88da8441c38 100644 --- a/dm/syncer/ddl_test.go +++ b/dm/syncer/ddl_test.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "strings" + "testing" "github.com/DATA-DOG/go-sqlmock" "github.com/go-mysql-org/go-mysql/mysql" @@ -26,6 +27,7 @@ import ( "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/util/filter" "github.com/pingcap/tiflow/dm/syncer/metrics" + "github.com/stretchr/testify/require" "go.uber.org/zap" regexprrouter "github.com/pingcap/tidb/util/regexpr-router" @@ -703,12 +705,13 @@ func (s *testDDLSuite) TestAdjustDatabaseCollation(c *C) { } } -func (s *testDDLSuite) TestAdjustCollation(c *C) { +func TestAdjustCollation(t *testing.T) { sqls := []string{ "create table `test`.`t1` (id int) CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci", "create table `test`.`t1` (id int) CHARSET=utf8mb4", "create table `test`.`t1` (id int) COLLATE=utf8mb4_general_ci", "create table `test`.`t1` (id int)", + "create table `test`.`t1` (name varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)", "create table `test`.`t1` (id int, name varchar(20) CHARACTER SET utf8mb4, work varchar(20))", "create table `test`.`t1` (id int, name varchar(20), work varchar(20))", "create table `test`.`t1` (id int, name varchar(20) COLLATE utf8mb4_general_ci, work varchar(20))", @@ -730,6 +733,7 @@ func (s *testDDLSuite) TestAdjustCollation(c *C) { "CREATE TABLE `test`.`t` (`id` INT) DEFAULT CHARACTER SET = UTF8MB4 DEFAULT COLLATE = UTF8MB4_GENERAL_CI", "CREATE TABLE `test`.`t` (`id` INT) DEFAULT COLLATE = UTF8MB4_GENERAL_CI", "CREATE TABLE `test`.`t` (`id` INT)", + "CREATE TABLE `test`.`t` (`name` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_bin)", "CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_general_ci,`work` VARCHAR(20))", "CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20),`work` VARCHAR(20))", "CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) COLLATE utf8mb4_general_ci,`work` VARCHAR(20))", @@ -768,13 +772,13 @@ func (s *testDDLSuite) TestAdjustCollation(c *C) { targetTables: []*filter.Table{tab}, } stmt, err := p.ParseOneStmt(sql, "", "") - c.Assert(err, IsNil) - c.Assert(stmt, NotNil) + require.NoError(t, err) + require.NotNil(t, stmt) ddlInfo.stmtCache = stmt adjustCollation(tctx, ddlInfo, statusVars, charsetAndDefaultCollationMap, idAndCollationMap) routedDDL, err := parserpkg.RenameDDLTable(ddlInfo.stmtCache, ddlInfo.targetTables) - c.Assert(err, IsNil) - c.Assert(routedDDL, Equals, expectedSQLs[i]) + require.NoError(t, err) + require.Equal(t, expectedSQLs[i], routedDDL) } } diff --git a/go.mod b/go.mod index 37b63e806f2..5873cea20ce 100644 --- a/go.mod +++ b/go.mod @@ -58,9 +58,9 @@ require ( github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 github.com/pingcap/kvproto v0.0.0-20220804022843-f006036b1277 github.com/pingcap/log v1.1.0 - github.com/pingcap/tidb v1.1.0-beta.0.20220809031647-1436ab9e7b4d + github.com/pingcap/tidb v1.1.0-beta.0.20220819091852-41b9e26d1ef4 github.com/pingcap/tidb-tools v6.1.1-0.20220715000306-1d2f00da8c3e+incompatible - github.com/pingcap/tidb/parser v0.0.0-20220809031647-1436ab9e7b4d + github.com/pingcap/tidb/parser v0.0.0-20220819091852-41b9e26d1ef4 github.com/prometheus/client_golang v1.12.2 github.com/prometheus/client_model v0.2.0 github.com/r3labs/diff v1.1.0 @@ -74,7 +74,7 @@ require ( github.com/swaggo/gin-swagger v1.2.0 github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476 github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 - github.com/tikv/client-go/v2 v2.0.1-0.20220729034404-e10841f2d158 + github.com/tikv/client-go/v2 v2.0.1-0.20220818084834-0d0ae0dcfb1f github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 github.com/tikv/pd/client v0.0.0-20220725055910-7187a7ab72db github.com/tinylib/msgp v1.1.6 @@ -129,7 +129,7 @@ require ( github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 // indirect github.com/ardielle/ardielle-go v1.5.2 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/blacktear23/go-proxyprotocol v1.0.0 // indirect + github.com/blacktear23/go-proxyprotocol v1.0.2 // indirect github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect github.com/carlmjohnson/flagext v0.21.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index ef2dc6c82f0..62d750f310f 100644 --- a/go.sum +++ b/go.sum @@ -166,8 +166,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= -github.com/blacktear23/go-proxyprotocol v1.0.0 h1:WmMmtZanGEfIHnJN9N3A4Pl6mM69D+GxEph2eOaCf7g= -github.com/blacktear23/go-proxyprotocol v1.0.0/go.mod h1:fbqiWSHMxaW0KsJ3SHjpxOMbTpIaQSMRn1GRd+oPyEw= +github.com/blacktear23/go-proxyprotocol v1.0.2 h1:zR7PZeoU0wAkElcIXenFiy3R56WB6A+UEVi4c6RH8wo= +github.com/blacktear23/go-proxyprotocol v1.0.2/go.mod h1:FSCbgnRZrQXazBLL5snfBbrcFSMtcmUDhSRb9OfFA1o= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= github.com/bradleyjkemp/cupaloy/v2 v2.5.0/go.mod h1:TD5UU0rdYTbu/TtuwFuWrtiRARuN7mtRipvs/bsShSE= github.com/bradleyjkemp/grpc-tools v0.2.5 h1:zZhwRxFktKIZliZ7g+V6zwNl0m9o/W1kvWJFWRxkZ/Q= @@ -1013,15 +1013,15 @@ github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d/go.mod h1:7j18ezaW github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 h1:HYbcxtnkN3s5tqrZ/z3eJS4j3Db8wMphEm1q10lY/TM= github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4/go.mod h1:sDCsM39cGiv2vwunZkaFA917vVkqDTGSPbbV7z4Oops= github.com/pingcap/tidb v1.1.0-beta.0.20220511160835-98c31070d958/go.mod h1:luW4sIZoLHY3bCWuKqyqk2QgMvF+/M7nWOXf/me0+fY= -github.com/pingcap/tidb v1.1.0-beta.0.20220809031647-1436ab9e7b4d h1:ZNargFmi0VykeOkFcPVdoHd4nRV5h5hrvjIX/Cz5LR4= -github.com/pingcap/tidb v1.1.0-beta.0.20220809031647-1436ab9e7b4d/go.mod h1:JeFDyJA2hfiXfE8wYKdortn6OO4y/1ptMjzX7kARLwk= +github.com/pingcap/tidb v1.1.0-beta.0.20220819091852-41b9e26d1ef4 h1:nAljfJKXkTLfMmlRQzy9Qp34Kj5tBtku/YCG9YezPqw= +github.com/pingcap/tidb v1.1.0-beta.0.20220819091852-41b9e26d1ef4/go.mod h1:ibrqg2O6i98YbT6al8tpoz824bcHQlQKyV7VxpC1RH0= github.com/pingcap/tidb-dashboard v0.0.0-20220117082709-e8076b5c79ba/go.mod h1:4hk/3owVGWdvI9Kx6yCqqvM1T5PVgwyQNyMQxD3rwfc= github.com/pingcap/tidb-tools v6.1.1-0.20220715000306-1d2f00da8c3e+incompatible h1:ftmrSd7avCEdTOkWx3O0UkS4yTBrlKQweRF8uqz9+No= github.com/pingcap/tidb-tools v6.1.1-0.20220715000306-1d2f00da8c3e+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e/go.mod h1:e1MGCA9Sg3T8jid8PKAEq5eYVuMMCq4n8gJ+Kqp4Plg= github.com/pingcap/tidb/parser v0.0.0-20220511160835-98c31070d958/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI= -github.com/pingcap/tidb/parser v0.0.0-20220809031647-1436ab9e7b4d h1:aF0pFWXUwwA9PQSLZo2pxeFTLfwbahUHCYXwNDYJBgI= -github.com/pingcap/tidb/parser v0.0.0-20220809031647-1436ab9e7b4d/go.mod h1:wjvp+T3/T9XYt0nKqGX3Kc1AKuyUcfno6LTc6b2A6ew= +github.com/pingcap/tidb/parser v0.0.0-20220819091852-41b9e26d1ef4 h1:NShT1vSsXXqCvcAe8+rIPKpMJ32qOCxbmAVqh3OQ6bc= +github.com/pingcap/tidb/parser v0.0.0-20220819091852-41b9e26d1ef4/go.mod h1:wjvp+T3/T9XYt0nKqGX3Kc1AKuyUcfno6LTc6b2A6ew= github.com/pingcap/tipb v0.0.0-20220215045658-d12dec7a7609/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= @@ -1200,8 +1200,8 @@ github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJH github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tikv/client-go/v2 v2.0.1-0.20220510032238-ff5e35ac2869/go.mod h1:0scaG+seu7L56apm+Gjz9vckyO7ABIzM6T7n00mrIXs= -github.com/tikv/client-go/v2 v2.0.1-0.20220729034404-e10841f2d158 h1:oCtRW/f0FZabdoLuvqxIewcmHR83RlsdN37dS0EBRTU= -github.com/tikv/client-go/v2 v2.0.1-0.20220729034404-e10841f2d158/go.mod h1:v3DEt8LS9olI6D6El17pYBWq7B28hw3NnDFTxQHDLpY= +github.com/tikv/client-go/v2 v2.0.1-0.20220818084834-0d0ae0dcfb1f h1:/nr7P8uzJQ7u3wPEBHCokrsVmuDvi/1x/zI/ydk5n8U= +github.com/tikv/client-go/v2 v2.0.1-0.20220818084834-0d0ae0dcfb1f/go.mod h1:v3DEt8LS9olI6D6El17pYBWq7B28hw3NnDFTxQHDLpY= github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 h1:lIfIwqe1HPa0suhMpiI200nYxau+rXWXTqZxSGg1HS4= github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800/go.mod h1:J/dj1zpEE9b7idgONGFttnXM+ncl88LmnkD/xDbq0hA= github.com/tikv/pd/client v0.0.0-20220307081149-841fa61e9710/go.mod h1:AtvppPwkiyUgQlR1W9qSqfTB+OsOIu19jDCOxOsPkmU= From aa4bcbd9c85b8d989405cc237c9e6ebfbd14529a Mon Sep 17 00:00:00 2001 From: lance6716 Date: Mon, 22 Aug 2022 13:04:19 +0800 Subject: [PATCH 2/2] fix lint Signed-off-by: lance6716 --- engine/jobmaster/dm/runtime/worker_status.go | 53 +++++++++++--------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/engine/jobmaster/dm/runtime/worker_status.go b/engine/jobmaster/dm/runtime/worker_status.go index 1daffd84f54..9ec3603a639 100644 --- a/engine/jobmaster/dm/runtime/worker_status.go +++ b/engine/jobmaster/dm/runtime/worker_status.go @@ -24,32 +24,35 @@ import ( // TODO: expose this config in lib var HeartbeatInterval = 3 * time.Second +/* + ,──────────────. ,────────────. ,─────────────. ,──────────────. + │WorkerCreating│ │WorkerOnline│ │WorkerOffline│ │WorkerFinished│ + `──────┬───────' `─────┬──────' `──────┬──────' `──────┬───────' + │ │ │ │ + CreateWorker │ │ │ │ +───────────────►│ │ │ │ + │ OnWorkerOnline │ │ │ + ├───────────────────►│ │ │ + │ │ OnWorkerOffline │ │ + │ ├───────────────────►│ │ + │ │ │ │ + │ │ │ │ + │ │ OnWorkerFinished │ │ + │ ├────────────────────┼──────────────────►│ + │ │ │ │ + │ OnWorkerOffline/OnWorkerDispacth │ │ + ├────────────────────┬───────────────────►│ │ + │ │ │ │ + │ │ │ │ + │ │ │ │ + │ │ │ │ + │ OnWorkerFinished │ │ │ + ├────────────────────┼────────────────────┼──────────────────►│ + │ │ │ │ + │ │ │ │ +*/ + // WorkerStage represents the stage of a worker. -// ,──────────────. ,────────────. ,─────────────. ,──────────────. -// │WorkerCreating│ │WorkerOnline│ │WorkerOffline│ │WorkerFinished│ -// `──────┬───────' `─────┬──────' `──────┬──────' `──────┬───────' -// │ │ │ │ -// CreateWorker │ │ │ │ -// ───────────────►│ │ │ │ -// │ OnWorkerOnline │ │ │ -// ├───────────────────►│ │ │ -// │ │ OnWorkerOffline │ │ -// │ ├───────────────────►│ │ -// │ │ │ │ -// │ │ │ │ -// │ │ OnWorkerFinished │ │ -// │ ├────────────────────┼──────────────────►│ -// │ │ │ │ -// │ OnWorkerOffline/OnWorkerDispacth │ │ -// ├────────────────────┬───────────────────►│ │ -// │ │ │ │ -// │ │ │ │ -// │ │ │ │ -// │ │ │ │ -// │ OnWorkerFinished │ │ │ -// ├────────────────────┼────────────────────┼──────────────────►│ -// │ │ │ │ -// │ │ │ │ type WorkerStage int // All available WorkerStage