Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

syncer(dm): fix collation compatibility behaviour #6833

Merged
merged 4 commits into from
Aug 22, 2022
Merged

syncer(dm): fix collation compatibility behaviour #6833

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1a2745e
syncer(dm): fix collation compatibility behaviour
lance6716 Aug 22, 2022
aa4bcbd
fix lint
lance6716 Aug 22, 2022
cb687db
Merge branch 'master' of github.com:pingcap/ticdc into fix-strict-col…
lance6716 Aug 22, 2022
5ce08c0
Merge branch 'master' into fix-strict-collation
ti-chi-bot Aug 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion dm/syncer/ddl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,11 +275,12 @@ func adjustCollation(tctx *tcontext.Context, ddlInfo *ddlInfo, statusVars []byte

// adjustColumnsCollation adds column's collation.
func adjustColumnsCollation(tctx *tcontext.Context, createStmt *ast.CreateTableStmt, charsetAndDefaultCollationMap map[string]string) {
ColumnLoop:
for _, col := range createStmt.Cols {
for _, options := range col.Options {
// already have 'Collation'
if options.Tp == ast.ColumnOptionCollate {
continue
continue ColumnLoop
}
}
fieldType := col.Tp
Expand Down
14 changes: 9 additions & 5 deletions dm/syncer/ddl_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"errors"
"fmt"
"strings"
"testing"

"github.com/DATA-DOG/go-sqlmock"
"github.com/go-mysql-org/go-mysql/mysql"
Expand All @@ -26,6 +27,7 @@ import (
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/util/filter"
"github.com/pingcap/tiflow/dm/syncer/metrics"
"github.com/stretchr/testify/require"
"go.uber.org/zap"

regexprrouter "github.com/pingcap/tidb/util/regexpr-router"
Expand Down Expand Up @@ -703,12 +705,13 @@ func (s *testDDLSuite) TestAdjustDatabaseCollation(c *C) {
}
}

func (s *testDDLSuite) TestAdjustCollation(c *C) {
func TestAdjustCollation(t *testing.T) {
sqls := []string{
"create table `test`.`t1` (id int) CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci",
"create table `test`.`t1` (id int) CHARSET=utf8mb4",
"create table `test`.`t1` (id int) COLLATE=utf8mb4_general_ci",
"create table `test`.`t1` (id int)",
"create table `test`.`t1` (name varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)",
"create table `test`.`t1` (id int, name varchar(20) CHARACTER SET utf8mb4, work varchar(20))",
"create table `test`.`t1` (id int, name varchar(20), work varchar(20))",
"create table `test`.`t1` (id int, name varchar(20) COLLATE utf8mb4_general_ci, work varchar(20))",
Expand All @@ -730,6 +733,7 @@ func (s *testDDLSuite) TestAdjustCollation(c *C) {
"CREATE TABLE `test`.`t` (`id` INT) DEFAULT CHARACTER SET = UTF8MB4 DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT) DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT)",
"CREATE TABLE `test`.`t` (`name` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_bin)",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_general_ci,`work` VARCHAR(20))",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20),`work` VARCHAR(20))",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) COLLATE utf8mb4_general_ci,`work` VARCHAR(20))",
Expand Down Expand Up @@ -768,13 +772,13 @@ func (s *testDDLSuite) TestAdjustCollation(c *C) {
targetTables: []*filter.Table{tab},
}
stmt, err := p.ParseOneStmt(sql, "", "")
c.Assert(err, IsNil)
c.Assert(stmt, NotNil)
require.NoError(t, err)
require.NotNil(t, stmt)
ddlInfo.stmtCache = stmt
adjustCollation(tctx, ddlInfo, statusVars, charsetAndDefaultCollationMap, idAndCollationMap)
routedDDL, err := parserpkg.RenameDDLTable(ddlInfo.stmtCache, ddlInfo.targetTables)
c.Assert(err, IsNil)
c.Assert(routedDDL, Equals, expectedSQLs[i])
require.NoError(t, err)
require.Equal(t, expectedSQLs[i], routedDDL)
}
}

Expand Down
53 changes: 28 additions & 25 deletions engine/jobmaster/dm/runtime/worker_status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,32 +24,35 @@ import (
// TODO: expose this config in lib
var HeartbeatInterval = 3 * time.Second

/*
,──────────────. ,────────────. ,─────────────. ,──────────────.
│WorkerCreating│ │WorkerOnline│ │WorkerOffline│ │WorkerFinished│
`──────┬───────' `─────┬──────' `──────┬──────' `──────┬───────'
│ │ │ │
CreateWorker │ │ │ │
───────────────►│ │ │ │
│ OnWorkerOnline │ │ │
├───────────────────►│ │ │
│ │ OnWorkerOffline │ │
│ ├───────────────────►│ │
│ │ │ │
│ │ │ │
│ │ OnWorkerFinished │ │
│ ├────────────────────┼──────────────────►│
│ │ │ │
│ OnWorkerOffline/OnWorkerDispacth │ │
├────────────────────┬───────────────────►│ │
│ │ │ │
│ │ │ │
│ │ │ │
│ │ │ │
│ OnWorkerFinished │ │ │
├────────────────────┼────────────────────┼──────────────────►│
│ │ │ │
│ │ │ │
*/

// WorkerStage represents the stage of a worker.
// ,──────────────. ,────────────. ,─────────────. ,──────────────.
// │WorkerCreating│ │WorkerOnline│ │WorkerOffline│ │WorkerFinished│
// `──────┬───────' `─────┬──────' `──────┬──────' `──────┬───────'
// │ │ │ │
// CreateWorker │ │ │ │
// ───────────────►│ │ │ │
// │ OnWorkerOnline │ │ │
// ├───────────────────►│ │ │
// │ │ OnWorkerOffline │ │
// │ ├───────────────────►│ │
// │ │ │ │
// │ │ │ │
// │ │ OnWorkerFinished │ │
// │ ├────────────────────┼──────────────────►│
// │ │ │ │
// │ OnWorkerOffline/OnWorkerDispacth │ │
// ├────────────────────┬───────────────────►│ │
// │ │ │ │
// │ │ │ │
// │ │ │ │
// │ │ │ │
// │ OnWorkerFinished │ │ │
// ├────────────────────┼────────────────────┼──────────────────►│
// │ │ │ │
// │ │ │ │
type WorkerStage int

// All available WorkerStage
Expand Down
8 changes: 4 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,9 @@ require (
github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3
github.com/pingcap/kvproto v0.0.0-20220804022843-f006036b1277
github.com/pingcap/log v1.1.0
github.com/pingcap/tidb v1.1.0-beta.0.20220809031647-1436ab9e7b4d
github.com/pingcap/tidb v1.1.0-beta.0.20220819091852-41b9e26d1ef4
github.com/pingcap/tidb-tools v6.1.1-0.20220715000306-1d2f00da8c3e+incompatible
github.com/pingcap/tidb/parser v0.0.0-20220809031647-1436ab9e7b4d
github.com/pingcap/tidb/parser v0.0.0-20220819091852-41b9e26d1ef4
github.com/prometheus/client_golang v1.12.2
github.com/prometheus/client_model v0.2.0
github.com/r3labs/diff v1.1.0
Expand All @@ -74,7 +74,7 @@ require (
github.com/swaggo/gin-swagger v1.2.0
github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954
github.com/tikv/client-go/v2 v2.0.1-0.20220729034404-e10841f2d158
github.com/tikv/client-go/v2 v2.0.1-0.20220818084834-0d0ae0dcfb1f
github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800
github.com/tikv/pd/client v0.0.0-20220725055910-7187a7ab72db
github.com/tinylib/msgp v1.1.6
Expand Down Expand Up @@ -129,7 +129,7 @@ require (
github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 // indirect
github.com/ardielle/ardielle-go v1.5.2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blacktear23/go-proxyprotocol v1.0.0 // indirect
github.com/blacktear23/go-proxyprotocol v1.0.2 // indirect
github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect
github.com/carlmjohnson/flagext v0.21.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
Expand Down
16 changes: 8 additions & 8 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU=
github.com/blacktear23/go-proxyprotocol v1.0.0 h1:WmMmtZanGEfIHnJN9N3A4Pl6mM69D+GxEph2eOaCf7g=
github.com/blacktear23/go-proxyprotocol v1.0.0/go.mod h1:fbqiWSHMxaW0KsJ3SHjpxOMbTpIaQSMRn1GRd+oPyEw=
github.com/blacktear23/go-proxyprotocol v1.0.2 h1:zR7PZeoU0wAkElcIXenFiy3R56WB6A+UEVi4c6RH8wo=
github.com/blacktear23/go-proxyprotocol v1.0.2/go.mod h1:FSCbgnRZrQXazBLL5snfBbrcFSMtcmUDhSRb9OfFA1o=
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
github.com/bradleyjkemp/cupaloy/v2 v2.5.0/go.mod h1:TD5UU0rdYTbu/TtuwFuWrtiRARuN7mtRipvs/bsShSE=
github.com/bradleyjkemp/grpc-tools v0.2.5 h1:zZhwRxFktKIZliZ7g+V6zwNl0m9o/W1kvWJFWRxkZ/Q=
Expand Down Expand Up @@ -1013,15 +1013,15 @@ github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d/go.mod h1:7j18ezaW
github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 h1:HYbcxtnkN3s5tqrZ/z3eJS4j3Db8wMphEm1q10lY/TM=
github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4/go.mod h1:sDCsM39cGiv2vwunZkaFA917vVkqDTGSPbbV7z4Oops=
github.com/pingcap/tidb v1.1.0-beta.0.20220511160835-98c31070d958/go.mod h1:luW4sIZoLHY3bCWuKqyqk2QgMvF+/M7nWOXf/me0+fY=
github.com/pingcap/tidb v1.1.0-beta.0.20220809031647-1436ab9e7b4d h1:ZNargFmi0VykeOkFcPVdoHd4nRV5h5hrvjIX/Cz5LR4=
github.com/pingcap/tidb v1.1.0-beta.0.20220809031647-1436ab9e7b4d/go.mod h1:JeFDyJA2hfiXfE8wYKdortn6OO4y/1ptMjzX7kARLwk=
github.com/pingcap/tidb v1.1.0-beta.0.20220819091852-41b9e26d1ef4 h1:nAljfJKXkTLfMmlRQzy9Qp34Kj5tBtku/YCG9YezPqw=
github.com/pingcap/tidb v1.1.0-beta.0.20220819091852-41b9e26d1ef4/go.mod h1:ibrqg2O6i98YbT6al8tpoz824bcHQlQKyV7VxpC1RH0=
github.com/pingcap/tidb-dashboard v0.0.0-20220117082709-e8076b5c79ba/go.mod h1:4hk/3owVGWdvI9Kx6yCqqvM1T5PVgwyQNyMQxD3rwfc=
github.com/pingcap/tidb-tools v6.1.1-0.20220715000306-1d2f00da8c3e+incompatible h1:ftmrSd7avCEdTOkWx3O0UkS4yTBrlKQweRF8uqz9+No=
github.com/pingcap/tidb-tools v6.1.1-0.20220715000306-1d2f00da8c3e+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e/go.mod h1:e1MGCA9Sg3T8jid8PKAEq5eYVuMMCq4n8gJ+Kqp4Plg=
github.com/pingcap/tidb/parser v0.0.0-20220511160835-98c31070d958/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI=
github.com/pingcap/tidb/parser v0.0.0-20220809031647-1436ab9e7b4d h1:aF0pFWXUwwA9PQSLZo2pxeFTLfwbahUHCYXwNDYJBgI=
github.com/pingcap/tidb/parser v0.0.0-20220809031647-1436ab9e7b4d/go.mod h1:wjvp+T3/T9XYt0nKqGX3Kc1AKuyUcfno6LTc6b2A6ew=
github.com/pingcap/tidb/parser v0.0.0-20220819091852-41b9e26d1ef4 h1:NShT1vSsXXqCvcAe8+rIPKpMJ32qOCxbmAVqh3OQ6bc=
github.com/pingcap/tidb/parser v0.0.0-20220819091852-41b9e26d1ef4/go.mod h1:wjvp+T3/T9XYt0nKqGX3Kc1AKuyUcfno6LTc6b2A6ew=
github.com/pingcap/tipb v0.0.0-20220215045658-d12dec7a7609/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4=
github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
Expand Down Expand Up @@ -1200,8 +1200,8 @@ github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJH
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tikv/client-go/v2 v2.0.1-0.20220510032238-ff5e35ac2869/go.mod h1:0scaG+seu7L56apm+Gjz9vckyO7ABIzM6T7n00mrIXs=
github.com/tikv/client-go/v2 v2.0.1-0.20220729034404-e10841f2d158 h1:oCtRW/f0FZabdoLuvqxIewcmHR83RlsdN37dS0EBRTU=
github.com/tikv/client-go/v2 v2.0.1-0.20220729034404-e10841f2d158/go.mod h1:v3DEt8LS9olI6D6El17pYBWq7B28hw3NnDFTxQHDLpY=
github.com/tikv/client-go/v2 v2.0.1-0.20220818084834-0d0ae0dcfb1f h1:/nr7P8uzJQ7u3wPEBHCokrsVmuDvi/1x/zI/ydk5n8U=
github.com/tikv/client-go/v2 v2.0.1-0.20220818084834-0d0ae0dcfb1f/go.mod h1:v3DEt8LS9olI6D6El17pYBWq7B28hw3NnDFTxQHDLpY=
github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 h1:lIfIwqe1HPa0suhMpiI200nYxau+rXWXTqZxSGg1HS4=
github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800/go.mod h1:J/dj1zpEE9b7idgONGFttnXM+ncl88LmnkD/xDbq0hA=
github.com/tikv/pd/client v0.0.0-20220307081149-841fa61e9710/go.mod h1:AtvppPwkiyUgQlR1W9qSqfTB+OsOIu19jDCOxOsPkmU=
Expand Down