Skip to content

Commit

Permalink
This is an automated cherry-pick of pingcap#6833
Browse files Browse the repository at this point in the history
Signed-off-by: ti-chi-bot <[email protected]>
  • Loading branch information
lance6716 authored and ti-chi-bot committed Aug 22, 2022
1 parent 147051d commit 2176ecc
Show file tree
Hide file tree
Showing 5 changed files with 170 additions and 2 deletions.
3 changes: 2 additions & 1 deletion dm/syncer/ddl.go
Original file line number Diff line number Diff line change
Expand Up @@ -273,11 +273,12 @@ func adjustCollation(tctx *tcontext.Context, ddlInfo *ddlInfo, statusVars []byte

// adjustColumnsCollation adds column's collation.
func adjustColumnsCollation(tctx *tcontext.Context, createStmt *ast.CreateTableStmt, charsetAndDefaultCollationMap map[string]string) {
ColumnLoop:
for _, col := range createStmt.Cols {
for _, options := range col.Options {
// already have 'Collation'
if options.Tp == ast.ColumnOptionCollate {
continue
continue ColumnLoop
}
}
fieldType := col.Tp
Expand Down
20 changes: 19 additions & 1 deletion dm/syncer/ddl_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,18 @@ import (
"errors"
"fmt"
"strings"
"testing"

"github.com/DATA-DOG/go-sqlmock"
. "github.com/pingcap/check"
"github.com/pingcap/tidb/parser"
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/util/filter"
<<<<<<< HEAD
=======
"github.com/pingcap/tiflow/dm/syncer/metrics"
"github.com/stretchr/testify/require"
>>>>>>> 750f0e4dd (syncer(dm): fix collation compatibility behaviour (#6833))
"go.uber.org/zap"

regexprrouter "github.com/pingcap/tidb/util/regexpr-router"
Expand Down Expand Up @@ -696,12 +702,13 @@ func (s *testDDLSuite) TestAdjustDatabaseCollation(c *C) {
}
}

func (s *testDDLSuite) TestAdjustCollation(c *C) {
func TestAdjustCollation(t *testing.T) {
sqls := []string{
"create table `test`.`t1` (id int) CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci",
"create table `test`.`t1` (id int) CHARSET=utf8mb4",
"create table `test`.`t1` (id int) COLLATE=utf8mb4_general_ci",
"create table `test`.`t1` (id int)",
"create table `test`.`t1` (name varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)",
"create table `test`.`t1` (id int, name varchar(20) CHARACTER SET utf8mb4, work varchar(20))",
"create table `test`.`t1` (id int, name varchar(20), work varchar(20))",
"create table `test`.`t1` (id int, name varchar(20) COLLATE utf8mb4_general_ci, work varchar(20))",
Expand All @@ -723,6 +730,7 @@ func (s *testDDLSuite) TestAdjustCollation(c *C) {
"CREATE TABLE `test`.`t` (`id` INT) DEFAULT CHARACTER SET = UTF8MB4 DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT) DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT)",
"CREATE TABLE `test`.`t` (`name` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_bin)",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_general_ci,`work` VARCHAR(20))",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20),`work` VARCHAR(20))",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) COLLATE utf8mb4_general_ci,`work` VARCHAR(20))",
Expand Down Expand Up @@ -758,13 +766,23 @@ func (s *testDDLSuite) TestAdjustCollation(c *C) {
targetTables: []*filter.Table{tab},
}
stmt, err := p.ParseOneStmt(sql, "", "")
<<<<<<< HEAD
c.Assert(err, IsNil)
c.Assert(stmt, NotNil)
ddlInfo.originStmt = stmt
adjustCollation(tctx, ddlInfo, statusVars, charsetAndDefaultCollationMap, idAndCollationMap)
routedDDL, err := parserpkg.RenameDDLTable(ddlInfo.originStmt, ddlInfo.targetTables)
c.Assert(err, IsNil)
c.Assert(routedDDL, Equals, expectedSQLs[i])
=======
require.NoError(t, err)
require.NotNil(t, stmt)
ddlInfo.stmtCache = stmt
adjustCollation(tctx, ddlInfo, statusVars, charsetAndDefaultCollationMap, idAndCollationMap)
routedDDL, err := parserpkg.RenameDDLTable(ddlInfo.stmtCache, ddlInfo.targetTables)
require.NoError(t, err)
require.Equal(t, expectedSQLs[i], routedDDL)
>>>>>>> 750f0e4dd (syncer(dm): fix collation compatibility behaviour (#6833))
}
}

Expand Down
115 changes: 115 additions & 0 deletions engine/jobmaster/dm/runtime/worker_status.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
// Copyright 2022 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package runtime

import (
"time"

"github.com/pingcap/tiflow/engine/framework"
frameModel "github.com/pingcap/tiflow/engine/framework/model"
)

// HeartbeatInterval is heartbeat interval for checking worker stage
// TODO: expose this config in lib
var HeartbeatInterval = 3 * time.Second

/*
,──────────────. ,────────────. ,─────────────. ,──────────────.
│WorkerCreating│ │WorkerOnline│ │WorkerOffline│ │WorkerFinished│
`──────┬───────' `─────┬──────' `──────┬──────' `──────┬───────'
│ │ │ │
CreateWorker │ │ │ │
───────────────►│ │ │ │
│ OnWorkerOnline │ │ │
├───────────────────►│ │ │
│ │ OnWorkerOffline │ │
│ ├───────────────────►│ │
│ │ │ │
│ │ │ │
│ │ OnWorkerFinished │ │
│ ├────────────────────┼──────────────────►│
│ │ │ │
│ OnWorkerOffline/OnWorkerDispacth │ │
├────────────────────┬───────────────────►│ │
│ │ │ │
│ │ │ │
│ │ │ │
│ │ │ │
│ OnWorkerFinished │ │ │
├────────────────────┼────────────────────┼──────────────────►│
│ │ │ │
│ │ │ │
*/

// WorkerStage represents the stage of a worker.
type WorkerStage int

// All available WorkerStage
const (
WorkerCreating WorkerStage = iota
WorkerOnline
WorkerFinished
WorkerOffline
// WorkerDestroying
)

// WorkerStatus manages worker state machine
type WorkerStatus struct {
TaskID string
ID frameModel.WorkerID
Unit framework.WorkerType
Stage WorkerStage
CfgModRevision uint64
// only use when creating, change to updatedTime if needed.
createdTime time.Time
}

// IsOffline checks whether worker stage is offline
func (w *WorkerStatus) IsOffline() bool {
return w.Stage == WorkerOffline
}

// IsTombStone returns whether the worker is tombstone, which means we don't need to stop it.
func (w *WorkerStatus) IsTombStone() bool {
return w.Stage == WorkerOffline || w.Stage == WorkerFinished || w.CreateFailed()
}

// CreateFailed checks whether the worker creation is failed
func (w *WorkerStatus) CreateFailed() bool {
return w.Stage == WorkerCreating && w.createdTime.Add(2*HeartbeatInterval).Before(time.Now())
}

// RunAsExpected returns whether a worker is running.
// Currently, we regard worker run as expected except it is offline.
func (w *WorkerStatus) RunAsExpected() bool {
return w.Stage == WorkerOnline || w.Stage == WorkerCreating || w.Stage == WorkerFinished
}

// InitWorkerStatus creates a new worker status and initializes it
func InitWorkerStatus(taskID string, unit framework.WorkerType, id frameModel.WorkerID) WorkerStatus {
workerStatus := NewWorkerStatus(taskID, unit, id, WorkerCreating, 0)
workerStatus.createdTime = time.Now()
return workerStatus
}

// NewWorkerStatus creates a new WorkerStatus instance
func NewWorkerStatus(taskID string, unit framework.WorkerType, id frameModel.WorkerID, stage WorkerStage, cfgModRevision uint64) WorkerStatus {
return WorkerStatus{
TaskID: taskID,
ID: id,
Unit: unit,
Stage: stage,
CfgModRevision: cfgModRevision,
}
}
14 changes: 14 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,15 @@ require (
github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3
github.com/pingcap/kvproto v0.0.0-20220517085838-12e2f5a9d167
github.com/pingcap/log v1.1.0
<<<<<<< HEAD
github.com/pingcap/tidb v1.1.0-beta.0.20220727102111-099c736ff30a
github.com/pingcap/tidb-tools v6.0.1-0.20220516050036-b3ea358e374a+incompatible
github.com/pingcap/tidb/parser v0.0.0-20220727102111-099c736ff30a
=======
github.com/pingcap/tidb v1.1.0-beta.0.20220819091852-41b9e26d1ef4
github.com/pingcap/tidb-tools v6.1.1-0.20220715000306-1d2f00da8c3e+incompatible
github.com/pingcap/tidb/parser v0.0.0-20220819091852-41b9e26d1ef4
>>>>>>> 750f0e4dd (syncer(dm): fix collation compatibility behaviour (#6833))
github.com/prometheus/client_golang v1.12.2
github.com/prometheus/client_model v0.2.0
github.com/r3labs/diff v1.1.0
Expand All @@ -66,7 +72,11 @@ require (
github.com/swaggo/gin-swagger v1.2.0
github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954
<<<<<<< HEAD
github.com/tikv/client-go/v2 v2.0.1-0.20220531092439-efebaeb9fe53
=======
github.com/tikv/client-go/v2 v2.0.1-0.20220818084834-0d0ae0dcfb1f
>>>>>>> 750f0e4dd (syncer(dm): fix collation compatibility behaviour (#6833))
github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800
github.com/tikv/pd/client v0.0.0-20220307081149-841fa61e9710
github.com/tinylib/msgp v1.1.6
Expand Down Expand Up @@ -114,7 +124,11 @@ require (
github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 // indirect
github.com/ardielle/ardielle-go v1.5.2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
<<<<<<< HEAD
github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d // indirect
=======
github.com/blacktear23/go-proxyprotocol v1.0.2 // indirect
>>>>>>> 750f0e4dd (syncer(dm): fix collation compatibility behaviour (#6833))
github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect
github.com/carlmjohnson/flagext v0.21.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
Expand Down
20 changes: 20 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,11 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQlvB2AYWme2bIB18r/SipGiMEVJYE9U0z+MGoU/LtQ=
github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU=
<<<<<<< HEAD
=======
github.com/blacktear23/go-proxyprotocol v1.0.2 h1:zR7PZeoU0wAkElcIXenFiy3R56WB6A+UEVi4c6RH8wo=
github.com/blacktear23/go-proxyprotocol v1.0.2/go.mod h1:FSCbgnRZrQXazBLL5snfBbrcFSMtcmUDhSRb9OfFA1o=
>>>>>>> 750f0e4dd (syncer(dm): fix collation compatibility behaviour (#6833))
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
github.com/bradleyjkemp/cupaloy/v2 v2.5.0/go.mod h1:TD5UU0rdYTbu/TtuwFuWrtiRARuN7mtRipvs/bsShSE=
github.com/bradleyjkemp/grpc-tools v0.2.5 h1:zZhwRxFktKIZliZ7g+V6zwNl0m9o/W1kvWJFWRxkZ/Q=
Expand Down Expand Up @@ -965,15 +970,25 @@ github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d/go.mod h1:7j18ezaW
github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 h1:HYbcxtnkN3s5tqrZ/z3eJS4j3Db8wMphEm1q10lY/TM=
github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4/go.mod h1:sDCsM39cGiv2vwunZkaFA917vVkqDTGSPbbV7z4Oops=
github.com/pingcap/tidb v1.1.0-beta.0.20220511160835-98c31070d958/go.mod h1:luW4sIZoLHY3bCWuKqyqk2QgMvF+/M7nWOXf/me0+fY=
<<<<<<< HEAD
github.com/pingcap/tidb v1.1.0-beta.0.20220727102111-099c736ff30a h1:pxJOc27XqLtVC4+AzhcriTlMTO+J+EQRHf7KfMbe0+8=
github.com/pingcap/tidb v1.1.0-beta.0.20220727102111-099c736ff30a/go.mod h1:z8u+NvIbYDO2f/Y1sJR0exTeCYNwFb/755Z4kZAlBM8=
=======
github.com/pingcap/tidb v1.1.0-beta.0.20220819091852-41b9e26d1ef4 h1:nAljfJKXkTLfMmlRQzy9Qp34Kj5tBtku/YCG9YezPqw=
github.com/pingcap/tidb v1.1.0-beta.0.20220819091852-41b9e26d1ef4/go.mod h1:ibrqg2O6i98YbT6al8tpoz824bcHQlQKyV7VxpC1RH0=
>>>>>>> 750f0e4dd (syncer(dm): fix collation compatibility behaviour (#6833))
github.com/pingcap/tidb-dashboard v0.0.0-20220117082709-e8076b5c79ba/go.mod h1:4hk/3owVGWdvI9Kx6yCqqvM1T5PVgwyQNyMQxD3rwfc=
github.com/pingcap/tidb-tools v6.0.1-0.20220516050036-b3ea358e374a+incompatible h1:dAfZKHR4Wu3bdFDNDsH2c4fWrkqbkgVlwY5pC4Nmp6Q=
github.com/pingcap/tidb-tools v6.0.1-0.20220516050036-b3ea358e374a+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e/go.mod h1:e1MGCA9Sg3T8jid8PKAEq5eYVuMMCq4n8gJ+Kqp4Plg=
github.com/pingcap/tidb/parser v0.0.0-20220511160835-98c31070d958/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI=
<<<<<<< HEAD
github.com/pingcap/tidb/parser v0.0.0-20220727102111-099c736ff30a h1:7QQtNzfZV+WrqOlSBEB3nW4ut+n0xXJx3GETevEDf4Y=
github.com/pingcap/tidb/parser v0.0.0-20220727102111-099c736ff30a/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI=
=======
github.com/pingcap/tidb/parser v0.0.0-20220819091852-41b9e26d1ef4 h1:NShT1vSsXXqCvcAe8+rIPKpMJ32qOCxbmAVqh3OQ6bc=
github.com/pingcap/tidb/parser v0.0.0-20220819091852-41b9e26d1ef4/go.mod h1:wjvp+T3/T9XYt0nKqGX3Kc1AKuyUcfno6LTc6b2A6ew=
>>>>>>> 750f0e4dd (syncer(dm): fix collation compatibility behaviour (#6833))
github.com/pingcap/tipb v0.0.0-20220215045658-d12dec7a7609/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
github.com/pingcap/tipb v0.0.0-20220314125451-bfb5c2c55188 h1:+46isFI9fR9R+nJVDMI55tCC/TCwp+bvVA4HLGEv1rY=
github.com/pingcap/tipb v0.0.0-20220314125451-bfb5c2c55188/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
Expand Down Expand Up @@ -1143,8 +1158,13 @@ github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJH
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tikv/client-go/v2 v2.0.1-0.20220510032238-ff5e35ac2869/go.mod h1:0scaG+seu7L56apm+Gjz9vckyO7ABIzM6T7n00mrIXs=
<<<<<<< HEAD
github.com/tikv/client-go/v2 v2.0.1-0.20220531092439-efebaeb9fe53 h1:zalDvjC3IhixTcqU1HQYJQtTI3npketLDprwLw1eGqI=
github.com/tikv/client-go/v2 v2.0.1-0.20220531092439-efebaeb9fe53/go.mod h1:VTlli8fRRpcpISj9I2IqroQmcAFfaTyBquiRhofOcDs=
=======
github.com/tikv/client-go/v2 v2.0.1-0.20220818084834-0d0ae0dcfb1f h1:/nr7P8uzJQ7u3wPEBHCokrsVmuDvi/1x/zI/ydk5n8U=
github.com/tikv/client-go/v2 v2.0.1-0.20220818084834-0d0ae0dcfb1f/go.mod h1:v3DEt8LS9olI6D6El17pYBWq7B28hw3NnDFTxQHDLpY=
>>>>>>> 750f0e4dd (syncer(dm): fix collation compatibility behaviour (#6833))
github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 h1:lIfIwqe1HPa0suhMpiI200nYxau+rXWXTqZxSGg1HS4=
github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800/go.mod h1:J/dj1zpEE9b7idgONGFttnXM+ncl88LmnkD/xDbq0hA=
github.com/tikv/pd/client v0.0.0-20220307081149-841fa61e9710 h1:jxgmKOscXSjaFEKQGRyY5qOpK8hLqxs2irb/uDJMtwk=
Expand Down

0 comments on commit 2176ecc

Please sign in to comment.