syncer(dm): fix collation compatibility behaviour (#6833)

dm/dm/master/server_test.go

@@ -1630,9 +1630,12 @@ func (t *testMaster) testNormalServerLifecycle(c *check.C, cfg *Config, checkLog
 
 func (t *testMaster) testHTTPInterface(c *check.C, url string, contain []byte) {
 	// we use HTTPS in some test cases.
-	tls, err := toolutils.NewTLS(pwd+"/tls_for_test/ca.pem", pwd+"/tls_for_test/dm.pem", pwd+"/tls_for_test/dm.key", url, []string{})
+	tlsConfig, err := toolutils.NewTLSConfig(
+		toolutils.WithCAPath(pwd+"/tls_for_test/ca.pem"),
+		toolutils.WithCertAndKeyPath(pwd+"/tls_for_test/dm.pem", pwd+"/tls_for_test/dm.key"),
+	)
 	c.Assert(err, check.IsNil)
-	cli := toolutils.ClientWithTLS(tls.TLSConfig())
+	cli := toolutils.ClientWithTLS(tlsConfig)
 
 	// nolint:noctx
 	resp, err := cli.Get(url)

dm/dumpling/dumpling.go

@@ -299,7 +299,7 @@ func (m *Dumpling) constructArgs(ctx context.Context) (*export.Config, error) {
 
 		dumpConfig.Security.SSLCABytes = db.Security.SSLCABytes
 		dumpConfig.Security.SSLCertBytes = db.Security.SSLCertBytes
-		dumpConfig.Security.SSLKEYBytes = db.Security.SSLKEYBytes
+		dumpConfig.Security.SSLKeyBytes = db.Security.SSLKEYBytes
 	}
 
 	// `true` means dumpling will release lock after working connection established

dm/pkg/conn/basedb.go

@@ -70,16 +70,14 @@ func (d *DefaultDBProviderImpl) Apply(config *config.DBConfig) (*BaseDB, error)
 		if loadErr := config.Security.LoadTLSContent(); loadErr != nil {
 			return nil, terror.ErrCtlLoadTLSCfg.Delegate(loadErr)
 		}
-		tlsConfig, err := util.ToTLSConfigWithVerifyByRawbytes(config.Security.SSLCABytes,
-			config.Security.SSLCertBytes, config.Security.SSLKEYBytes, config.Security.CertAllowedCN)
+		tlsConfig, err := util.NewTLSConfig(
+			util.WithCAContent(config.Security.SSLCABytes),
+			util.WithCertAndKeyContent(config.Security.SSLCertBytes, config.Security.SSLKEYBytes),
+			util.WithVerifyCommonName(config.Security.CertAllowedCN),
+		)
 		if err != nil {
 			return nil, terror.ErrConnInvalidTLSConfig.Delegate(err)
 		}
-		// NOTE for local test(use a self-signed or invalid certificate), we don't need to check CA file.
-		// see more here https://github.com/go-sql-driver/mysql#tls
-		if config.Host == "127.0.0.1" || len(config.Security.SSLCertBytes) == 0 || len(config.Security.SSLKEYBytes) == 0 {
-			tlsConfig.InsecureSkipVerify = true
-		}
 
 		name := "dm" + strconv.FormatInt(atomic.AddInt64(&customID, 1), 10)
 		err = mysql.RegisterTLSConfig(name, tlsConfig)

dm/relay/relay.go

@@ -427,8 +427,10 @@ type recoverResult struct {
 // doRecovering tries to recover the current binlog file.
 // 1. read events from the file
 // 2.
-//    a. update the position with the event's position if the transaction finished
-//    b. update the GTID set with the event's GTID if the transaction finished
+//
+//	a. update the position with the event's position if the transaction finished
+//	b. update the GTID set with the event's GTID if the transaction finished
+//
 // 3. truncate any incomplete events/transactions
 // now, we think a transaction finished if we received a XIDEvent or DDL in QueryEvent
 // NOTE: handle cases when file size > 4GB.
@@ -544,10 +546,10 @@ func (r *Relay) preprocessEvent(e *replication.BinlogEvent, parser2 *parser.Pars
 }
 
 // handleEvents handles binlog events, including:
-//   1. read events from upstream
-//   2. transform events
-//   3. write events into relay log files
-//   4. update metadata if needed.
+//  1. read events from upstream
+//  2. transform events
+//  3. write events into relay log files
+//  4. update metadata if needed.
 func (r *Relay) handleEvents(
 	ctx context.Context,
 	reader2 Reader,
@@ -1129,14 +1131,14 @@ func (r *Relay) setSyncConfig() error {
 		if loadErr := r.cfg.From.Security.LoadTLSContent(); loadErr != nil {
 			return terror.ErrCtlLoadTLSCfg.Delegate(loadErr)
 		}
-		tlsConfig, err = util.ToTLSConfigWithVerifyByRawbytes(r.cfg.From.Security.SSLCABytes,
-			r.cfg.From.Security.SSLCertBytes, r.cfg.From.Security.SSLKEYBytes, r.cfg.From.Security.CertAllowedCN)
+		tlsConfig, err = util.NewTLSConfig(
+			util.WithCAContent(r.cfg.From.Security.SSLCABytes),
+			util.WithCertAndKeyContent(r.cfg.From.Security.SSLCertBytes, r.cfg.From.Security.SSLKEYBytes),
+			util.WithVerifyCommonName(r.cfg.From.Security.CertAllowedCN),
+		)
 		if err != nil {
 			return terror.ErrConnInvalidTLSConfig.Delegate(err)
 		}
-		if tlsConfig != nil {
-			tlsConfig.InsecureSkipVerify = true
-		}
 	}
 
 	syncerCfg := replication.BinlogSyncerConfig{

dm/syncer/ddl.go

@@ -273,11 +273,12 @@ func adjustCollation(tctx *tcontext.Context, ddlInfo *ddlInfo, statusVars []byte
 
 // adjustColumnsCollation adds column's collation.
 func adjustColumnsCollation(tctx *tcontext.Context, createStmt *ast.CreateTableStmt, charsetAndDefaultCollationMap map[string]string) {
+ColumnLoop:
 	for _, col := range createStmt.Cols {
 		for _, options := range col.Options {
 			// already have 'Collation'
 			if options.Tp == ast.ColumnOptionCollate {
-				continue
+				continue ColumnLoop
 			}
 		}
 		fieldType := col.Tp

dm/syncer/ddl_test.go

@@ -18,12 +18,14 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"testing"
 
 	"github.com/DATA-DOG/go-sqlmock"
 	. "github.com/pingcap/check"
 	"github.com/pingcap/tidb/parser"
 	"github.com/pingcap/tidb/parser/ast"
 	"github.com/pingcap/tidb/util/filter"
+	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 
 	regexprrouter "github.com/pingcap/tidb/util/regexpr-router"
@@ -696,12 +698,13 @@ func (s *testDDLSuite) TestAdjustDatabaseCollation(c *C) {
 	}
 }
 
-func (s *testDDLSuite) TestAdjustCollation(c *C) {
+func TestAdjustCollation(t *testing.T) {
 	sqls := []string{
 		"create table `test`.`t1` (id int) CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci",
 		"create table `test`.`t1` (id int) CHARSET=utf8mb4",
 		"create table `test`.`t1` (id int) COLLATE=utf8mb4_general_ci",
 		"create table `test`.`t1` (id int)",
+		"create table `test`.`t1` (name varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)",
 		"create table `test`.`t1` (id int, name varchar(20) CHARACTER SET utf8mb4, work varchar(20))",
 		"create table `test`.`t1` (id int, name varchar(20), work varchar(20))",
 		"create table `test`.`t1` (id int, name varchar(20) COLLATE utf8mb4_general_ci, work varchar(20))",
@@ -723,6 +726,7 @@ func (s *testDDLSuite) TestAdjustCollation(c *C) {
 		"CREATE TABLE `test`.`t` (`id` INT) DEFAULT CHARACTER SET = UTF8MB4 DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
 		"CREATE TABLE `test`.`t` (`id` INT) DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
 		"CREATE TABLE `test`.`t` (`id` INT)",
+		"CREATE TABLE `test`.`t` (`name` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_bin)",
 		"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_general_ci,`work` VARCHAR(20))",
 		"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20),`work` VARCHAR(20))",
 		"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) COLLATE utf8mb4_general_ci,`work` VARCHAR(20))",
@@ -758,13 +762,13 @@ func (s *testDDLSuite) TestAdjustCollation(c *C) {
 			targetTables: []*filter.Table{tab},
 		}
 		stmt, err := p.ParseOneStmt(sql, "", "")
-		c.Assert(err, IsNil)
-		c.Assert(stmt, NotNil)
+		require.NoError(t, err)
+		require.NotNil(t, stmt)
 		ddlInfo.originStmt = stmt
 		adjustCollation(tctx, ddlInfo, statusVars, charsetAndDefaultCollationMap, idAndCollationMap)
 		routedDDL, err := parserpkg.RenameDDLTable(ddlInfo.originStmt, ddlInfo.targetTables)
-		c.Assert(err, IsNil)
-		c.Assert(routedDDL, Equals, expectedSQLs[i])
+		require.NoError(t, err)
+		require.Equal(t, expectedSQLs[i], routedDDL)
 	}
 }
 

dm/syncer/util.go

@@ -150,14 +150,14 @@ func subtaskCfg2BinlogSyncerCfg(cfg *config.SubTaskConfig, timezone *time.Locati
 		if loadErr := cfg.From.Security.LoadTLSContent(); loadErr != nil {
 			return replication.BinlogSyncerConfig{}, terror.ErrCtlLoadTLSCfg.Delegate(loadErr)
 		}
-		tlsConfig, err = util.ToTLSConfigWithVerifyByRawbytes(cfg.From.Security.SSLCABytes,
-			cfg.From.Security.SSLCertBytes, cfg.From.Security.SSLKEYBytes, cfg.From.Security.CertAllowedCN)
+		tlsConfig, err = util.NewTLSConfig(
+			util.WithCAContent(cfg.From.Security.SSLCABytes),
+			util.WithCertAndKeyContent(cfg.From.Security.SSLCertBytes, cfg.From.Security.SSLKEYBytes),
+			util.WithVerifyCommonName(cfg.From.Security.CertAllowedCN),
+		)
 		if err != nil {
 			return replication.BinlogSyncerConfig{}, terror.ErrConnInvalidTLSConfig.Delegate(err)
 		}
-		if tlsConfig != nil {
-			tlsConfig.InsecureSkipVerify = true
-		}
 	}
 
 	syncCfg := replication.BinlogSyncerConfig{

go.mod

@@ -50,9 +50,9 @@ require (
 	github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3
 	github.com/pingcap/kvproto v0.0.0-20220517085838-12e2f5a9d167
 	github.com/pingcap/log v1.1.0
-	github.com/pingcap/tidb v1.1.0-beta.0.20220727102111-099c736ff30a
+	github.com/pingcap/tidb v1.1.0-beta.0.20221012073351-67fb27107f2a
 	github.com/pingcap/tidb-tools v6.0.1-0.20220516050036-b3ea358e374a+incompatible
-	github.com/pingcap/tidb/parser v0.0.0-20220727102111-099c736ff30a
+	github.com/pingcap/tidb/parser v0.0.0-20221012073351-67fb27107f2a
 	github.com/prometheus/client_golang v1.12.2
 	github.com/prometheus/client_model v0.2.0
 	github.com/r3labs/diff v1.1.0
@@ -66,7 +66,7 @@ require (
 	github.com/swaggo/gin-swagger v1.2.0
 	github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476
 	github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954
-	github.com/tikv/client-go/v2 v2.0.1-0.20220531092439-efebaeb9fe53
+	github.com/tikv/client-go/v2 v2.0.1-0.20220930044212-ee63982f7a77
 	github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800
 	github.com/tikv/pd/client v0.0.0-20220307081149-841fa61e9710
 	github.com/tinylib/msgp v1.1.6

go.sum

@@ -965,15 +965,15 @@ github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d/go.mod h1:7j18ezaW
 github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 h1:HYbcxtnkN3s5tqrZ/z3eJS4j3Db8wMphEm1q10lY/TM=
 github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4/go.mod h1:sDCsM39cGiv2vwunZkaFA917vVkqDTGSPbbV7z4Oops=
 github.com/pingcap/tidb v1.1.0-beta.0.20220511160835-98c31070d958/go.mod h1:luW4sIZoLHY3bCWuKqyqk2QgMvF+/M7nWOXf/me0+fY=
-github.com/pingcap/tidb v1.1.0-beta.0.20220727102111-099c736ff30a h1:pxJOc27XqLtVC4+AzhcriTlMTO+J+EQRHf7KfMbe0+8=
-github.com/pingcap/tidb v1.1.0-beta.0.20220727102111-099c736ff30a/go.mod h1:z8u+NvIbYDO2f/Y1sJR0exTeCYNwFb/755Z4kZAlBM8=
+github.com/pingcap/tidb v1.1.0-beta.0.20221012073351-67fb27107f2a h1:61Jf7Dtn7A79D0Ebul8jF2eF/vnnVFlLc0pNER7Ziqo=
+github.com/pingcap/tidb v1.1.0-beta.0.20221012073351-67fb27107f2a/go.mod h1:9zcObRa6uTxohJ7J+V6/Quja9IXJEyIB0SDXfhJso2I=
 github.com/pingcap/tidb-dashboard v0.0.0-20220117082709-e8076b5c79ba/go.mod h1:4hk/3owVGWdvI9Kx6yCqqvM1T5PVgwyQNyMQxD3rwfc=
 github.com/pingcap/tidb-tools v6.0.1-0.20220516050036-b3ea358e374a+incompatible h1:dAfZKHR4Wu3bdFDNDsH2c4fWrkqbkgVlwY5pC4Nmp6Q=
 github.com/pingcap/tidb-tools v6.0.1-0.20220516050036-b3ea358e374a+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e/go.mod h1:e1MGCA9Sg3T8jid8PKAEq5eYVuMMCq4n8gJ+Kqp4Plg=
 github.com/pingcap/tidb/parser v0.0.0-20220511160835-98c31070d958/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI=
-github.com/pingcap/tidb/parser v0.0.0-20220727102111-099c736ff30a h1:7QQtNzfZV+WrqOlSBEB3nW4ut+n0xXJx3GETevEDf4Y=
-github.com/pingcap/tidb/parser v0.0.0-20220727102111-099c736ff30a/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI=
+github.com/pingcap/tidb/parser v0.0.0-20221012073351-67fb27107f2a h1:+m/kHFPGiil9YLnCHTlSUlhrk1MyPv7w6V8M+QooUHw=
+github.com/pingcap/tidb/parser v0.0.0-20221012073351-67fb27107f2a/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI=
 github.com/pingcap/tipb v0.0.0-20220215045658-d12dec7a7609/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
 github.com/pingcap/tipb v0.0.0-20220314125451-bfb5c2c55188 h1:+46isFI9fR9R+nJVDMI55tCC/TCwp+bvVA4HLGEv1rY=
 github.com/pingcap/tipb v0.0.0-20220314125451-bfb5c2c55188/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
@@ -1143,8 +1143,8 @@ github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJH
 github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tikv/client-go/v2 v2.0.1-0.20220510032238-ff5e35ac2869/go.mod h1:0scaG+seu7L56apm+Gjz9vckyO7ABIzM6T7n00mrIXs=
-github.com/tikv/client-go/v2 v2.0.1-0.20220531092439-efebaeb9fe53 h1:zalDvjC3IhixTcqU1HQYJQtTI3npketLDprwLw1eGqI=
-github.com/tikv/client-go/v2 v2.0.1-0.20220531092439-efebaeb9fe53/go.mod h1:VTlli8fRRpcpISj9I2IqroQmcAFfaTyBquiRhofOcDs=
+github.com/tikv/client-go/v2 v2.0.1-0.20220930044212-ee63982f7a77 h1:T54ypK32yzFCVAbIutiXmBuW2qUYzYRySHIPquO8kXI=
+github.com/tikv/client-go/v2 v2.0.1-0.20220930044212-ee63982f7a77/go.mod h1:VTlli8fRRpcpISj9I2IqroQmcAFfaTyBquiRhofOcDs=
 github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 h1:lIfIwqe1HPa0suhMpiI200nYxau+rXWXTqZxSGg1HS4=
 github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800/go.mod h1:J/dj1zpEE9b7idgONGFttnXM+ncl88LmnkD/xDbq0hA=
 github.com/tikv/pd/client v0.0.0-20220307081149-841fa61e9710 h1:jxgmKOscXSjaFEKQGRyY5qOpK8hLqxs2irb/uDJMtwk=