diff --git a/dm/dm/master/server_test.go b/dm/dm/master/server_test.go index 1933edc862d..0f60c208a42 100644 --- a/dm/dm/master/server_test.go +++ b/dm/dm/master/server_test.go @@ -1630,9 +1630,12 @@ func (t *testMaster) testNormalServerLifecycle(c *check.C, cfg *Config, checkLog func (t *testMaster) testHTTPInterface(c *check.C, url string, contain []byte) { // we use HTTPS in some test cases. - tls, err := toolutils.NewTLS(pwd+"/tls_for_test/ca.pem", pwd+"/tls_for_test/dm.pem", pwd+"/tls_for_test/dm.key", url, []string{}) + tlsConfig, err := toolutils.NewTLSConfig( + toolutils.WithCAPath(pwd+"/tls_for_test/ca.pem"), + toolutils.WithCertAndKeyPath(pwd+"/tls_for_test/dm.pem", pwd+"/tls_for_test/dm.key"), + ) c.Assert(err, check.IsNil) - cli := toolutils.ClientWithTLS(tls.TLSConfig()) + cli := toolutils.ClientWithTLS(tlsConfig) // nolint:noctx resp, err := cli.Get(url) diff --git a/dm/dumpling/dumpling.go b/dm/dumpling/dumpling.go index f12e981de5f..6528931864e 100644 --- a/dm/dumpling/dumpling.go +++ b/dm/dumpling/dumpling.go @@ -299,7 +299,7 @@ func (m *Dumpling) constructArgs(ctx context.Context) (*export.Config, error) { dumpConfig.Security.SSLCABytes = db.Security.SSLCABytes dumpConfig.Security.SSLCertBytes = db.Security.SSLCertBytes - dumpConfig.Security.SSLKEYBytes = db.Security.SSLKEYBytes + dumpConfig.Security.SSLKeyBytes = db.Security.SSLKEYBytes } // `true` means dumpling will release lock after working connection established diff --git a/dm/pkg/conn/basedb.go b/dm/pkg/conn/basedb.go index 4fbf6408b70..3f6607746c8 100644 --- a/dm/pkg/conn/basedb.go +++ b/dm/pkg/conn/basedb.go @@ -70,16 +70,14 @@ func (d *DefaultDBProviderImpl) Apply(config *config.DBConfig) (*BaseDB, error) if loadErr := config.Security.LoadTLSContent(); loadErr != nil { return nil, terror.ErrCtlLoadTLSCfg.Delegate(loadErr) } - tlsConfig, err := util.ToTLSConfigWithVerifyByRawbytes(config.Security.SSLCABytes, - config.Security.SSLCertBytes, config.Security.SSLKEYBytes, config.Security.CertAllowedCN) + tlsConfig, err := util.NewTLSConfig( + util.WithCAContent(config.Security.SSLCABytes), + util.WithCertAndKeyContent(config.Security.SSLCertBytes, config.Security.SSLKEYBytes), + util.WithVerifyCommonName(config.Security.CertAllowedCN), + ) if err != nil { return nil, terror.ErrConnInvalidTLSConfig.Delegate(err) } - // NOTE for local test(use a self-signed or invalid certificate), we don't need to check CA file. - // see more here https://github.com/go-sql-driver/mysql#tls - if config.Host == "127.0.0.1" || len(config.Security.SSLCertBytes) == 0 || len(config.Security.SSLKEYBytes) == 0 { - tlsConfig.InsecureSkipVerify = true - } name := "dm" + strconv.FormatInt(atomic.AddInt64(&customID, 1), 10) err = mysql.RegisterTLSConfig(name, tlsConfig) diff --git a/dm/relay/relay.go b/dm/relay/relay.go index 197b0053e9a..027eae5686e 100644 --- a/dm/relay/relay.go +++ b/dm/relay/relay.go @@ -427,8 +427,10 @@ type recoverResult struct { // doRecovering tries to recover the current binlog file. // 1. read events from the file // 2. -// a. update the position with the event's position if the transaction finished -// b. update the GTID set with the event's GTID if the transaction finished +// +// a. update the position with the event's position if the transaction finished +// b. update the GTID set with the event's GTID if the transaction finished +// // 3. truncate any incomplete events/transactions // now, we think a transaction finished if we received a XIDEvent or DDL in QueryEvent // NOTE: handle cases when file size > 4GB. @@ -544,10 +546,10 @@ func (r *Relay) preprocessEvent(e *replication.BinlogEvent, parser2 *parser.Pars } // handleEvents handles binlog events, including: -// 1. read events from upstream -// 2. transform events -// 3. write events into relay log files -// 4. update metadata if needed. +// 1. read events from upstream +// 2. transform events +// 3. write events into relay log files +// 4. update metadata if needed. func (r *Relay) handleEvents( ctx context.Context, reader2 Reader, @@ -1129,14 +1131,14 @@ func (r *Relay) setSyncConfig() error { if loadErr := r.cfg.From.Security.LoadTLSContent(); loadErr != nil { return terror.ErrCtlLoadTLSCfg.Delegate(loadErr) } - tlsConfig, err = util.ToTLSConfigWithVerifyByRawbytes(r.cfg.From.Security.SSLCABytes, - r.cfg.From.Security.SSLCertBytes, r.cfg.From.Security.SSLKEYBytes, r.cfg.From.Security.CertAllowedCN) + tlsConfig, err = util.NewTLSConfig( + util.WithCAContent(r.cfg.From.Security.SSLCABytes), + util.WithCertAndKeyContent(r.cfg.From.Security.SSLCertBytes, r.cfg.From.Security.SSLKEYBytes), + util.WithVerifyCommonName(r.cfg.From.Security.CertAllowedCN), + ) if err != nil { return terror.ErrConnInvalidTLSConfig.Delegate(err) } - if tlsConfig != nil { - tlsConfig.InsecureSkipVerify = true - } } syncerCfg := replication.BinlogSyncerConfig{ diff --git a/dm/syncer/ddl.go b/dm/syncer/ddl.go index 9dd9169a71b..f5d4df620f0 100644 --- a/dm/syncer/ddl.go +++ b/dm/syncer/ddl.go @@ -273,11 +273,12 @@ func adjustCollation(tctx *tcontext.Context, ddlInfo *ddlInfo, statusVars []byte // adjustColumnsCollation adds column's collation. func adjustColumnsCollation(tctx *tcontext.Context, createStmt *ast.CreateTableStmt, charsetAndDefaultCollationMap map[string]string) { +ColumnLoop: for _, col := range createStmt.Cols { for _, options := range col.Options { // already have 'Collation' if options.Tp == ast.ColumnOptionCollate { - continue + continue ColumnLoop } } fieldType := col.Tp diff --git a/dm/syncer/ddl_test.go b/dm/syncer/ddl_test.go index 09015f7097d..9aa078c2582 100644 --- a/dm/syncer/ddl_test.go +++ b/dm/syncer/ddl_test.go @@ -18,12 +18,14 @@ import ( "errors" "fmt" "strings" + "testing" "github.com/DATA-DOG/go-sqlmock" . "github.com/pingcap/check" "github.com/pingcap/tidb/parser" "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/util/filter" + "github.com/stretchr/testify/require" "go.uber.org/zap" regexprrouter "github.com/pingcap/tidb/util/regexpr-router" @@ -696,12 +698,13 @@ func (s *testDDLSuite) TestAdjustDatabaseCollation(c *C) { } } -func (s *testDDLSuite) TestAdjustCollation(c *C) { +func TestAdjustCollation(t *testing.T) { sqls := []string{ "create table `test`.`t1` (id int) CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci", "create table `test`.`t1` (id int) CHARSET=utf8mb4", "create table `test`.`t1` (id int) COLLATE=utf8mb4_general_ci", "create table `test`.`t1` (id int)", + "create table `test`.`t1` (name varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)", "create table `test`.`t1` (id int, name varchar(20) CHARACTER SET utf8mb4, work varchar(20))", "create table `test`.`t1` (id int, name varchar(20), work varchar(20))", "create table `test`.`t1` (id int, name varchar(20) COLLATE utf8mb4_general_ci, work varchar(20))", @@ -723,6 +726,7 @@ func (s *testDDLSuite) TestAdjustCollation(c *C) { "CREATE TABLE `test`.`t` (`id` INT) DEFAULT CHARACTER SET = UTF8MB4 DEFAULT COLLATE = UTF8MB4_GENERAL_CI", "CREATE TABLE `test`.`t` (`id` INT) DEFAULT COLLATE = UTF8MB4_GENERAL_CI", "CREATE TABLE `test`.`t` (`id` INT)", + "CREATE TABLE `test`.`t` (`name` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_bin)", "CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_general_ci,`work` VARCHAR(20))", "CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20),`work` VARCHAR(20))", "CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) COLLATE utf8mb4_general_ci,`work` VARCHAR(20))", @@ -758,13 +762,13 @@ func (s *testDDLSuite) TestAdjustCollation(c *C) { targetTables: []*filter.Table{tab}, } stmt, err := p.ParseOneStmt(sql, "", "") - c.Assert(err, IsNil) - c.Assert(stmt, NotNil) + require.NoError(t, err) + require.NotNil(t, stmt) ddlInfo.originStmt = stmt adjustCollation(tctx, ddlInfo, statusVars, charsetAndDefaultCollationMap, idAndCollationMap) routedDDL, err := parserpkg.RenameDDLTable(ddlInfo.originStmt, ddlInfo.targetTables) - c.Assert(err, IsNil) - c.Assert(routedDDL, Equals, expectedSQLs[i]) + require.NoError(t, err) + require.Equal(t, expectedSQLs[i], routedDDL) } } diff --git a/dm/syncer/util.go b/dm/syncer/util.go index f8c54d1a4b6..c1176684267 100644 --- a/dm/syncer/util.go +++ b/dm/syncer/util.go @@ -150,14 +150,14 @@ func subtaskCfg2BinlogSyncerCfg(cfg *config.SubTaskConfig, timezone *time.Locati if loadErr := cfg.From.Security.LoadTLSContent(); loadErr != nil { return replication.BinlogSyncerConfig{}, terror.ErrCtlLoadTLSCfg.Delegate(loadErr) } - tlsConfig, err = util.ToTLSConfigWithVerifyByRawbytes(cfg.From.Security.SSLCABytes, - cfg.From.Security.SSLCertBytes, cfg.From.Security.SSLKEYBytes, cfg.From.Security.CertAllowedCN) + tlsConfig, err = util.NewTLSConfig( + util.WithCAContent(cfg.From.Security.SSLCABytes), + util.WithCertAndKeyContent(cfg.From.Security.SSLCertBytes, cfg.From.Security.SSLKEYBytes), + util.WithVerifyCommonName(cfg.From.Security.CertAllowedCN), + ) if err != nil { return replication.BinlogSyncerConfig{}, terror.ErrConnInvalidTLSConfig.Delegate(err) } - if tlsConfig != nil { - tlsConfig.InsecureSkipVerify = true - } } syncCfg := replication.BinlogSyncerConfig{ diff --git a/go.mod b/go.mod index a63437fef67..29a7b083f2f 100644 --- a/go.mod +++ b/go.mod @@ -50,9 +50,9 @@ require ( github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 github.com/pingcap/kvproto v0.0.0-20220517085838-12e2f5a9d167 github.com/pingcap/log v1.1.0 - github.com/pingcap/tidb v1.1.0-beta.0.20220727102111-099c736ff30a + github.com/pingcap/tidb v1.1.0-beta.0.20221012073351-67fb27107f2a github.com/pingcap/tidb-tools v6.0.1-0.20220516050036-b3ea358e374a+incompatible - github.com/pingcap/tidb/parser v0.0.0-20220727102111-099c736ff30a + github.com/pingcap/tidb/parser v0.0.0-20221012073351-67fb27107f2a github.com/prometheus/client_golang v1.12.2 github.com/prometheus/client_model v0.2.0 github.com/r3labs/diff v1.1.0 @@ -66,7 +66,7 @@ require ( github.com/swaggo/gin-swagger v1.2.0 github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476 github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 - github.com/tikv/client-go/v2 v2.0.1-0.20220531092439-efebaeb9fe53 + github.com/tikv/client-go/v2 v2.0.1-0.20220930044212-ee63982f7a77 github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 github.com/tikv/pd/client v0.0.0-20220307081149-841fa61e9710 github.com/tinylib/msgp v1.1.6 diff --git a/go.sum b/go.sum index 4f9a5cff69a..e810875b951 100644 --- a/go.sum +++ b/go.sum @@ -965,15 +965,15 @@ github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d/go.mod h1:7j18ezaW github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 h1:HYbcxtnkN3s5tqrZ/z3eJS4j3Db8wMphEm1q10lY/TM= github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4/go.mod h1:sDCsM39cGiv2vwunZkaFA917vVkqDTGSPbbV7z4Oops= github.com/pingcap/tidb v1.1.0-beta.0.20220511160835-98c31070d958/go.mod h1:luW4sIZoLHY3bCWuKqyqk2QgMvF+/M7nWOXf/me0+fY= -github.com/pingcap/tidb v1.1.0-beta.0.20220727102111-099c736ff30a h1:pxJOc27XqLtVC4+AzhcriTlMTO+J+EQRHf7KfMbe0+8= -github.com/pingcap/tidb v1.1.0-beta.0.20220727102111-099c736ff30a/go.mod h1:z8u+NvIbYDO2f/Y1sJR0exTeCYNwFb/755Z4kZAlBM8= +github.com/pingcap/tidb v1.1.0-beta.0.20221012073351-67fb27107f2a h1:61Jf7Dtn7A79D0Ebul8jF2eF/vnnVFlLc0pNER7Ziqo= +github.com/pingcap/tidb v1.1.0-beta.0.20221012073351-67fb27107f2a/go.mod h1:9zcObRa6uTxohJ7J+V6/Quja9IXJEyIB0SDXfhJso2I= github.com/pingcap/tidb-dashboard v0.0.0-20220117082709-e8076b5c79ba/go.mod h1:4hk/3owVGWdvI9Kx6yCqqvM1T5PVgwyQNyMQxD3rwfc= github.com/pingcap/tidb-tools v6.0.1-0.20220516050036-b3ea358e374a+incompatible h1:dAfZKHR4Wu3bdFDNDsH2c4fWrkqbkgVlwY5pC4Nmp6Q= github.com/pingcap/tidb-tools v6.0.1-0.20220516050036-b3ea358e374a+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e/go.mod h1:e1MGCA9Sg3T8jid8PKAEq5eYVuMMCq4n8gJ+Kqp4Plg= github.com/pingcap/tidb/parser v0.0.0-20220511160835-98c31070d958/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI= -github.com/pingcap/tidb/parser v0.0.0-20220727102111-099c736ff30a h1:7QQtNzfZV+WrqOlSBEB3nW4ut+n0xXJx3GETevEDf4Y= -github.com/pingcap/tidb/parser v0.0.0-20220727102111-099c736ff30a/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI= +github.com/pingcap/tidb/parser v0.0.0-20221012073351-67fb27107f2a h1:+m/kHFPGiil9YLnCHTlSUlhrk1MyPv7w6V8M+QooUHw= +github.com/pingcap/tidb/parser v0.0.0-20221012073351-67fb27107f2a/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI= github.com/pingcap/tipb v0.0.0-20220215045658-d12dec7a7609/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pingcap/tipb v0.0.0-20220314125451-bfb5c2c55188 h1:+46isFI9fR9R+nJVDMI55tCC/TCwp+bvVA4HLGEv1rY= github.com/pingcap/tipb v0.0.0-20220314125451-bfb5c2c55188/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= @@ -1143,8 +1143,8 @@ github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJH github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tikv/client-go/v2 v2.0.1-0.20220510032238-ff5e35ac2869/go.mod h1:0scaG+seu7L56apm+Gjz9vckyO7ABIzM6T7n00mrIXs= -github.com/tikv/client-go/v2 v2.0.1-0.20220531092439-efebaeb9fe53 h1:zalDvjC3IhixTcqU1HQYJQtTI3npketLDprwLw1eGqI= -github.com/tikv/client-go/v2 v2.0.1-0.20220531092439-efebaeb9fe53/go.mod h1:VTlli8fRRpcpISj9I2IqroQmcAFfaTyBquiRhofOcDs= +github.com/tikv/client-go/v2 v2.0.1-0.20220930044212-ee63982f7a77 h1:T54ypK32yzFCVAbIutiXmBuW2qUYzYRySHIPquO8kXI= +github.com/tikv/client-go/v2 v2.0.1-0.20220930044212-ee63982f7a77/go.mod h1:VTlli8fRRpcpISj9I2IqroQmcAFfaTyBquiRhofOcDs= github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 h1:lIfIwqe1HPa0suhMpiI200nYxau+rXWXTqZxSGg1HS4= github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800/go.mod h1:J/dj1zpEE9b7idgONGFttnXM+ncl88LmnkD/xDbq0hA= github.com/tikv/pd/client v0.0.0-20220307081149-841fa61e9710 h1:jxgmKOscXSjaFEKQGRyY5qOpK8hLqxs2irb/uDJMtwk=