From 7cc688c036782ae16238123ad2b2fa17e07bb4d2 Mon Sep 17 00:00:00 2001 From: bb7133 Date: Mon, 29 Apr 2019 19:13:39 +0800 Subject: [PATCH] infoschema, domain, ddl: fix upper cased charset names (#10272) --- ddl/db_integration_test.go | 45 ++++++++++++++++++++++++++++++++++++++ ddl/db_test.go | 2 +- domain/domain.go | 1 + go.mod | 2 +- go.sum | 5 ++--- infoschema/builder.go | 16 ++++++++++++++ 6 files changed, 66 insertions(+), 5 deletions(-) diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go index 81e35dc563745..5ee6b31999c74 100644 --- a/ddl/db_integration_test.go +++ b/ddl/db_integration_test.go @@ -751,6 +751,51 @@ func (s *testIntegrationSuite7) TestCaseInsensitiveCharsetAndCollate(c *C) { tk.MustExec("create table t2(id int) ENGINE=InnoDB DEFAULT CHARSET=Utf8 COLLATE=utf8_BIN;") tk.MustExec("create table t3(id int) ENGINE=InnoDB DEFAULT CHARSET=Utf8mb4 COLLATE=utf8MB4_BIN;") tk.MustExec("create table t4(id int) ENGINE=InnoDB DEFAULT CHARSET=Utf8mb4 COLLATE=utf8MB4_general_ci;") + + tk.MustExec("create table t5(a varchar(20)) ENGINE=InnoDB DEFAULT CHARSET=UTF8MB4 COLLATE=UTF8MB4_GENERAL_CI;") + tk.MustExec("insert into t5 values ('特克斯和凯科斯群岛')") + + db, ok := domain.GetDomain(s.ctx).InfoSchema().SchemaByName(model.NewCIStr("test_charset_collate")) + c.Assert(ok, IsTrue) + tbl := testGetTableByName(c, s.ctx, "test_charset_collate", "t5") + tblInfo := tbl.Meta().Clone() + c.Assert(tblInfo.Charset, Equals, "utf8mb4") + c.Assert(tblInfo.Columns[0].Charset, Equals, "utf8mb4") + + tblInfo.Version = model.TableInfoVersion2 + tblInfo.Charset = "UTF8MB4" + + updateTableInfo := func(tblInfo *model.TableInfo) { + mockCtx := mock.NewContext() + mockCtx.Store = s.store + err := mockCtx.NewTxn(context.Background()) + c.Assert(err, IsNil) + txn, err := mockCtx.Txn(true) + c.Assert(err, IsNil) + mt := meta.NewMeta(txn) + c.Assert(ok, IsTrue) + err = mt.UpdateTable(db.ID, tblInfo) + c.Assert(err, IsNil) + err = txn.Commit(context.Background()) + c.Assert(err, IsNil) + } + updateTableInfo(tblInfo) + tk.MustExec("alter table t5 add column b varchar(10);") // load latest schema. + + tblInfo = testGetTableByName(c, s.ctx, "test_charset_collate", "t5").Meta() + c.Assert(tblInfo.Charset, Equals, "utf8mb4") + c.Assert(tblInfo.Columns[0].Charset, Equals, "utf8mb4") + + // For model.TableInfoVersion3, it is believed that all charsets / collations are lower-cased, do not do case-convert + tblInfo = tblInfo.Clone() + tblInfo.Version = model.TableInfoVersion3 + tblInfo.Charset = "UTF8MB4" + updateTableInfo(tblInfo) + tk.MustExec("alter table t5 add column c varchar(10);") // load latest schema. + + tblInfo = testGetTableByName(c, s.ctx, "test_charset_collate", "t5").Meta() + c.Assert(tblInfo.Charset, Equals, "UTF8MB4") + c.Assert(tblInfo.Columns[0].Charset, Equals, "utf8mb4") } func (s *testIntegrationSuite3) TestZeroFillCreateTable(c *C) { diff --git a/ddl/db_test.go b/ddl/db_test.go index c5867b6648576..5c10247ba7f76 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -2292,7 +2292,7 @@ func (s *testDBSuite4) TestCheckTooBigFieldLength(c *C) { assertErrorCode(c, s.tk, "alter table tr_04 add column b varchar(20000) charset utf8mb4;", tmysql.ErrTooBigFieldlength) assertErrorCode(c, s.tk, "alter table tr_04 convert to character set utf8mb4;", tmysql.ErrTooBigFieldlength) assertErrorCode(c, s.tk, "create table tr (id int, name varchar(30000), purchased date ) default charset=utf8 collate=utf8_bin;", tmysql.ErrTooBigFieldlength) - assertErrorCode(c, s.tk, "create table tr (id int, name varchar(20000) charset utf8mb4, purchased date ) default charset=utf8 collate=utf8;", tmysql.ErrTooBigFieldlength) + assertErrorCode(c, s.tk, "create table tr (id int, name varchar(20000) charset utf8mb4, purchased date ) default charset=utf8 collate=utf8_bin;", tmysql.ErrTooBigFieldlength) assertErrorCode(c, s.tk, "create table tr (id int, name varchar(65536), purchased date ) default charset=latin1;", tmysql.ErrTooBigFieldlength) s.tk.MustExec("drop table if exists tr_05;") diff --git a/domain/domain.go b/domain/domain.go index 008348cee7eff..814452ea9870a 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -194,6 +194,7 @@ func (do *Domain) fetchSchemasWithTables(schemas []*model.DBInfo, m *meta.Meta, // schema is not public, can't be used outside. continue } + infoschema.ConvertCharsetCollateToLowerCaseIfNeed(tbl) di.Tables = append(di.Tables, tbl) } } diff --git a/go.mod b/go.mod index 7be334aa05bc4..19a9d81e6ca3c 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190427000002-f3ecae036b23 + github.com/pingcap/parser v0.0.0-20190429074050-34bcb57d11a6 github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669 github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index d0b3f9a8a1c0b..7cddadde9083e 100644 --- a/go.sum +++ b/go.sum @@ -160,8 +160,8 @@ github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190427000002-f3ecae036b23 h1:9RUvFHNfDplW4KZ90FMOC7QEDLORa15AlozhX0P8cCg= -github.com/pingcap/parser v0.0.0-20190427000002-f3ecae036b23/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190429074050-34bcb57d11a6 h1:m35H8yhrMcATKp8yUVUHAmvuzIT8xjL+ialXyBrBWHU= +github.com/pingcap/parser v0.0.0-20190429074050-34bcb57d11a6/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669 h1:ZoKjndm/Ig7Ru/wojrQkc/YLUttUdQXoH77gtuWCvL4= github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669/go.mod h1:MUCxRzOkYiWZtlyi4MhxjCIj9PgQQ/j+BLNGm7aUsnM= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= @@ -292,7 +292,6 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/infoschema/builder.go b/infoschema/builder.go index 65eba8279c1b0..5b0446648f46f 100644 --- a/infoschema/builder.go +++ b/infoschema/builder.go @@ -16,6 +16,7 @@ package infoschema import ( "fmt" "sort" + "strings" "github.com/pingcap/errors" "github.com/pingcap/parser/charset" @@ -172,6 +173,7 @@ func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID i fmt.Sprintf("(Table ID %d)", tableID), ) } + ConvertCharsetCollateToLowerCaseIfNeed(tblInfo) ConvertOldVersionUTF8ToUTF8MB4IfNeed(tblInfo) if alloc == nil { @@ -197,6 +199,20 @@ func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID i return nil } +// ConvertCharsetCollateToLowerCaseIfNeed convert the charset / collation of table and its columns to lower case, +// if the table's version is prior to TableInfoVersion3. +func ConvertCharsetCollateToLowerCaseIfNeed(tbInfo *model.TableInfo) { + if tbInfo.Version >= model.TableInfoVersion3 { + return + } + tbInfo.Charset = strings.ToLower(tbInfo.Charset) + tbInfo.Collate = strings.ToLower(tbInfo.Collate) + for _, col := range tbInfo.Columns { + col.Charset = strings.ToLower(col.Charset) + col.Collate = strings.ToLower(col.Collate) + } +} + // ConvertOldVersionUTF8ToUTF8MB4IfNeed convert old version UTF8 to UTF8MB4 if config.TreatOldVersionUTF8AsUTF8MB4 is enable. func ConvertOldVersionUTF8ToUTF8MB4IfNeed(tbInfo *model.TableInfo) { if tbInfo.Version >= model.TableInfoVersion2 || !config.GetGlobalConfig().TreatOldVersionUTF8AsUTF8MB4 {