From df07f0de07cc830e7ffb3741fad3b6b90319aa47 Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Mon, 20 Nov 2023 20:21:33 +0800 Subject: [PATCH 1/9] done Signed-off-by: wjhuang2016 --- .../clustertablestest/cluster_tables_test.go | 6 ++++ pkg/session/bootstrap.go | 29 +++++++++++++------ pkg/sessionctx/variable/session.go | 1 + 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/pkg/infoschema/test/clustertablestest/cluster_tables_test.go b/pkg/infoschema/test/clustertablestest/cluster_tables_test.go index c741b42f7eb29..0f8ca8cf6b4fa 100644 --- a/pkg/infoschema/test/clustertablestest/cluster_tables_test.go +++ b/pkg/infoschema/test/clustertablestest/cluster_tables_test.go @@ -41,6 +41,7 @@ import ( "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/auth" "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/privilege/privileges" "github.com/pingcap/tidb/pkg/server" "github.com/pingcap/tidb/pkg/store/helper" "github.com/pingcap/tidb/pkg/store/mockstore" @@ -842,6 +843,11 @@ func TestMDLView(t *testing.T) { {"add column", "create table t(a int)", "alter table test.t add column b int", []string{"select 1", "select * from t"}, "[\"begin\",\"select ?\",\"select * from `t`\"]"}, {"change column in 1 step", "create table t(a int)", "alter table test.t change column a b int", []string{"select 1", "select * from t"}, "[\"begin\",\"select ?\",\"select * from `t`\"]"}, } + save := privileges.SkipWithGrant + privileges.SkipWithGrant = true + defer func() { + privileges.SkipWithGrant = save + }() for _, c := range testCases { t.Run(c.name, func(t *testing.T) { // setup suite diff --git a/pkg/session/bootstrap.go b/pkg/session/bootstrap.go index bc3b231aa5787..7ace90a650e70 100644 --- a/pkg/session/bootstrap.go +++ b/pkg/session/bootstrap.go @@ -459,19 +459,18 @@ const ( );` // CreateMDLView is a view about metadata locks. CreateMDLView = `CREATE OR REPLACE VIEW mysql.tidb_mdl_view as ( - SELECT job_id, + SELECT ddl_jobs.job_id, db_name, table_name, query, session_id, - txnstart, + cluster_tidb_trx.start_time, tidb_decode_sql_digests(all_sql_digests, 4096) AS SQL_DIGESTS FROM information_schema.ddl_jobs, - information_schema.cluster_tidb_trx, - information_schema.cluster_processlist - WHERE (ddl_jobs.state != 'synced' and ddl_jobs.state != 'cancelled') - AND Find_in_set(ddl_jobs.table_id, cluster_tidb_trx.related_table_ids) - AND cluster_tidb_trx.session_id = cluster_processlist.id + mysql.tidb_mdl_info, + information_schema.cluster_tidb_trx + WHERE ddl_jobs.job_id = tidb_mdl_info.job_id + AND CONCAT(',', tidb_mdl_info.table_ids, ',') REGEXP CONCAT(',', REPLACE(cluster_tidb_trx.related_table_ids, ',', '|'), ',') != 0 );` // CreatePlanReplayerStatusTable is a table about plan replayer status @@ -1022,14 +1021,18 @@ const ( // write mDDLTableVersion into `mysql.tidb` table version178 = 178 - // vresion 179 + // version 179 // enlarge `VARIABLE_VALUE` of `mysql.global_variables` from `varchar(1024)` to `varchar(16383)`. version179 = 179 + + // version 180 + // replace `mysql.tidb_mdl_view` table + version180 = 180 ) // currentBootstrapVersion is defined as a variable, so we can modify its value for testing. // please make sure this is the largest version -var currentBootstrapVersion int64 = version179 +var currentBootstrapVersion int64 = version180 // DDL owner key's expired time is ManagerSessionTTL seconds, we should wait the time and give more time to have a chance to finish it. var internalSQLTimeout = owner.ManagerSessionTTL + 15 @@ -1184,6 +1187,7 @@ var ( upgradeToVer177, upgradeToVer178, upgradeToVer179, + upgradeToVer180, } ) @@ -2892,6 +2896,13 @@ func upgradeToVer179(s Session, ver int64) { doReentrantDDL(s, "ALTER TABLE mysql.global_variables MODIFY COLUMN `VARIABLE_VALUE` varchar(16383)") } +func upgradeToVer180(s Session, ver int64) { + if ver >= version180 { + return + } + doReentrantDDL(s, CreateMDLView) +} + func writeOOMAction(s Session) { comment := "oom-action is `log` by default in v3.0.x, `cancel` by default in v4.0.11+" mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, %?) ON DUPLICATE KEY UPDATE VARIABLE_VALUE= %?`, diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index 23e7df757cbbd..d0cda97208f79 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -2084,6 +2084,7 @@ func NewSessionVars(hctx HookContext) *SessionVars { if EnableRowLevelChecksum.Load() { vars.EnableRowLevelChecksum = true } + vars.systems[MaxAllowedPacket] = strconv.Itoa(int(DefMaxAllowedPacket)) vars.systems[CharacterSetConnection], vars.systems[CollationConnection] = charset.GetDefaultCharsetAndCollate() return vars } From 624c93cd273d82e9c1e991dcf6be6b757dbdd56b Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Mon, 8 Jan 2024 15:53:01 +0800 Subject: [PATCH 2/9] refine Signed-off-by: wjhuang2016 --- pkg/session/bootstrap.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/session/bootstrap.go b/pkg/session/bootstrap.go index 7ace90a650e70..6ca065f4dc715 100644 --- a/pkg/session/bootstrap.go +++ b/pkg/session/bootstrap.go @@ -459,17 +459,17 @@ const ( );` // CreateMDLView is a view about metadata locks. CreateMDLView = `CREATE OR REPLACE VIEW mysql.tidb_mdl_view as ( - SELECT ddl_jobs.job_id, - db_name, - table_name, - query, + SELECT tidb_mdl_info.job_id, + JSON_EXTRACT(cast(cast(job_meta as char) as json), "$.schema_name") as db_name, + JSON_EXTRACT(cast(cast(job_meta as char) as json), "$.table_name") as table_name, + JSON_EXTRACT(cast(cast(job_meta as char) as json), "$.query") as query, session_id, cluster_tidb_trx.start_time, tidb_decode_sql_digests(all_sql_digests, 4096) AS SQL_DIGESTS - FROM information_schema.ddl_jobs, + FROM mysql.tidb_ddl_job, mysql.tidb_mdl_info, information_schema.cluster_tidb_trx - WHERE ddl_jobs.job_id = tidb_mdl_info.job_id + WHERE tidb_ddl_job.job_id=tidb_mdl_info.job_id AND CONCAT(',', tidb_mdl_info.table_ids, ',') REGEXP CONCAT(',', REPLACE(cluster_tidb_trx.related_table_ids, ',', '|'), ',') != 0 );` From 7b2f53046a8ad93082660aa9e242f12e7a3e97a1 Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Mon, 8 Jan 2024 16:11:01 +0800 Subject: [PATCH 3/9] fix bazel Signed-off-by: wjhuang2016 --- pkg/infoschema/test/clustertablestest/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/infoschema/test/clustertablestest/BUILD.bazel b/pkg/infoschema/test/clustertablestest/BUILD.bazel index 90eec484641ad..3a8dd9de576e2 100644 --- a/pkg/infoschema/test/clustertablestest/BUILD.bazel +++ b/pkg/infoschema/test/clustertablestest/BUILD.bazel @@ -25,6 +25,7 @@ go_test( "//pkg/parser/mysql", "//pkg/parser/terror", "//pkg/planner/core", + "//pkg/privilege/privileges", "//pkg/server", "//pkg/session", "//pkg/session/txninfo", From 6caac1cdf4f932d8e2f3f6fa9453f327d9e327f5 Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Mon, 8 Jan 2024 16:23:37 +0800 Subject: [PATCH 4/9] fix Signed-off-by: wjhuang2016 --- pkg/session/bootstrap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/session/bootstrap.go b/pkg/session/bootstrap.go index 956369bfbe161..533b17a0269f9 100644 --- a/pkg/session/bootstrap.go +++ b/pkg/session/bootstrap.go @@ -1067,7 +1067,7 @@ const ( // currentBootstrapVersion is defined as a variable, so we can modify its value for testing. // please make sure this is the largest version -var currentBootstrapVersion int64 = version182 +var currentBootstrapVersion int64 = version183 // DDL owner key's expired time is ManagerSessionTTL seconds, we should wait the time and give more time to have a chance to finish it. var internalSQLTimeout = owner.ManagerSessionTTL + 15 From b649ad637e04a9e5f8418c3325b8af055702ac2d Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Mon, 8 Jan 2024 17:18:26 +0800 Subject: [PATCH 5/9] fix Signed-off-by: wjhuang2016 --- pkg/session/bootstrap.go | 6 +++--- pkg/session/session.go | 4 ++++ pkg/sessionctx/variable/session.go | 1 - 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/session/bootstrap.go b/pkg/session/bootstrap.go index 533b17a0269f9..428d2dd2701a6 100644 --- a/pkg/session/bootstrap.go +++ b/pkg/session/bootstrap.go @@ -461,9 +461,9 @@ const ( // CreateMDLView is a view about metadata locks. CreateMDLView = `CREATE OR REPLACE VIEW mysql.tidb_mdl_view as ( SELECT tidb_mdl_info.job_id, - JSON_EXTRACT(cast(cast(job_meta as char) as json), "$.schema_name") as db_name, - JSON_EXTRACT(cast(cast(job_meta as char) as json), "$.table_name") as table_name, - JSON_EXTRACT(cast(cast(job_meta as char) as json), "$.query") as query, + JSON_UNQUOTE(JSON_EXTRACT(cast(cast(job_meta as char) as json), "$.schema_name")) as db_name, + JSON_UNQUOTE(JSON_EXTRACT(cast(cast(job_meta as char) as json), "$.table_name")) as table_name, + JSON_UNQUOTE(JSON_EXTRACT(cast(cast(job_meta as char) as json), "$.query")) as query, session_id, cluster_tidb_trx.start_time, tidb_decode_sql_digests(all_sql_digests, 4096) AS SQL_DIGESTS diff --git a/pkg/session/session.go b/pkg/session/session.go index e042644a65064..81c2a39f7a846 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -3509,6 +3509,10 @@ func runInBootstrapSession(store kv.Storage, bootstrap func(types.Session)) { // Bootstrap fail will cause program exit. logutil.BgLogger().Fatal("createSession error", zap.Error(err)) } + err = s.sessionVars.SetSystemVar(variable.MaxAllowedPacket, strconv.FormatUint(variable.DefMaxAllowedPacket, 10)) + if err != nil { + logutil.BgLogger().Error("set system variable max_allowed_packet error", zap.Error(err)) + } // For the bootstrap SQLs, the following variables should be compatible with old TiDB versions. s.sessionVars.EnableClusteredIndex = variable.ClusteredIndexDefModeIntOnly diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index 1f04d7102080a..4471d1e706ac6 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -2091,7 +2091,6 @@ func NewSessionVars(hctx HookContext) *SessionVars { if EnableRowLevelChecksum.Load() { vars.EnableRowLevelChecksum = true } - vars.systems[MaxAllowedPacket] = strconv.Itoa(int(DefMaxAllowedPacket)) vars.systems[CharacterSetConnection], vars.systems[CollationConnection] = charset.GetDefaultCharsetAndCollate() return vars } From 4aa07d74b4723929c2b5b50ad91d3faf545c418c Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Mon, 8 Jan 2024 20:26:37 +0800 Subject: [PATCH 6/9] try Signed-off-by: wjhuang2016 --- pkg/sessionctx/variable/session.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index 4471d1e706ac6..c60771eac76ba 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -52,6 +52,7 @@ import ( "github.com/pingcap/tidb/pkg/util/disk" "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/kvcache" + "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mathutil" "github.com/pingcap/tidb/pkg/util/memory" "github.com/pingcap/tidb/pkg/util/replayer" @@ -66,6 +67,7 @@ import ( "github.com/tikv/client-go/v2/tikv" "github.com/twmb/murmur3" atomic2 "go.uber.org/atomic" + "go.uber.org/zap" "golang.org/x/exp/maps" ) @@ -2091,6 +2093,10 @@ func NewSessionVars(hctx HookContext) *SessionVars { if EnableRowLevelChecksum.Load() { vars.EnableRowLevelChecksum = true } + err := vars.SetSystemVar(MaxAllowedPacket, strconv.FormatUint(DefMaxAllowedPacket, 10)) + if err != nil { + logutil.BgLogger().Error("set system variable max_allowed_packet error", zap.Error(err)) + } vars.systems[CharacterSetConnection], vars.systems[CollationConnection] = charset.GetDefaultCharsetAndCollate() return vars } From d94c3690e9c89fb7d88e5e855f47a3dd092668a8 Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Mon, 8 Jan 2024 20:43:45 +0800 Subject: [PATCH 7/9] fix Signed-off-by: wjhuang2016 --- pkg/session/session.go | 9 +++++---- pkg/sessionctx/variable/session.go | 6 ------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/session/session.go b/pkg/session/session.go index 81c2a39f7a846..3e8b11546c996 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -3509,10 +3509,6 @@ func runInBootstrapSession(store kv.Storage, bootstrap func(types.Session)) { // Bootstrap fail will cause program exit. logutil.BgLogger().Fatal("createSession error", zap.Error(err)) } - err = s.sessionVars.SetSystemVar(variable.MaxAllowedPacket, strconv.FormatUint(variable.DefMaxAllowedPacket, 10)) - if err != nil { - logutil.BgLogger().Error("set system variable max_allowed_packet error", zap.Error(err)) - } // For the bootstrap SQLs, the following variables should be compatible with old TiDB versions. s.sessionVars.EnableClusteredIndex = variable.ClusteredIndexDefModeIntOnly @@ -3711,6 +3707,11 @@ func (s *session) loadCommonGlobalVariablesIfNeeded() error { } if s.Value(sessionctx.Initing) != nil { // When running bootstrap or upgrade, we should not access global storage. + // But we need to init max_allowed_packet to use concat function during bootstrap or upgrade. + err := vars.SetSystemVar(variable.MaxAllowedPacket, strconv.FormatUint(variable.DefMaxAllowedPacket, 10)) + if err != nil { + logutil.BgLogger().Error("set system variable max_allowed_packet error", zap.Error(err)) + } return nil } diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index c60771eac76ba..4471d1e706ac6 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -52,7 +52,6 @@ import ( "github.com/pingcap/tidb/pkg/util/disk" "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/kvcache" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mathutil" "github.com/pingcap/tidb/pkg/util/memory" "github.com/pingcap/tidb/pkg/util/replayer" @@ -67,7 +66,6 @@ import ( "github.com/tikv/client-go/v2/tikv" "github.com/twmb/murmur3" atomic2 "go.uber.org/atomic" - "go.uber.org/zap" "golang.org/x/exp/maps" ) @@ -2093,10 +2091,6 @@ func NewSessionVars(hctx HookContext) *SessionVars { if EnableRowLevelChecksum.Load() { vars.EnableRowLevelChecksum = true } - err := vars.SetSystemVar(MaxAllowedPacket, strconv.FormatUint(DefMaxAllowedPacket, 10)) - if err != nil { - logutil.BgLogger().Error("set system variable max_allowed_packet error", zap.Error(err)) - } vars.systems[CharacterSetConnection], vars.systems[CollationConnection] = charset.GetDefaultCharsetAndCollate() return vars } From 7a1d3f8d1e1829e74ae088099b04925f68ffc9aa Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Mon, 8 Jan 2024 21:15:13 +0800 Subject: [PATCH 8/9] add ut Signed-off-by: wjhuang2016 --- .../clustertablestest/cluster_tables_test.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/infoschema/test/clustertablestest/cluster_tables_test.go b/pkg/infoschema/test/clustertablestest/cluster_tables_test.go index 5894e9bcc81eb..307b930bcc13a 100644 --- a/pkg/infoschema/test/clustertablestest/cluster_tables_test.go +++ b/pkg/infoschema/test/clustertablestest/cluster_tables_test.go @@ -834,13 +834,15 @@ func (s *clusterTablesSuite) newTestKitWithRoot(t *testing.T) *testkit.TestKit { func TestMDLView(t *testing.T) { testCases := []struct { name string - createTable string + createTable []string ddl string queryInTxn []string sqlDigest string }{ - {"add column", "create table t(a int)", "alter table test.t add column b int", []string{"select 1", "select * from t"}, "[\"begin\",\"select ?\",\"select * from `t`\"]"}, - {"change column in 1 step", "create table t(a int)", "alter table test.t change column a b int", []string{"select 1", "select * from t"}, "[\"begin\",\"select ?\",\"select * from `t`\"]"}, + {"add column", []string{"create table t(a int)"}, "alter table test.t add column b int", []string{"select 1", "select * from t"}, "[\"begin\",\"select ?\",\"select * from `t`\"]"}, + {"change column in 1 step", []string{"create table t(a int)"}, "alter table test.t change column a b int", []string{"select 1", "select * from t"}, "[\"begin\",\"select ?\",\"select * from `t`\"]"}, + {"rename tables", []string{"create table t(a int)", "create table t1(a int)"}, "rename table test.t to test.t2, test.t1 to test.t3", []string{"select 1", "select * from t"}, "[\"begin\",\"select ?\",\"select * from `t`\"]"}, + {"err don't show rollbackdone ddl", []string{"create table t(a int)", "insert into t values (1);", "insert into t values (1);", "alter table t add unique idx(id);"}, "alter table test.t add column b int", []string{"select 1", "select * from t"}, "[\"begin\",\"select ?\",\"select * from `t`\"]"}, } save := privileges.SkipWithGrant privileges.SkipWithGrant = true @@ -861,7 +863,13 @@ func TestMDLView(t *testing.T) { tk3 := s.newTestKitWithRoot(t) tk.MustExec("use test") tk.MustExec("set global tidb_enable_metadata_lock=1") - tk.MustExec(c.createTable) + for _, cr := range c.createTable { + if strings.Contains(c.name, "err") { + _, _ = tk.Exec(cr) + } else { + tk.MustExec(cr) + } + } tk.MustExec("begin") for _, q := range c.queryInTxn { From cd956d8b3ef4249cc40847d640d5014cd28354db Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Mon, 8 Jan 2024 23:25:24 +0800 Subject: [PATCH 9/9] fix Signed-off-by: wjhuang2016 --- pkg/session/bootstraptest/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/session/bootstraptest/BUILD.bazel b/pkg/session/bootstraptest/BUILD.bazel index 4b25d2e72d946..a483ef5977496 100644 --- a/pkg/session/bootstraptest/BUILD.bazel +++ b/pkg/session/bootstraptest/BUILD.bazel @@ -2,7 +2,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_test") go_test( name = "bootstraptest_test", - timeout = "short", + timeout = "moderate", srcs = [ "bootstrap_upgrade_test.go", #keep "main_test.go",