From 099a974f8fb6c6852f7c238425dc49188fbac841 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Tue, 21 Mar 2023 16:43:17 +0800 Subject: [PATCH 01/10] bootstrap: use BatchCreateTableWithInfo to speed up Signed-off-by: lance6716 --- session/bootstrap.go | 144 +++++++++++++++----------------------- session/bootstrap_test.go | 2 +- 2 files changed, 57 insertions(+), 89 deletions(-) diff --git a/session/bootstrap.go b/session/bootstrap.go index b4841d3e38928..ca0af6df8d032 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -40,6 +40,7 @@ import ( "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/owner" "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/auth" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/parser/mysql" @@ -586,8 +587,27 @@ const ( PRIMARY KEY (job_id), KEY (create_time), KEY (create_user));` + // please make sure newly added CREATE TABLE consts are also added to below + // allCreateTables ) +var allCreateTables = []string{ + CreateUserTable, CreateGlobalPrivTable, CreateDBPrivTable, + CreateTablePrivTable, CreateColumnPrivTable, CreateGlobalVariablesTable, + CreateTiDBTable, CreateHelpTopic, CreateStatsMetaTable, + CreateStatsColsTable, CreateStatsBucketsTable, CreateGCDeleteRangeTable, + CreateGCDeleteRangeDoneTable, CreateStatsFeedbackTable, CreateBindInfoTable, + CreateRoleEdgesTable, CreateDefaultRolesTable, CreateStatsTopNTable, + CreateStatsFMSketchTable, CreateExprPushdownBlacklist, CreateOptRuleBlacklist, + CreateStatsExtended, CreateSchemaIndexUsageTable, CreateGlobalGrantsTable, + CreateCapturePlanBaselinesBlacklist, CreateColumnStatsUsageTable, + CreateTableCacheMetaTable, CreateAnalyzeOptionsTable, CreateStatsHistory, + CreateStatsMetaHistory, CreateAnalyzeJobs, CreateAdvisoryLocks, + CreatePlanReplayerStatusTable, CreatePlanReplayerTaskTable, + CreateStatsTableLocked, CreatePasswordHistory, CreateTTLTableStatus, + CreateTTLTask, CreateTTLJobHistory, CreateGlobalTask, CreateLoadDataJobs, +} + // bootstrap initiates system DB for a store. func bootstrap(s Session) { startTime := time.Now() @@ -613,7 +633,7 @@ func bootstrap(s Session) { // To reduce conflict when multiple TiDB-server start at the same time. // Actually only one server need to do the bootstrap. So we chose DDL owner to do this. if dom.DDL().OwnerManager().IsOwner() { - doDDLWorks(s) + doDDLWorks(s, dom.DDL()) doDMLWorks(s) runBootstrapSQLFile = true logutil.BgLogger().Info("bootstrap successful", @@ -1747,11 +1767,6 @@ func upgradeToVer57(s Session, ver int64) { insertBuiltinBindInfoRow(s) } -func initBindInfoTable(s Session) { - mustExecute(s, CreateBindInfoTable) - insertBuiltinBindInfoRow(s) -} - func insertBuiltinBindInfoRow(s Session) { mustExecute(s, `INSERT HIGH_PRIORITY INTO mysql.bind_info(original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source) VALUES (%?, %?, "mysql", %?, "0000-00-00 00:00:00", "0000-00-00 00:00:00", "", "", %?)`, @@ -2460,94 +2475,47 @@ func getBootstrapVersion(s Session) (int64, error) { } // doDDLWorks executes DDL statements in bootstrap stage. -func doDDLWorks(s Session) { +func doDDLWorks(s Session, d ddl.DDL) { // Create a test database. mustExecute(s, "CREATE DATABASE IF NOT EXISTS test") // Create system db. mustExecute(s, "CREATE DATABASE IF NOT EXISTS %n", mysql.SystemDB) - // Create user table. - mustExecute(s, CreateUserTable) - // Create password history. - mustExecute(s, CreatePasswordHistory) - // Create privilege tables. - mustExecute(s, CreateGlobalPrivTable) - mustExecute(s, CreateDBPrivTable) - mustExecute(s, CreateTablePrivTable) - mustExecute(s, CreateColumnPrivTable) - // Create global system variable table. - mustExecute(s, CreateGlobalVariablesTable) - // Create TiDB table. - mustExecute(s, CreateTiDBTable) - // Create help table. - mustExecute(s, CreateHelpTopic) - // Create stats_meta table. - mustExecute(s, CreateStatsMetaTable) - // Create stats_columns table. - mustExecute(s, CreateStatsColsTable) - // Create stats_buckets table. - mustExecute(s, CreateStatsBucketsTable) - // Create gc_delete_range table. - mustExecute(s, CreateGCDeleteRangeTable) - // Create gc_delete_range_done table. - mustExecute(s, CreateGCDeleteRangeDoneTable) - // Create stats_feedback table. - mustExecute(s, CreateStatsFeedbackTable) - // Create role_edges table. - mustExecute(s, CreateRoleEdgesTable) - // Create default_roles table. - mustExecute(s, CreateDefaultRolesTable) - // Create bind_info table. - initBindInfoTable(s) - // Create stats_topn_store table. - mustExecute(s, CreateStatsTopNTable) - // Create expr_pushdown_blacklist table. - mustExecute(s, CreateExprPushdownBlacklist) - // Create opt_rule_blacklist table. - mustExecute(s, CreateOptRuleBlacklist) - // Create stats_extended table. - mustExecute(s, CreateStatsExtended) - // Create schema_index_usage. - mustExecute(s, CreateSchemaIndexUsageTable) - // Create stats_fm_sketch table. - mustExecute(s, CreateStatsFMSketchTable) - // Create global_grants - mustExecute(s, CreateGlobalGrantsTable) - // Create capture_plan_baselines_blacklist - mustExecute(s, CreateCapturePlanBaselinesBlacklist) - // Create column_stats_usage table - mustExecute(s, CreateColumnStatsUsageTable) - // Create table_cache_meta table. - mustExecute(s, CreateTableCacheMetaTable) - // Create analyze_options table. - mustExecute(s, CreateAnalyzeOptionsTable) - // Create stats_history table. - mustExecute(s, CreateStatsHistory) - // Create stats_meta_history table. - mustExecute(s, CreateStatsMetaHistory) - // Create analyze_jobs table. - mustExecute(s, CreateAnalyzeJobs) - // Create advisory_locks table. - mustExecute(s, CreateAdvisoryLocks) - // Create mdl view. - mustExecute(s, CreateMDLView) - // Create plan_replayer_status table - mustExecute(s, CreatePlanReplayerStatusTable) - // Create plan_replayer_task table - mustExecute(s, CreatePlanReplayerTaskTable) - // Create stats_meta_table_locked table - mustExecute(s, CreateStatsTableLocked) - // Create tidb_ttl_table_status table - mustExecute(s, CreateTTLTableStatus) - // Create tidb_ttl_task table - mustExecute(s, CreateTTLTask) - // Create tidb_ttl_job_history table - mustExecute(s, CreateTTLJobHistory) - // Create tidb_global_task table - mustExecute(s, CreateGlobalTask) + p := parser.New() + tableInfos := make([]*model.TableInfo, 0, len(allCreateTables)) + for _, createTable := range allCreateTables { + stmt, err := p.ParseOneStmt(createTable, "", "") + if err != nil { + logutil.BgLogger().Fatal("ParseOneStmt error", zap.Error(err), zap.Stack("stack")) + } + tblInfo, err := ddl.BuildTableInfoFromAST(stmt.(*ast.CreateTableStmt)) + if err != nil { + logutil.BgLogger().Fatal("BuildTableInfoFromAST error", zap.Error(err), zap.Stack("stack")) + } + tableInfos = append(tableInfos, tblInfo) + } + batchCreateTable(s, d, tableInfos) + + // init bind_info table. + insertBuiltinBindInfoRow(s) // Create default resource group mustExecute(s, CreateDefaultResourceGroup) - // Create load_data_jobs - mustExecute(s, CreateLoadDataJobs) +} + +func batchCreateTable(s Session, d ddl.DDL, tableInfos []*model.TableInfo) { + err := d.BatchCreateTableWithInfo(s, model.NewCIStr(mysql.SystemDB), tableInfos, ddl.OnExistIgnore) + if err == nil { + return + } + if kv.ErrEntryTooLarge.Equal(err) { + if len(tableInfos) == 1 { + logutil.BgLogger().Fatal("too large TableInfo", zap.Any("tableInfo", tableInfos[0]), zap.Stack("stack")) + } + mid := len(tableInfos)/2 + 1 + batchCreateTable(s, d, tableInfos[:mid]) + batchCreateTable(s, d, tableInfos[mid:]) + } else { + logutil.BgLogger().Fatal("batchCreateTable meet error", zap.Error(err), zap.Stack("stack")) + } } // doBootstrapSQLFile executes SQL commands in a file as the last stage of bootstrap. diff --git a/session/bootstrap_test.go b/session/bootstrap_test.go index dddb2f7dcd7a6..54f2c1792d6bf 100644 --- a/session/bootstrap_test.go +++ b/session/bootstrap_test.go @@ -164,7 +164,7 @@ func TestBootstrapWithError(t *testing.T) { b, err := checkBootstrapped(se) require.False(t, b) require.NoError(t, err) - doDDLWorks(se) + doDDLWorks(se, dom.DDL()) } dom, err := domap.Get(store) From c13cbc5acfbe479616a3006847e3f8a116907a25 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Tue, 21 Mar 2023 16:53:41 +0800 Subject: [PATCH 02/10] fix CI Signed-off-by: lance6716 --- session/bootstrap.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/session/bootstrap.go b/session/bootstrap.go index ca0af6df8d032..91547eb338304 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -46,6 +46,7 @@ import ( "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/parser/terror" "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/types" @@ -2502,6 +2503,8 @@ func doDDLWorks(s Session, d ddl.DDL) { } func batchCreateTable(s Session, d ddl.DDL, tableInfos []*model.TableInfo) { + s.SetValue(sessionctx.QueryString, "skip") + err := d.BatchCreateTableWithInfo(s, model.NewCIStr(mysql.SystemDB), tableInfos, ddl.OnExistIgnore) if err == nil { return From 5efd9878a70cccd41cab64b124fbd438e3d8838b Mon Sep 17 00:00:00 2001 From: lance6716 Date: Tue, 21 Mar 2023 17:04:15 +0800 Subject: [PATCH 03/10] fix CI Signed-off-by: lance6716 --- session/bootstrap.go | 10 +++++++--- session/bootstrap_test.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/session/bootstrap.go b/session/bootstrap.go index 91547eb338304..2de54f788ef2b 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -634,7 +634,7 @@ func bootstrap(s Session) { // To reduce conflict when multiple TiDB-server start at the same time. // Actually only one server need to do the bootstrap. So we chose DDL owner to do this. if dom.DDL().OwnerManager().IsOwner() { - doDDLWorks(s, dom.DDL()) + doDDLWorks(s, dom) doDMLWorks(s) runBootstrapSQLFile = true logutil.BgLogger().Info("bootstrap successful", @@ -2476,7 +2476,7 @@ func getBootstrapVersion(s Session) (int64, error) { } // doDDLWorks executes DDL statements in bootstrap stage. -func doDDLWorks(s Session, d ddl.DDL) { +func doDDLWorks(s Session, dom *domain.Domain) { // Create a test database. mustExecute(s, "CREATE DATABASE IF NOT EXISTS test") // Create system db. @@ -2494,7 +2494,11 @@ func doDDLWorks(s Session, d ddl.DDL) { } tableInfos = append(tableInfos, tblInfo) } - batchCreateTable(s, d, tableInfos) + batchCreateTable(s, dom.DDL(), tableInfos) + err := dom.Reload() + if err != nil { + logutil.BgLogger().Fatal("Reload error", zap.Error(err), zap.Stack("stack")) + } // init bind_info table. insertBuiltinBindInfoRow(s) diff --git a/session/bootstrap_test.go b/session/bootstrap_test.go index 54f2c1792d6bf..b26c4bf051dcd 100644 --- a/session/bootstrap_test.go +++ b/session/bootstrap_test.go @@ -164,7 +164,7 @@ func TestBootstrapWithError(t *testing.T) { b, err := checkBootstrapped(se) require.False(t, b) require.NoError(t, err) - doDDLWorks(se, dom.DDL()) + doDDLWorks(se, dom) } dom, err := domap.Get(store) From 0220324705d2f891216454ec9b2c4860d45faf00 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Tue, 21 Mar 2023 17:17:54 +0800 Subject: [PATCH 04/10] try fix CI Signed-off-by: lance6716 --- ddl/ddl_api.go | 5 +++++ session/bootstrap.go | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 504954bdcfc11..7df12577ee178 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -2221,6 +2221,11 @@ func BuildTableInfoFromAST(s *ast.CreateTableStmt) (*model.TableInfo, error) { return buildTableInfoWithCheck(mock.NewContext(), s, mysql.DefaultCharset, "", nil) } +// BuildTableInfoWithContext builds model.TableInfo from a SQL statement using given context. +func BuildTableInfoWithContext(ctx sessionctx.Context, s *ast.CreateTableStmt) (*model.TableInfo, error) { + return buildTableInfoWithCheck(ctx, s, mysql.DefaultCharset, "", nil) +} + // buildTableInfoWithCheck builds model.TableInfo from a SQL statement. // Note: TableID and PartitionIDs are left as uninitialized value. func buildTableInfoWithCheck(ctx sessionctx.Context, s *ast.CreateTableStmt, dbCharset, dbCollate string, placementPolicyRef *model.PolicyRefInfo) (*model.TableInfo, error) { diff --git a/session/bootstrap.go b/session/bootstrap.go index 2de54f788ef2b..bd727c3a87cd6 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -2488,7 +2488,7 @@ func doDDLWorks(s Session, dom *domain.Domain) { if err != nil { logutil.BgLogger().Fatal("ParseOneStmt error", zap.Error(err), zap.Stack("stack")) } - tblInfo, err := ddl.BuildTableInfoFromAST(stmt.(*ast.CreateTableStmt)) + tblInfo, err := ddl.BuildTableInfoWithContext(s, stmt.(*ast.CreateTableStmt)) if err != nil { logutil.BgLogger().Fatal("BuildTableInfoFromAST error", zap.Error(err), zap.Stack("stack")) } @@ -2517,7 +2517,7 @@ func batchCreateTable(s Session, d ddl.DDL, tableInfos []*model.TableInfo) { if len(tableInfos) == 1 { logutil.BgLogger().Fatal("too large TableInfo", zap.Any("tableInfo", tableInfos[0]), zap.Stack("stack")) } - mid := len(tableInfos)/2 + 1 + mid := len(tableInfos) / 2 batchCreateTable(s, d, tableInfos[:mid]) batchCreateTable(s, d, tableInfos[mid:]) } else { From f4e71298fb0b2cde7465e4f6770a6d2ef6b5b493 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Tue, 21 Mar 2023 17:44:09 +0800 Subject: [PATCH 05/10] fix CI Signed-off-by: lance6716 --- ddl/schematracker/checker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddl/schematracker/checker.go b/ddl/schematracker/checker.go index 9477d478388ad..419f27da33a10 100644 --- a/ddl/schematracker/checker.go +++ b/ddl/schematracker/checker.go @@ -472,8 +472,8 @@ func (d Checker) CreateTableWithInfo(ctx sessionctx.Context, schema model.CIStr, // BatchCreateTableWithInfo implements the DDL interface. func (d Checker) BatchCreateTableWithInfo(ctx sessionctx.Context, schema model.CIStr, info []*model.TableInfo, cs ...ddl.CreateTableWithInfoConfigurier) error { - //TODO implement me - panic("implement me") + // currently this is only used in bootstrap, we don't need to check it. + return d.realDDL.BatchCreateTableWithInfo(ctx, schema, info, cs...) } // CreatePlacementPolicyWithInfo implements the DDL interface. From 4516948a16198db11afaaee1c3f0f24192283f5f Mon Sep 17 00:00:00 2001 From: lance6716 Date: Tue, 21 Mar 2023 17:57:30 +0800 Subject: [PATCH 06/10] fix missing MDL view Signed-off-by: lance6716 --- session/bootstrap.go | 1 + 1 file changed, 1 insertion(+) diff --git a/session/bootstrap.go b/session/bootstrap.go index bd727c3a87cd6..91813ba21cc8d 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -2499,6 +2499,7 @@ func doDDLWorks(s Session, dom *domain.Domain) { if err != nil { logutil.BgLogger().Fatal("Reload error", zap.Error(err), zap.Stack("stack")) } + mustExecute(s, CreateMDLView) // init bind_info table. insertBuiltinBindInfoRow(s) From de3dbc5fb2e1131ba08946cd7224d289ea42df46 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Tue, 21 Mar 2023 18:49:53 +0800 Subject: [PATCH 07/10] retain TableID Signed-off-by: lance6716 --- ddl/ddl.go | 9 ++++++ ddl/ddl_api.go | 2 ++ session/bootstrap.go | 72 ++++++++++++++++++++++++++++++-------------- 3 files changed, 60 insertions(+), 23 deletions(-) diff --git a/ddl/ddl.go b/ddl/ddl.go index f67272de3994c..305b13feef58d 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -112,6 +112,7 @@ type AllocTableIDIf func(*model.TableInfo) bool type CreateTableWithInfoConfig struct { OnExist OnExist ShouldAllocTableID AllocTableIDIf + SkipTableID int } // CreateTableWithInfoConfigurier is the "diff" which can be applied to the @@ -144,6 +145,14 @@ func (a AllocTableIDIf) Apply(c *CreateTableWithInfoConfig) { c.ShouldAllocTableID = a } +// SkipTableID is used to let BatchCreateTableWithInfo mimic the behaviour of CreateTable. +type SkipTableID int + +// Apply implements Configurier. +func (a SkipTableID) Apply(c *CreateTableWithInfoConfig) { + c.SkipTableID = int(a) +} + const ( // OnExistError throws an error on name collision. OnExistError OnExist = iota diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 7df12577ee178..dea9742b2d986 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -2603,6 +2603,7 @@ func (d *ddl) BatchCreateTableWithInfo(ctx sessionctx.Context, } } + totalID *= 1 + c.SkipTableID genIDs, err := d.genGlobalIDs(totalID) if err != nil { return errors.Trace(err) @@ -2617,6 +2618,7 @@ func (d *ddl) BatchCreateTableWithInfo(ctx sessionctx.Context, parts.Definitions[i].ID, genIDs = genIDs[0], genIDs[1:] } } + genIDs = genIDs[c.SkipTableID:] } job, err := d.createTableWithInfoJob(ctx, dbName, info, c.OnExist, true) diff --git a/session/bootstrap.go b/session/bootstrap.go index 91813ba21cc8d..18c21f2849b0c 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -589,25 +589,31 @@ const ( KEY (create_time), KEY (create_user));` // please make sure newly added CREATE TABLE consts are also added to below - // allCreateTables + // allCreateTablesPart2 ) -var allCreateTables = []string{ - CreateUserTable, CreateGlobalPrivTable, CreateDBPrivTable, - CreateTablePrivTable, CreateColumnPrivTable, CreateGlobalVariablesTable, - CreateTiDBTable, CreateHelpTopic, CreateStatsMetaTable, - CreateStatsColsTable, CreateStatsBucketsTable, CreateGCDeleteRangeTable, - CreateGCDeleteRangeDoneTable, CreateStatsFeedbackTable, CreateBindInfoTable, - CreateRoleEdgesTable, CreateDefaultRolesTable, CreateStatsTopNTable, - CreateStatsFMSketchTable, CreateExprPushdownBlacklist, CreateOptRuleBlacklist, - CreateStatsExtended, CreateSchemaIndexUsageTable, CreateGlobalGrantsTable, - CreateCapturePlanBaselinesBlacklist, CreateColumnStatsUsageTable, - CreateTableCacheMetaTable, CreateAnalyzeOptionsTable, CreateStatsHistory, - CreateStatsMetaHistory, CreateAnalyzeJobs, CreateAdvisoryLocks, - CreatePlanReplayerStatusTable, CreatePlanReplayerTaskTable, - CreateStatsTableLocked, CreatePasswordHistory, CreateTTLTableStatus, - CreateTTLTask, CreateTTLJobHistory, CreateGlobalTask, CreateLoadDataJobs, -} +var ( + allCreateTablesPart1 = []string{ + CreateUserTable, CreatePasswordHistory, CreateGlobalPrivTable, + CreateDBPrivTable, CreateTablePrivTable, CreateColumnPrivTable, + CreateGlobalVariablesTable, CreateTiDBTable, CreateHelpTopic, + CreateStatsMetaTable, CreateStatsColsTable, CreateStatsBucketsTable, + CreateGCDeleteRangeTable, CreateGCDeleteRangeDoneTable, CreateStatsFeedbackTable, + CreateRoleEdgesTable, CreateDefaultRolesTable, CreateBindInfoTable, + CreateStatsTopNTable, CreateExprPushdownBlacklist, CreateOptRuleBlacklist, + CreateStatsExtended, CreateSchemaIndexUsageTable, CreateStatsFMSketchTable, + CreateGlobalGrantsTable, CreateCapturePlanBaselinesBlacklist, + CreateColumnStatsUsageTable, CreateTableCacheMetaTable, CreateAnalyzeOptionsTable, + CreateStatsHistory, CreateStatsMetaHistory, CreateAnalyzeJobs, + CreateAdvisoryLocks, + } + // we should create MDL view between allCreateTablesPart1 and allCreateTablesPart2 + allCreateTablesPart2 = []string{ + CreatePlanReplayerStatusTable, CreatePlanReplayerTaskTable, + CreateStatsTableLocked, CreateTTLTableStatus, CreateTTLTask, + CreateTTLJobHistory, CreateGlobalTask, CreateLoadDataJobs, + } +) // bootstrap initiates system DB for a store. func bootstrap(s Session) { @@ -2482,8 +2488,8 @@ func doDDLWorks(s Session, dom *domain.Domain) { // Create system db. mustExecute(s, "CREATE DATABASE IF NOT EXISTS %n", mysql.SystemDB) p := parser.New() - tableInfos := make([]*model.TableInfo, 0, len(allCreateTables)) - for _, createTable := range allCreateTables { + tableInfosPart1 := make([]*model.TableInfo, 0, len(allCreateTablesPart1)) + for _, createTable := range allCreateTablesPart1 { stmt, err := p.ParseOneStmt(createTable, "", "") if err != nil { logutil.BgLogger().Fatal("ParseOneStmt error", zap.Error(err), zap.Stack("stack")) @@ -2492,14 +2498,28 @@ func doDDLWorks(s Session, dom *domain.Domain) { if err != nil { logutil.BgLogger().Fatal("BuildTableInfoFromAST error", zap.Error(err), zap.Stack("stack")) } - tableInfos = append(tableInfos, tblInfo) + tableInfosPart1 = append(tableInfosPart1, tblInfo) } - batchCreateTable(s, dom.DDL(), tableInfos) + batchCreateTable(s, dom.DDL(), tableInfosPart1) + mustExecute(s, CreateMDLView) + tableInfosPart2 := make([]*model.TableInfo, 0, len(allCreateTablesPart2)) + for _, createTable := range allCreateTablesPart2 { + stmt, err := p.ParseOneStmt(createTable, "", "") + if err != nil { + logutil.BgLogger().Fatal("ParseOneStmt error", zap.Error(err), zap.Stack("stack")) + } + tblInfo, err := ddl.BuildTableInfoWithContext(s, stmt.(*ast.CreateTableStmt)) + if err != nil { + logutil.BgLogger().Fatal("BuildTableInfoFromAST error", zap.Error(err), zap.Stack("stack")) + } + tableInfosPart2 = append(tableInfosPart2, tblInfo) + } + batchCreateTable(s, dom.DDL(), tableInfosPart2) + err := dom.Reload() if err != nil { logutil.BgLogger().Fatal("Reload error", zap.Error(err), zap.Stack("stack")) } - mustExecute(s, CreateMDLView) // init bind_info table. insertBuiltinBindInfoRow(s) @@ -2510,7 +2530,13 @@ func doDDLWorks(s Session, dom *domain.Domain) { func batchCreateTable(s Session, d ddl.DDL, tableInfos []*model.TableInfo) { s.SetValue(sessionctx.QueryString, "skip") - err := d.BatchCreateTableWithInfo(s, model.NewCIStr(mysql.SystemDB), tableInfos, ddl.OnExistIgnore) + err := d.BatchCreateTableWithInfo( + s, + model.NewCIStr(mysql.SystemDB), + tableInfos, + ddl.OnExistIgnore, + ddl.SkipTableID(1), + ) if err == nil { return } From 021de88667b363e3616b0a1f63bdace8b0bc4d6f Mon Sep 17 00:00:00 2001 From: lance6716 Date: Tue, 21 Mar 2023 19:24:45 +0800 Subject: [PATCH 08/10] fix CI Signed-off-by: lance6716 --- executor/executor_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 65ece51b47877..0084d057d0304 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -5663,9 +5663,10 @@ func TestAdmin(t *testing.T) { tk.MustExec("use test") tk.MustExec("drop table if exists admin_test2") tk.MustExec("create table admin_test2 (c1 int, c2 int, c3 int default 1, index (c1))") + // we cn only see last 10 queries result := tk.MustQuery(`admin show ddl job queries 1, 1, 1`) result.Check(testkit.Rows()) - result = tk.MustQuery(`admin show ddl job queries 1, 2, 3, 4`) + result = tk.MustQuery(`admin show ddl job queries 1, 2`) result.Check(testkit.Rows()) historyJobs, err = ddl.GetLastNHistoryDDLJobs(meta.NewMeta(txn), ddl.DefNumHistoryJobs) result = tk.MustQuery(fmt.Sprintf("admin show ddl job queries %d", historyJobs[0].ID)) From 935bcd228437fcbf0933f0a769085c80fe19e58e Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 22 Mar 2023 17:45:33 +0800 Subject: [PATCH 09/10] Update executor/executor_test.go Co-authored-by: lance6716 --- executor/executor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 0084d057d0304..c8e43581936ea 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -5663,7 +5663,7 @@ func TestAdmin(t *testing.T) { tk.MustExec("use test") tk.MustExec("drop table if exists admin_test2") tk.MustExec("create table admin_test2 (c1 int, c2 int, c3 int default 1, index (c1))") - // we cn only see last 10 queries + // we can only see last 10 queries result := tk.MustQuery(`admin show ddl job queries 1, 1, 1`) result.Check(testkit.Rows()) result = tk.MustQuery(`admin show ddl job queries 1, 2`) From 98b8e9e294a89355430882de0d215f4a5e1705b9 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Wed, 22 Mar 2023 19:28:02 +0800 Subject: [PATCH 10/10] address comment Signed-off-by: lance6716 --- session/bootstrap.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/session/bootstrap.go b/session/bootstrap.go index 18c21f2849b0c..20629425cbda5 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -2501,6 +2501,10 @@ func doDDLWorks(s Session, dom *domain.Domain) { tableInfosPart1 = append(tableInfosPart1, tblInfo) } batchCreateTable(s, dom.DDL(), tableInfosPart1) + // batchCreateTable is an optimization, in order to keep same behaviour with + // before (such as assign same table ID to these tables), we need to execute + // the DDLs in order. CreateMDLView is a view that cannot use batchCreateTable, + // so it breaks tables into allCreateTablesPart1 and allCreateTablesPart2. mustExecute(s, CreateMDLView) tableInfosPart2 := make([]*model.TableInfo, 0, len(allCreateTablesPart2)) for _, createTable := range allCreateTablesPart2 {