From 5c341d021e12a62c7be1fb573738ec13c21f034c Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 15 Sep 2022 14:29:53 +0800 Subject: [PATCH 01/11] support multi Signed-off-by: yisaer --- executor/builder.go | 12 +- executor/plan_replayer.go | 292 +++++++++++++++++++++++--------------- server/conn.go | 27 ++++ 3 files changed, 218 insertions(+), 113 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index 649faeeae3ad5..2a34bc8862075 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -977,8 +977,16 @@ func (b *executorBuilder) buildPlanReplayer(v *plannercore.PlanReplayer) Executo } e := &PlanReplayerExec{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()), - ExecStmts: []ast.StmtNode{v.ExecStmt}, - Analyze: v.Analyze, + DumpInfo: &PlanReplayerDumpInfo{ + Analyze: v.Analyze, + Path: v.File, + ctx: b.ctx, + }, + } + if v.ExecStmt != nil { + e.DumpInfo.ExecStmts = []ast.StmtNode{v.ExecStmt} + } else { + e.baseExecutor = newBaseExecutor(b.ctx, nil, v.ID()) } return e } diff --git a/executor/plan_replayer.go b/executor/plan_replayer.go index b213717d137c9..22bc9b34fcf70 100644 --- a/executor/plan_replayer.go +++ b/executor/plan_replayer.go @@ -34,6 +34,7 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/parser" "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx" @@ -58,103 +59,84 @@ const ( globalBindingFile = "global_bindings.sql" ) -// PlanReplayerExec represents a plan replayer executor. -type PlanReplayerExec struct { - baseExecutor - ExecStmts []ast.StmtNode - Analyze bool +type planReplayerDumpKeyType int - endFlag bool +func (k planReplayerDumpKeyType) String() string { + return "plan_replayer_dump_var" } -type tableNamePair struct { - DBName string - TableName string - IsView bool -} +// PlanReplayerDumpVarKey is a variable key for plan replayer dump. +const PlanReplayerDumpVarKey planReplayerDumpKeyType = 0 -type tableNameExtractor struct { - ctx context.Context - executor sqlexec.RestrictedSQLExecutor - is infoschema.InfoSchema - curDB model.CIStr - names map[tableNamePair]struct{} - cteNames map[string]struct{} - err error +// PlanReplayerExec represents a plan replayer executor. +type PlanReplayerExec struct { + baseExecutor + DumpInfo *PlanReplayerDumpInfo + endFlag bool } -func (tne *tableNameExtractor) Enter(in ast.Node) (ast.Node, bool) { - if _, ok := in.(*ast.TableName); ok { - return in, true - } - return in, false +type PlanReplayerDumpInfo struct { + ExecStmts []ast.StmtNode + Analyze bool + Path string + File *os.File + FileName string + ctx sessionctx.Context } -func (tne *tableNameExtractor) Leave(in ast.Node) (ast.Node, bool) { - if tne.err != nil { - return in, true +// Next implements the Executor Next interface. +func (e *PlanReplayerExec) Next(ctx context.Context, req *chunk.Chunk) error { + req.GrowAndReset(e.maxChunkSize) + if e.endFlag { + return nil } - if t, ok := in.(*ast.TableName); ok { - isView, err := tne.handleIsView(t) + err := e.createFile(domain.GetPlanReplayerDirName()) + if err != nil { + return err + } + if len(e.DumpInfo.Path) > 0 { + err = e.setKey() if err != nil { - tne.err = err - return in, true - } - tp := tableNamePair{DBName: t.Schema.L, TableName: t.Name.L, IsView: isView} - if tp.DBName == "" { - tp.DBName = tne.curDB.L - } - if _, ok := tne.names[tp]; !ok { - tne.names[tp] = struct{}{} - } - } else if s, ok := in.(*ast.SelectStmt); ok { - if s.With != nil && len(s.With.CTEs) > 0 { - for _, cte := range s.With.CTEs { - tne.cteNames[cte.Name.L] = struct{}{} - } + return err } + e.endFlag = true + return nil } - return in, true -} - -func (tne *tableNameExtractor) handleIsView(t *ast.TableName) (bool, error) { - schema := t.Schema - if schema.L == "" { - schema = tne.curDB - } - table := t.Name - isView := tne.is.TableIsView(schema, table) - if !isView { - return false, nil - } - viewTbl, err := tne.is.TableByName(schema, table) - if err != nil { - return false, err + if e.DumpInfo.ExecStmts == nil { + return errors.New("plan replayer: sql is empty") } - sql := viewTbl.Meta().View.SelectStmt - node, err := tne.executor.ParseWithParams(tne.ctx, sql) + err = e.DumpInfo.dump(ctx) if err != nil { - return false, err + return err } - node.Accept(tne) - return true, nil + req.AppendString(0, e.DumpInfo.FileName) + e.endFlag = true + return nil } -// Next implements the Executor Next interface. -func (e *PlanReplayerExec) Next(ctx context.Context, req *chunk.Chunk) error { - req.GrowAndReset(e.maxChunkSize) - if e.endFlag { - return nil - } - if e.ExecStmts == nil { - return errors.New("plan replayer: sql is empty") +func (e *PlanReplayerExec) createFile(path string) error { + // Create path + err := os.MkdirAll(path, os.ModePerm) + if err != nil { + return errors.AddStack(err) } - res, err := e.dump(ctx, domain.GetPlanReplayerDirName()) + + // Generate key and create zip file + time := time.Now().UnixNano() + b := make([]byte, 16) + //nolint: gosec + _, err = rand.Read(b) if err != nil { return err } - req.AppendString(0, res) - e.endFlag = true + key := base64.URLEncoding.EncodeToString(b) + fileName := fmt.Sprintf("replayer_%v_%v.zip", key, time) + zf, err := os.Create(filepath.Join(path, fileName)) + if err != nil { + return errors.AddStack(err) + } + e.DumpInfo.File = zf + e.DumpInfo.FileName = fileName return nil } @@ -187,28 +169,9 @@ func (e *PlanReplayerExec) Next(ctx context.Context, req *chunk.Chunk) error { |-explain2.txt |-.... */ -func (e *PlanReplayerExec) dump(ctx context.Context, path string) (fileName string, err error) { - // Create path - err = os.MkdirAll(path, os.ModePerm) - if err != nil { - return "", errors.AddStack(err) - } - - // Generate key and create zip file - time := time.Now().UnixNano() - b := make([]byte, 16) - //nolint: gosec - _, err = rand.Read(b) - if err != nil { - return "", err - } - key := base64.URLEncoding.EncodeToString(b) - fileName = fmt.Sprintf("replayer_%v_%v.zip", key, time) - zf, err := os.Create(filepath.Join(path, fileName)) - if err != nil { - return "", errors.AddStack(err) - } - +func (e *PlanReplayerDumpInfo) dump(ctx context.Context) (err error) { + fileName := e.FileName + zf := e.File // Create zip writer zw := zip.NewWriter(zf) defer func() { @@ -224,12 +187,12 @@ func (e *PlanReplayerExec) dump(ctx context.Context, path string) (fileName stri // Dump config if err = dumpConfig(zw); err != nil { - return "", err + return err } // Dump meta if err = dumpMeta(zw); err != nil { - return "", err + return err } // Retrieve current DB @@ -240,50 +203,50 @@ func (e *PlanReplayerExec) dump(ctx context.Context, path string) (fileName stri // Retrieve all tables pairs, err := e.extractTableNames(ctx, e.ExecStmts, dbName) if err != nil { - return "", errors.AddStack(fmt.Errorf("plan replayer: invalid SQL text, err: %v", err)) + return errors.AddStack(fmt.Errorf("plan replayer: invalid SQL text, err: %v", err)) } // Dump Schema and View if err = dumpSchemas(e.ctx, zw, pairs); err != nil { - return "", err + return err } // Dump tables tiflash replicas if err = dumpTiFlashReplica(e.ctx, zw, pairs); err != nil { - return "", err + return err } // Dump stats if err = dumpStats(zw, pairs, do); err != nil { - return "", err + return err } // Dump variables if err = dumpVariables(e.ctx, zw); err != nil { - return "", err + return err } // Dump sql if err = dumpSQLs(e.ExecStmts, zw); err != nil { - return "", err + return err } // Dump session bindings if err = dumpSessionBindings(e.ctx, zw); err != nil { - return "", err + return err } // Dump global bindings if err = dumpGlobalBindings(e.ctx, zw); err != nil { - return "", err + return err } // Dump explain if err = dumpExplain(e.ctx, zw, e.ExecStmts, e.Analyze); err != nil { - return "", err + return err } - return fileName, nil + return nil } func dumpConfig(zw *zip.Writer) error { @@ -497,7 +460,7 @@ func dumpExplain(ctx sessionctx.Context, zw *zip.Writer, execStmts []ast.StmtNod return nil } -func (e *PlanReplayerExec) extractTableNames(ctx context.Context, +func (e *PlanReplayerDumpInfo) extractTableNames(ctx context.Context, ExecStmts []ast.StmtNode, curDB model.CIStr) (map[tableNamePair]struct{}, error) { tableExtractor := &tableNameExtractor{ ctx: ctx, @@ -528,6 +491,40 @@ func (e *PlanReplayerExec) extractTableNames(ctx context.Context, return r, nil } +func (e *PlanReplayerExec) setKey() error { + if len(e.DumpInfo.Path) == 0 { + return errors.New("plan replayer: file path is empty") + } + val := e.ctx.Value(PlanReplayerDumpVarKey) + if val != nil { + e.ctx.SetValue(PlanReplayerDumpVarKey, nil) + return errors.New("plan replayer: previous plan replayer dump option isn't closed normally, please try again") + } + e.ctx.SetValue(PlanReplayerDumpVarKey, e.DumpInfo) + logutil.BgLogger().Info("plan replayer dump", zap.String("filename", e.DumpInfo.FileName)) + return nil +} + +// DumpSQLsFromFile dumps plan replayer results for sqls from file +func (e *PlanReplayerDumpInfo) DumpSQLsFromFile(ctx context.Context, b []byte) error { + sqls := strings.Split(string(b), ";") + p := parser.New() + e.ExecStmts = make([]ast.StmtNode, 0) + for _, sql := range sqls { + s := strings.Trim(sql, "\n") + if len(s) < 1 { + continue + } + node, err := p.ParseOneStmt(s, "", "") + if err != nil { + // TODO: add error info + return err + } + e.ExecStmts = append(e.ExecStmts, node) + } + return e.dump(ctx) +} + func getStatsForTable(do *domain.Domain, pair tableNamePair) (*handle.JSONTable, error) { is := do.InfoSchema() h := do.StatsHandle() @@ -925,3 +922,76 @@ func (e *PlanReplayerLoadInfo) Update(data []byte) error { } return nil } + +type tableNamePair struct { + DBName string + TableName string + IsView bool +} + +type tableNameExtractor struct { + ctx context.Context + executor sqlexec.RestrictedSQLExecutor + is infoschema.InfoSchema + curDB model.CIStr + names map[tableNamePair]struct{} + cteNames map[string]struct{} + err error +} + +func (tne *tableNameExtractor) Enter(in ast.Node) (ast.Node, bool) { + if _, ok := in.(*ast.TableName); ok { + return in, true + } + return in, false +} + +func (tne *tableNameExtractor) Leave(in ast.Node) (ast.Node, bool) { + if tne.err != nil { + return in, true + } + if t, ok := in.(*ast.TableName); ok { + isView, err := tne.handleIsView(t) + if err != nil { + tne.err = err + return in, true + } + tp := tableNamePair{DBName: t.Schema.L, TableName: t.Name.L, IsView: isView} + if tp.DBName == "" { + tp.DBName = tne.curDB.L + } + if _, ok := tne.names[tp]; !ok { + tne.names[tp] = struct{}{} + } + } else if s, ok := in.(*ast.SelectStmt); ok { + if s.With != nil && len(s.With.CTEs) > 0 { + for _, cte := range s.With.CTEs { + tne.cteNames[cte.Name.L] = struct{}{} + } + } + } + return in, true +} + +func (tne *tableNameExtractor) handleIsView(t *ast.TableName) (bool, error) { + schema := t.Schema + if schema.L == "" { + schema = tne.curDB + } + table := t.Name + isView := tne.is.TableIsView(schema, table) + if !isView { + return false, nil + } + viewTbl, err := tne.is.TableByName(schema, table) + if err != nil { + return false, err + } + sql := viewTbl.Meta().View.SelectStmt + node, err := tne.executor.ParseWithParams(tne.ctx, sql) + if err != nil { + return false, err + } + node.Accept(tne) + return true, nil +} diff --git a/server/conn.go b/server/conn.go index 84ef340af5108..9a69b5b1eaec9 100644 --- a/server/conn.go +++ b/server/conn.go @@ -1815,6 +1815,24 @@ func (cc *clientConn) handlePlanReplayerLoad(ctx context.Context, planReplayerLo return planReplayerLoadInfo.Update(data) } +func (cc *clientConn) handlePlanReplayerDump(ctx context.Context, e *executor.PlanReplayerDumpInfo) error { + if cc.capability&mysql.ClientLocalFiles == 0 { + return errNotAllowedCommand + } + if e == nil { + return errors.New("plan replayer dump: executor is empty") + } + data, err := cc.getDataFromPath(ctx, e.Path) + if err != nil { + logutil.BgLogger().Error(err.Error()) + return err + } + if len(data) == 0 { + return nil + } + return e.DumpSQLsFromFile(ctx, data) +} + func (cc *clientConn) audit(eventType plugin.GeneralEvent) { err := plugin.ForeachPlugin(plugin.Audit, func(p *plugin.Plugin) error { audit := plugin.DeclareAuditManifest(p.Manifest) @@ -2117,6 +2135,15 @@ func (cc *clientConn) handleQuerySpecial(ctx context.Context, status uint16) (bo } } + planReplayerDump := cc.ctx.Value(executor.PlanReplayerDumpVarKey) + if planReplayerDump != nil { + handled = true + defer cc.ctx.SetValue(executor.PlanReplayerDumpVarKey, nil) + //nolint:forcetypeassert + if err := cc.handlePlanReplayerDump(ctx, planReplayerDump.(*executor.PlanReplayerDumpInfo)); err != nil { + return handled, err + } + } return handled, cc.writeOkWith(ctx, mysql.OKHeader, true, status) } From a22ca73bc8ad4307d7403119f190dfd5c95ed134 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 15 Sep 2022 14:45:58 +0800 Subject: [PATCH 02/11] support var Signed-off-by: yisaer --- executor/executor_test.go | 1 + executor/plan_replayer.go | 1 + server/plan_replayer_test.go | 8 ++++++++ sessionctx/variable/session.go | 3 +++ sessionctx/variable/sysvar.go | 5 +++++ sessionctx/variable/tidb_vars.go | 3 +++ 6 files changed, 21 insertions(+) diff --git a/executor/executor_test.go b/executor/executor_test.go index 8cfd71df003ca..91235ac74ad89 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -163,6 +163,7 @@ func TestPlanReplayer(t *testing.T) { tk.MustExec("plan replayer dump explain select * from t1 where t1.a > (with cte1 as (select 1) select count(1) from cte1);") tk.MustExec("plan replayer dump explain select * from v1") tk.MustExec("plan replayer dump explain select * from v2") + require.True(t, len(tk.Session().GetSessionVars().LastPlanReplayerToken) > 0) } func TestShow(t *testing.T) { diff --git a/executor/plan_replayer.go b/executor/plan_replayer.go index b213717d137c9..e780a223ff9f1 100644 --- a/executor/plan_replayer.go +++ b/executor/plan_replayer.go @@ -155,6 +155,7 @@ func (e *PlanReplayerExec) Next(ctx context.Context, req *chunk.Chunk) error { } req.AppendString(0, res) e.endFlag = true + e.ctx.GetSessionVars().LastPlanReplayerToken = res return nil } diff --git a/server/plan_replayer_test.go b/server/plan_replayer_test.go index 2ea91677d8438..2a00bc0db04de 100644 --- a/server/plan_replayer_test.go +++ b/server/plan_replayer_test.go @@ -136,5 +136,13 @@ func prepareData4PlanReplayer(t *testing.T, client *testServerClient, statHandle var filename string err = rows.Scan(&filename) require.NoError(t, err) + rows.Close() + rows = tk.MustQuery("select @@tidb_last_plan_replayer_token") + require.True(t, rows.Next(), "unexpected data") + var filename2 string + err = rows.Scan(&filename2) + require.NoError(t, err) + rows.Close() + require.Equal(t, filename, filename2) return filename } diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index ecb4b5b93c638..e7a2a96bd68b0 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1270,6 +1270,9 @@ type SessionVars struct { // ranges would exceed the limit, it chooses less accurate ranges such as full range. 0 indicates that there is no // memory limit for ranges. RangeMaxSize int64 + + // LastPlanReplayerToken indicates the last plan replayer token + LastPlanReplayerToken string } // GetPreparedStmtByName returns the prepared statement specified by stmtName. diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 680feb4460a55..6c30867675404 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -333,6 +333,11 @@ var defaultSysVars = []*SysVar{ } return string(info), nil }}, + {Scope: ScopeSession, Name: TiDBLastPlanReplayerToken, Value: "", ReadOnly: true, + GetSession: func(s *SessionVars) (string, error) { + return s.LastPlanReplayerToken, nil + }, + }, /* The system variables below have INSTANCE scope */ {Scope: ScopeInstance, Name: TiDBLogFileMaxDays, Value: strconv.Itoa(config.GetGlobalConfig().Log.File.MaxDays), Type: TypeInt, MinValue: 0, MaxValue: math.MaxInt32, SetGlobal: func(s *SessionVars, val string) error { maxAge, err := strconv.ParseInt(val, 10, 32) diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 8067f4ffb6ad5..8d9f845a07c78 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -91,6 +91,9 @@ const ( // TiDBLastDDLInfo is used to get the last ddl info within the current session. TiDBLastDDLInfo = "tidb_last_ddl_info" + // TiDBLastPlanReplayerToken is used to get the last plan replayer token within the current session + TiDBLastPlanReplayerToken = "tidb_last_plan_replayer_token" + // TiDBConfig is a read-only variable that shows the config of the current server. TiDBConfig = "tidb_config" From 0f078cee56f2432f121e82ffb4113818473f9765 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 15 Sep 2022 15:08:21 +0800 Subject: [PATCH 03/11] support var Signed-off-by: yisaer --- executor/plan_replayer.go | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/executor/plan_replayer.go b/executor/plan_replayer.go index 9d848f12e7620..d4902393819b4 100644 --- a/executor/plan_replayer.go +++ b/executor/plan_replayer.go @@ -59,15 +59,6 @@ const ( globalBindingFile = "global_bindings.sql" ) -type planReplayerDumpKeyType int - -func (k planReplayerDumpKeyType) String() string { - return "plan_replayer_dump_var" -} - -// PlanReplayerDumpVarKey is a variable key for plan replayer dump. -const PlanReplayerDumpVarKey planReplayerDumpKeyType = 0 - // PlanReplayerExec represents a plan replayer executor. type PlanReplayerExec struct { baseExecutor @@ -95,11 +86,12 @@ func (e *PlanReplayerExec) Next(ctx context.Context, req *chunk.Chunk) error { return err } if len(e.DumpInfo.Path) > 0 { - err = e.setKey() + err = e.prepare() if err != nil { return err } e.endFlag = true + e.ctx.GetSessionVars().LastPlanReplayerToken = e.DumpInfo.FileName return nil } if e.DumpInfo.ExecStmts == nil { @@ -111,7 +103,7 @@ func (e *PlanReplayerExec) Next(ctx context.Context, req *chunk.Chunk) error { } req.AppendString(0, e.DumpInfo.FileName) e.endFlag = true - e.ctx.GetSessionVars().LastPlanReplayerToken = res + e.ctx.GetSessionVars().LastPlanReplayerToken = e.DumpInfo.FileName return nil } @@ -492,7 +484,7 @@ func (e *PlanReplayerDumpInfo) extractTableNames(ctx context.Context, return r, nil } -func (e *PlanReplayerExec) setKey() error { +func (e *PlanReplayerExec) prepare() error { if len(e.DumpInfo.Path) == 0 { return errors.New("plan replayer: file path is empty") } @@ -502,7 +494,6 @@ func (e *PlanReplayerExec) setKey() error { return errors.New("plan replayer: previous plan replayer dump option isn't closed normally, please try again") } e.ctx.SetValue(PlanReplayerDumpVarKey, e.DumpInfo) - logutil.BgLogger().Info("plan replayer dump", zap.String("filename", e.DumpInfo.FileName)) return nil } @@ -641,14 +632,17 @@ type PlanReplayerLoadInfo struct { Ctx sessionctx.Context } -type planReplayerLoadKeyType int +type planReplayerKeyType int -func (k planReplayerLoadKeyType) String() string { +func (k planReplayerKeyType) String() string { return "plan_replayer_load_var" } // PlanReplayerLoadVarKey is a variable key for plan replayer load. -const PlanReplayerLoadVarKey planReplayerLoadKeyType = 0 +const PlanReplayerLoadVarKey planReplayerKeyType = 0 + +// PlanReplayerDumpVarKey is a variable key for plan replayer dump. +const PlanReplayerDumpVarKey planReplayerKeyType = 1 // Next implements the Executor Next interface. func (e *PlanReplayerLoadExec) Next(ctx context.Context, req *chunk.Chunk) error { From 5e1660e8485302bbb94e40515d946fe0fbd86af1 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 15 Sep 2022 15:13:40 +0800 Subject: [PATCH 04/11] support var Signed-off-by: yisaer --- executor/executor_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 91235ac74ad89..e01eac6cf31c8 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -152,17 +152,17 @@ func TestPlanReplayer(t *testing.T) { tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int, b int, index idx_a(a))") tk.MustExec("alter table t set tiflash replica 1") - tk.MustExec("plan replayer dump explain select * from t where a=10") - tk.MustExec("plan replayer dump explain select /*+ read_from_storage(tiflash[t]) */ * from t") + tk.MustQuery("plan replayer dump explain select * from t where a=10") + tk.MustQuery("plan replayer dump explain select /*+ read_from_storage(tiflash[t]) */ * from t") tk.MustExec("create table t1 (a int)") tk.MustExec("create table t2 (a int)") tk.MustExec("create definer=`root`@`127.0.0.1` view v1 as select * from t1") tk.MustExec("create definer=`root`@`127.0.0.1` view v2 as select * from v1") - tk.MustExec("plan replayer dump explain with tmp as (select a from t1 group by t1.a) select * from tmp, t2 where t2.a=tmp.a;") - tk.MustExec("plan replayer dump explain select * from t1 where t1.a > (with cte1 as (select 1) select count(1) from cte1);") - tk.MustExec("plan replayer dump explain select * from v1") - tk.MustExec("plan replayer dump explain select * from v2") + tk.MustQuery("plan replayer dump explain with tmp as (select a from t1 group by t1.a) select * from tmp, t2 where t2.a=tmp.a;") + tk.MustQuery("plan replayer dump explain select * from t1 where t1.a > (with cte1 as (select 1) select count(1) from cte1);") + tk.MustQuery("plan replayer dump explain select * from v1") + tk.MustQuery("plan replayer dump explain select * from v2") require.True(t, len(tk.Session().GetSessionVars().LastPlanReplayerToken) > 0) } From b0d4195d2ac705aff255cefada0ab036d94ce9c6 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 15 Sep 2022 15:20:34 +0800 Subject: [PATCH 05/11] add test Signed-off-by: yisaer --- executor/executor_test.go | 2 ++ executor/testdata/plan_replayer_test.txt | 1 + 2 files changed, 3 insertions(+) create mode 100644 executor/testdata/plan_replayer_test.txt diff --git a/executor/executor_test.go b/executor/executor_test.go index e01eac6cf31c8..752bf09a4d66f 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -154,6 +154,8 @@ func TestPlanReplayer(t *testing.T) { tk.MustExec("alter table t set tiflash replica 1") tk.MustQuery("plan replayer dump explain select * from t where a=10") tk.MustQuery("plan replayer dump explain select /*+ read_from_storage(tiflash[t]) */ * from t") + tk.MustExec("plan replayer dump explain './test/data/plan_replayer_test.txt'") + require.True(t, len(tk.Session().GetSessionVars().LastPlanReplayerToken) > 0) tk.MustExec("create table t1 (a int)") tk.MustExec("create table t2 (a int)") diff --git a/executor/testdata/plan_replayer_test.txt b/executor/testdata/plan_replayer_test.txt new file mode 100644 index 0000000000000..0a9959f55ad3e --- /dev/null +++ b/executor/testdata/plan_replayer_test.txt @@ -0,0 +1 @@ +select * from t;select * from t; From 3dcac8747efeb9d402a57553dfedd82776ed36ff Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 15 Sep 2022 18:40:37 +0800 Subject: [PATCH 06/11] add test Signed-off-by: yisaer --- executor/plan_replayer.go | 147 +++++++++++++++++++------------------- 1 file changed, 74 insertions(+), 73 deletions(-) diff --git a/executor/plan_replayer.go b/executor/plan_replayer.go index 9e4064fdfdcc7..f573b221e713a 100644 --- a/executor/plan_replayer.go +++ b/executor/plan_replayer.go @@ -66,6 +66,7 @@ type PlanReplayerExec struct { endFlag bool } +// PlanReplayerDumpInfo indicates dump info type PlanReplayerDumpInfo struct { ExecStmts []ast.StmtNode Analyze bool @@ -75,6 +76,79 @@ type PlanReplayerDumpInfo struct { ctx sessionctx.Context } +type tableNamePair struct { + DBName string + TableName string + IsView bool +} + +type tableNameExtractor struct { + ctx context.Context + executor sqlexec.RestrictedSQLExecutor + is infoschema.InfoSchema + curDB model.CIStr + names map[tableNamePair]struct{} + cteNames map[string]struct{} + err error +} + +func (tne *tableNameExtractor) Enter(in ast.Node) (ast.Node, bool) { + if _, ok := in.(*ast.TableName); ok { + return in, true + } + return in, false +} + +func (tne *tableNameExtractor) Leave(in ast.Node) (ast.Node, bool) { + if tne.err != nil { + return in, true + } + if t, ok := in.(*ast.TableName); ok { + isView, err := tne.handleIsView(t) + if err != nil { + tne.err = err + return in, true + } + tp := tableNamePair{DBName: t.Schema.L, TableName: t.Name.L, IsView: isView} + if tp.DBName == "" { + tp.DBName = tne.curDB.L + } + if _, ok := tne.names[tp]; !ok { + tne.names[tp] = struct{}{} + } + } else if s, ok := in.(*ast.SelectStmt); ok { + if s.With != nil && len(s.With.CTEs) > 0 { + for _, cte := range s.With.CTEs { + tne.cteNames[cte.Name.L] = struct{}{} + } + } + } + return in, true +} + +func (tne *tableNameExtractor) handleIsView(t *ast.TableName) (bool, error) { + schema := t.Schema + if schema.L == "" { + schema = tne.curDB + } + table := t.Name + isView := tne.is.TableIsView(schema, table) + if !isView { + return false, nil + } + viewTbl, err := tne.is.TableByName(schema, table) + if err != nil { + return false, err + } + sql := viewTbl.Meta().View.SelectStmt + node, err := tne.executor.ParseWithParams(tne.ctx, sql) + if err != nil { + return false, err + } + node.Accept(tne) + return true, nil +} + // Next implements the Executor Next interface. func (e *PlanReplayerExec) Next(ctx context.Context, req *chunk.Chunk) error { req.GrowAndReset(e.maxChunkSize) @@ -918,76 +992,3 @@ func (e *PlanReplayerLoadInfo) Update(data []byte) error { } return nil } - -type tableNamePair struct { - DBName string - TableName string - IsView bool -} - -type tableNameExtractor struct { - ctx context.Context - executor sqlexec.RestrictedSQLExecutor - is infoschema.InfoSchema - curDB model.CIStr - names map[tableNamePair]struct{} - cteNames map[string]struct{} - err error -} - -func (tne *tableNameExtractor) Enter(in ast.Node) (ast.Node, bool) { - if _, ok := in.(*ast.TableName); ok { - return in, true - } - return in, false -} - -func (tne *tableNameExtractor) Leave(in ast.Node) (ast.Node, bool) { - if tne.err != nil { - return in, true - } - if t, ok := in.(*ast.TableName); ok { - isView, err := tne.handleIsView(t) - if err != nil { - tne.err = err - return in, true - } - tp := tableNamePair{DBName: t.Schema.L, TableName: t.Name.L, IsView: isView} - if tp.DBName == "" { - tp.DBName = tne.curDB.L - } - if _, ok := tne.names[tp]; !ok { - tne.names[tp] = struct{}{} - } - } else if s, ok := in.(*ast.SelectStmt); ok { - if s.With != nil && len(s.With.CTEs) > 0 { - for _, cte := range s.With.CTEs { - tne.cteNames[cte.Name.L] = struct{}{} - } - } - } - return in, true -} - -func (tne *tableNameExtractor) handleIsView(t *ast.TableName) (bool, error) { - schema := t.Schema - if schema.L == "" { - schema = tne.curDB - } - table := t.Name - isView := tne.is.TableIsView(schema, table) - if !isView { - return false, nil - } - viewTbl, err := tne.is.TableByName(schema, table) - if err != nil { - return false, err - } - sql := viewTbl.Meta().View.SelectStmt - node, err := tne.executor.ParseWithParams(tne.ctx, sql) - if err != nil { - return false, err - } - node.Accept(tne) - return true, nil -} From 47028ed34b3ce3e1877867fdbb7d5a22c6598819 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 15 Sep 2022 18:42:29 +0800 Subject: [PATCH 07/11] add test Signed-off-by: yisaer --- executor/plan_replayer.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/executor/plan_replayer.go b/executor/plan_replayer.go index f573b221e713a..81f976870d987 100644 --- a/executor/plan_replayer.go +++ b/executor/plan_replayer.go @@ -584,8 +584,7 @@ func (e *PlanReplayerDumpInfo) DumpSQLsFromFile(ctx context.Context, b []byte) e } node, err := p.ParseOneStmt(s, "", "") if err != nil { - // TODO: add error info - return err + return fmt.Errorf("parse sql error, sql:%v, err:%v", s, err) } e.ExecStmts = append(e.ExecStmts, node) } From 4add2a2255db417968300494b5504d616d21d522 Mon Sep 17 00:00:00 2001 From: yisaer Date: Fri, 16 Sep 2022 11:40:33 +0800 Subject: [PATCH 08/11] address the comment Signed-off-by: yisaer --- executor/plan_replayer.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/executor/plan_replayer.go b/executor/plan_replayer.go index 81f976870d987..cfd0780f091bc 100644 --- a/executor/plan_replayer.go +++ b/executor/plan_replayer.go @@ -164,6 +164,7 @@ func (e *PlanReplayerExec) Next(ctx context.Context, req *chunk.Chunk) error { if err != nil { return err } + // As we can only read file from handleSpecialQuery, thus we store the file token in the session var and return nil here. e.endFlag = true e.ctx.GetSessionVars().LastPlanReplayerToken = e.DumpInfo.FileName return nil @@ -204,7 +205,6 @@ func (e *PlanReplayerExec) createFile(path string) error { } e.DumpInfo.File = zf e.DumpInfo.FileName = fileName - e.ctx.GetSessionVars().LastPlanReplayerToken = e.DumpInfo.FileName return nil } @@ -310,11 +310,7 @@ func (e *PlanReplayerDumpInfo) dump(ctx context.Context) (err error) { } // Dump explain - if err = dumpExplain(e.ctx, zw, e.ExecStmts, e.Analyze); err != nil { - return err - } - - return nil + return dumpExplain(e.ctx, zw, e.ExecStmts, e.Analyze) } func dumpConfig(zw *zip.Writer) error { @@ -560,9 +556,6 @@ func (e *PlanReplayerDumpInfo) extractTableNames(ctx context.Context, } func (e *PlanReplayerExec) prepare() error { - if len(e.DumpInfo.Path) == 0 { - return errors.New("plan replayer: file path is empty") - } val := e.ctx.Value(PlanReplayerDumpVarKey) if val != nil { e.ctx.SetValue(PlanReplayerDumpVarKey, nil) From 9d00da5482a7e135ede72d41f5a2e104120d2ac0 Mon Sep 17 00:00:00 2001 From: yisaer Date: Fri, 16 Sep 2022 11:58:27 +0800 Subject: [PATCH 09/11] address the comment Signed-off-by: yisaer --- executor/plan_replayer.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/executor/plan_replayer.go b/executor/plan_replayer.go index cfd0780f091bc..8232b171bcc30 100644 --- a/executor/plan_replayer.go +++ b/executor/plan_replayer.go @@ -699,17 +699,23 @@ type PlanReplayerLoadInfo struct { Ctx sessionctx.Context } -type planReplayerKeyType int +type planReplayerDumpVarKey int -func (k planReplayerKeyType) String() string { +func (k planReplayerDumpVarKey) String() string { + return "plan_replayer_dump_var" +} + +type planReplayerLoadKeyType int + +func (k planReplayerLoadKeyType) String() string { return "plan_replayer_load_var" } // PlanReplayerLoadVarKey is a variable key for plan replayer load. -const PlanReplayerLoadVarKey planReplayerKeyType = 0 +const PlanReplayerLoadVarKey planReplayerLoadKeyType = 0 // PlanReplayerDumpVarKey is a variable key for plan replayer dump. -const PlanReplayerDumpVarKey planReplayerKeyType = 1 +const PlanReplayerDumpVarKey planReplayerDumpVarKey = 1 // Next implements the Executor Next interface. func (e *PlanReplayerLoadExec) Next(ctx context.Context, req *chunk.Chunk) error { From da44e03814d10317af17bd63dbeccc74e2242b92 Mon Sep 17 00:00:00 2001 From: yisaer Date: Fri, 16 Sep 2022 12:29:18 +0800 Subject: [PATCH 10/11] address the comment Signed-off-by: yisaer address the comment Signed-off-by: yisaer revise Signed-off-by: yisaer revise Signed-off-by: yisaer --- executor/executor_test.go | 4 ++-- executor/plan_replayer.go | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 752bf09a4d66f..bcea3e48489a5 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -152,10 +152,10 @@ func TestPlanReplayer(t *testing.T) { tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int, b int, index idx_a(a))") tk.MustExec("alter table t set tiflash replica 1") + tk.MustExec("plan replayer dump explain './testdata/plan_replayer_test.txt'") + require.True(t, len(tk.Session().GetSessionVars().LastPlanReplayerToken) > 0) tk.MustQuery("plan replayer dump explain select * from t where a=10") tk.MustQuery("plan replayer dump explain select /*+ read_from_storage(tiflash[t]) */ * from t") - tk.MustExec("plan replayer dump explain './test/data/plan_replayer_test.txt'") - require.True(t, len(tk.Session().GetSessionVars().LastPlanReplayerToken) > 0) tk.MustExec("create table t1 (a int)") tk.MustExec("create table t2 (a int)") diff --git a/executor/plan_replayer.go b/executor/plan_replayer.go index 8232b171bcc30..33fdf5da4507c 100644 --- a/executor/plan_replayer.go +++ b/executor/plan_replayer.go @@ -34,7 +34,6 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/infoschema" - "github.com/pingcap/tidb/parser" "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx" @@ -164,9 +163,9 @@ func (e *PlanReplayerExec) Next(ctx context.Context, req *chunk.Chunk) error { if err != nil { return err } - // As we can only read file from handleSpecialQuery, thus we store the file token in the session var and return nil here. + // As we can only read file from handleSpecialQuery, thus we store the file token in the session var during `dump` + // and return nil here. e.endFlag = true - e.ctx.GetSessionVars().LastPlanReplayerToken = e.DumpInfo.FileName return nil } if e.DumpInfo.ExecStmts == nil { @@ -178,7 +177,6 @@ func (e *PlanReplayerExec) Next(ctx context.Context, req *chunk.Chunk) error { } req.AppendString(0, e.DumpInfo.FileName) e.endFlag = true - e.ctx.GetSessionVars().LastPlanReplayerToken = e.DumpInfo.FileName return nil } @@ -310,7 +308,12 @@ func (e *PlanReplayerDumpInfo) dump(ctx context.Context) (err error) { } // Dump explain - return dumpExplain(e.ctx, zw, e.ExecStmts, e.Analyze) + if err = dumpExplain(e.ctx, zw, e.ExecStmts, e.Analyze); err != nil { + return err + } + + e.ctx.GetSessionVars().LastPlanReplayerToken = e.FileName + return nil } func dumpConfig(zw *zip.Writer) error { @@ -568,14 +571,13 @@ func (e *PlanReplayerExec) prepare() error { // DumpSQLsFromFile dumps plan replayer results for sqls from file func (e *PlanReplayerDumpInfo) DumpSQLsFromFile(ctx context.Context, b []byte) error { sqls := strings.Split(string(b), ";") - p := parser.New() e.ExecStmts = make([]ast.StmtNode, 0) for _, sql := range sqls { s := strings.Trim(sql, "\n") if len(s) < 1 { continue } - node, err := p.ParseOneStmt(s, "", "") + node, err := e.ctx.(sqlexec.RestrictedSQLExecutor).ParseWithParams(ctx, s) if err != nil { return fmt.Errorf("parse sql error, sql:%v, err:%v", s, err) } @@ -699,9 +701,9 @@ type PlanReplayerLoadInfo struct { Ctx sessionctx.Context } -type planReplayerDumpVarKey int +type planReplayerDumpKeyType int -func (k planReplayerDumpVarKey) String() string { +func (k planReplayerDumpKeyType) String() string { return "plan_replayer_dump_var" } @@ -715,7 +717,7 @@ func (k planReplayerLoadKeyType) String() string { const PlanReplayerLoadVarKey planReplayerLoadKeyType = 0 // PlanReplayerDumpVarKey is a variable key for plan replayer dump. -const PlanReplayerDumpVarKey planReplayerDumpVarKey = 1 +const PlanReplayerDumpVarKey planReplayerDumpKeyType = 1 // Next implements the Executor Next interface. func (e *PlanReplayerLoadExec) Next(ctx context.Context, req *chunk.Chunk) error { From a788bd61466cce00e3923a3f33d8741121c0fe34 Mon Sep 17 00:00:00 2001 From: yisaer Date: Fri, 16 Sep 2022 13:20:14 +0800 Subject: [PATCH 11/11] fix test Signed-off-by: yisaer --- executor/executor_test.go | 2 -- executor/testdata/plan_replayer_test.txt | 1 - 2 files changed, 3 deletions(-) delete mode 100644 executor/testdata/plan_replayer_test.txt diff --git a/executor/executor_test.go b/executor/executor_test.go index bcea3e48489a5..e01eac6cf31c8 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -152,8 +152,6 @@ func TestPlanReplayer(t *testing.T) { tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int, b int, index idx_a(a))") tk.MustExec("alter table t set tiflash replica 1") - tk.MustExec("plan replayer dump explain './testdata/plan_replayer_test.txt'") - require.True(t, len(tk.Session().GetSessionVars().LastPlanReplayerToken) > 0) tk.MustQuery("plan replayer dump explain select * from t where a=10") tk.MustQuery("plan replayer dump explain select /*+ read_from_storage(tiflash[t]) */ * from t") diff --git a/executor/testdata/plan_replayer_test.txt b/executor/testdata/plan_replayer_test.txt deleted file mode 100644 index 0a9959f55ad3e..0000000000000 --- a/executor/testdata/plan_replayer_test.txt +++ /dev/null @@ -1 +0,0 @@ -select * from t;select * from t;