diff --git a/bindinfo/bind_test.go b/bindinfo/bind_test.go index 1e5544dce8376..6d7ed2f0e4392 100644 --- a/bindinfo/bind_test.go +++ b/bindinfo/bind_test.go @@ -24,6 +24,7 @@ import ( "time" . "github.com/pingcap/check" + "github.com/pingcap/failpoint" "github.com/pingcap/parser" "github.com/pingcap/parser/auth" "github.com/pingcap/parser/model" @@ -60,6 +61,7 @@ func TestT(t *testing.T) { } var _ = Suite(&testSuite{}) +var _ = SerialSuites(&testSerialSuite{}) type testSuite struct { cluster testutils.Cluster @@ -153,6 +155,44 @@ func (s *testSuite) cleanBindingEnv(tk *testkit.TestKit) { s.domain.BindHandle().Clear() } +type testSerialSuite struct { + cluster testutils.Cluster + store kv.Storage + domain *domain.Domain +} + +func (s *testSerialSuite) SetUpSuite(c *C) { + flag.Lookup("mockTikv") + useMockTikv := *mockTikv + if useMockTikv { + store, err := mockstore.NewMockStore( + mockstore.WithClusterInspector(func(c testutils.Cluster) { + mockstore.BootstrapWithSingleStore(c) + s.cluster = c + }), + ) + c.Assert(err, IsNil) + s.store = store + session.SetSchemaLease(0) + session.DisableStats4Test() + } + bindinfo.Lease = 0 + d, err := session.BootstrapSession(s.store) + c.Assert(err, IsNil) + d.SetStatsUpdating(true) + s.domain = d +} + +func (s *testSerialSuite) TearDownSuite(c *C) { + s.domain.Close() + s.store.Close() +} + +func (s *testSerialSuite) cleanBindingEnv(tk *testkit.TestKit) { + tk.MustExec("delete from mysql.bind_info where source != 'builtin'") + s.domain.BindHandle().Clear() +} + func normalizeWithDefaultDB(c *C, sql, db string) (string, string) { testParser := parser.New() stmt, err := testParser.ParseOneStmt(sql, "", "") @@ -1259,20 +1299,11 @@ func (s *testSuite) TestRuntimeHintsInEvolveTasks(c *C) { tk.MustExec("set @@tidb_evolve_plan_baselines=1") tk.MustExec("create table t(a int, b int, c int, index idx_a(a), index idx_b(b), index idx_c(c))") - // these runtime hints which don't be contained by the original binding should be ignored tk.MustExec("create global binding for select * from t where a >= 1 and b >= 1 and c = 0 using select * from t use index(idx_a) where a >= 1 and b >= 1 and c = 0") tk.MustQuery("select /*+ MAX_EXECUTION_TIME(5000) */* from t where a >= 4 and b >= 1 and c = 0") tk.MustExec("admin flush bindings") rows := tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 2) - c.Assert(rows[1][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` `idx_c`)*/ * FROM `test`.`t` WHERE `a` >= 4 AND `b` >= 1 AND `c` = 0") // MAX_EXECUTION_TIME is ignored - - s.cleanBindingEnv(tk) - tk.MustExec("create global binding for select * from t where a >= 1 and b >= 1 and c = 0 using select /*+ MAX_EXECUTION_TIME(5000) */* from t use index(idx_a) where a >= 1 and b >= 1 and c = 0") - tk.MustQuery("select /*+ MAX_EXECUTION_TIME(5000) */* from t where a >= 4 and b >= 1 and c = 0") - tk.MustExec("admin flush bindings") - rows = tk.MustQuery("show global bindings").Rows() - c.Assert(len(rows), Equals, 2) c.Assert(rows[1][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` `idx_c`), max_execution_time(5000)*/ * FROM `test`.`t` WHERE `a` >= 4 AND `b` >= 1 AND `c` = 0") } @@ -2046,13 +2077,13 @@ func (s *testSuite) TestIssue20417(c *C) { rows = tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 2) c.Assert(rows[1][0], Equals, "select * from `test` . `t` where `c` = ?") - c.Assert(rows[1][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` `idxc`), use_index(`t` `idxc`)*/ * FROM `test`.`t` WHERE `c` = 3924541") + c.Assert(rows[1][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` `idxc`)*/ * FROM `test`.`t` WHERE `c` = 3924541") c.Assert(rows[1][3], Equals, "pending verify") tk.MustExec("admin evolve bindings") rows = tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 2) c.Assert(rows[1][0], Equals, "select * from `test` . `t` where `c` = ?") - c.Assert(rows[1][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` `idxc`), use_index(`t` `idxc`)*/ * FROM `test`.`t` WHERE `c` = 3924541") + c.Assert(rows[1][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` `idxc`)*/ * FROM `test`.`t` WHERE `c` = 3924541") status := rows[1][3].(string) c.Assert(status == "using" || status == "rejected", IsTrue) tk.MustExec("set @@tidb_evolve_plan_baselines=0") @@ -2197,3 +2228,15 @@ func (s *testSuite) TestGCBindRecord(c *C) { tk.MustQuery("show global bindings").Check(testkit.Rows()) tk.MustQuery("select status from mysql.bind_info where original_sql = 'select * from `test` . `t` where `a` = ?'").Check(testkit.Rows()) } + +func (s *testSerialSuite) TestOptimizeOnlyOnce(c *C) { + tk := testkit.NewTestKit(c, s.store) + s.cleanBindingEnv(tk) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b int, index idxa(a))") + tk.MustExec("create global binding for select * from t using select * from t use index(idxa)") + c.Assert(failpoint.Enable("github.com/pingcap/tidb/planner/checkOptimizeCountOne", "return"), IsNil) + tk.MustQuery("select * from t").Check(testkit.Rows()) + c.Assert(failpoint.Disable("github.com/pingcap/tidb/planner/checkOptimizeCountOne"), IsNil) +} diff --git a/planner/optimize.go b/planner/optimize.go index 64afe1b329cb7..80225befe1b42 100644 --- a/planner/optimize.go +++ b/planner/optimize.go @@ -23,6 +23,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/tidb/bindinfo" @@ -91,16 +92,16 @@ func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is in } tableHints := hint.ExtractTableHintsFromStmtNode(node, sctx) - stmtHints, warns := handleStmtHints(tableHints) - sessVars.StmtCtx.StmtHints = stmtHints + originStmtHints, originStmtHintsOffs, warns := handleStmtHints(tableHints) + sessVars.StmtCtx.StmtHints = originStmtHints for _, warn := range warns { - sctx.GetSessionVars().StmtCtx.AppendWarning(warn) + sessVars.StmtCtx.AppendWarning(warn) } warns = warns[:0] - for name, val := range stmtHints.SetVars { + for name, val := range originStmtHints.SetVars { err := variable.SetStmtVar(sessVars, name, val) if err != nil { - sctx.GetSessionVars().StmtCtx.AppendWarning(err) + sessVars.StmtCtx.AppendWarning(err) } } @@ -119,115 +120,125 @@ func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is in return fp, fp.OutputNames(), nil } } - sctx.PrepareTSFuture(ctx) - bestPlan, names, _, err := optimize(ctx, sctx, node, is) - if err != nil { - return nil, nil, err - } - if !(sessVars.UsePlanBaselines || sessVars.EvolvePlanBaselines) { - return bestPlan, names, nil - } + useBinding := sessVars.UsePlanBaselines stmtNode, ok := node.(ast.StmtNode) if !ok { - return bestPlan, names, nil - } - bindRecord, scope, err := getBindRecord(sctx, stmtNode) - if err != nil { - return nil, nil, err - } - if bindRecord == nil { - return bestPlan, names, nil + useBinding = false } - if sctx.GetSessionVars().SelectLimit != math.MaxUint64 { - sctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("sql_select_limit is set, so plan binding is not activated")) - return bestPlan, names, nil + if useBinding && sessVars.SelectLimit != math.MaxUint64 { + sessVars.StmtCtx.AppendWarning(errors.New("sql_select_limit is set, ignore SQL bindings")) + useBinding = false } - err = setFoundInBinding(sctx, true) - if err != nil { - return nil, nil, err - } - bestPlanHint := plannercore.GenHintsFromPhysicalPlan(bestPlan) - if len(bindRecord.Bindings) > 0 { - orgBinding := bindRecord.Bindings[0] // the first is the original binding - for _, tbHint := range tableHints { // consider table hints which contained by the original binding - if orgBinding.Hint.ContainTableHint(tbHint.HintName.String()) { - bestPlanHint = append(bestPlanHint, tbHint) + var ( + bindRecord *bindinfo.BindRecord + scope string + err error + ) + if useBinding { + bindRecord, scope, err = getBindRecord(sctx, stmtNode) + if err != nil || bindRecord == nil || len(bindRecord.Bindings) == 0 { + useBinding = false + } + } + + var names types.NameSlice + var bestPlan, bestPlanFromBind plannercore.Plan + if useBinding { + minCost := math.MaxFloat64 + var ( + bindStmtHints stmtctx.StmtHints + chosenBinding bindinfo.Binding + ) + originHints := hint.CollectHint(stmtNode) + // bindRecord must be not nil when coming here, try to find the best binding. + for _, binding := range bindRecord.Bindings { + if binding.Status != bindinfo.Using { + continue + } + metrics.BindUsageCounter.WithLabelValues(scope).Inc() + hint.BindHint(stmtNode, binding.Hint) + curStmtHints, _, curWarns := handleStmtHints(binding.Hint.GetFirstTableHints()) + sessVars.StmtCtx.StmtHints = curStmtHints + plan, curNames, cost, err := optimize(ctx, sctx, node, is) + if err != nil { + binding.Status = bindinfo.Invalid + handleInvalidBindRecord(ctx, sctx, scope, bindinfo.BindRecord{ + OriginalSQL: bindRecord.OriginalSQL, + Db: bindRecord.Db, + Bindings: []bindinfo.Binding{binding}, + }) + continue + } + if cost < minCost { + bindStmtHints, warns, minCost, names, bestPlanFromBind, chosenBinding = curStmtHints, curWarns, cost, curNames, plan, binding } } - } - bestPlanHintStr := hint.RestoreOptimizerHints(bestPlanHint) - - defer func() { - sessVars.StmtCtx.StmtHints = stmtHints - for _, warn := range warns { - sctx.GetSessionVars().StmtCtx.AppendWarning(warn) - } - }() - binding := bindRecord.FindBinding(bestPlanHintStr) - // If the best bestPlan is in baselines, just use it. - if binding != nil && binding.Status == bindinfo.Using { - if sctx.GetSessionVars().UsePlanBaselines { - stmtHints, warns = handleStmtHints(binding.Hint.GetFirstTableHints()) + if bestPlanFromBind == nil { + sessVars.StmtCtx.AppendWarning(errors.New("no plan generated from bindings")) + } else { + bestPlan = bestPlanFromBind + sessVars.StmtCtx.StmtHints = bindStmtHints + for _, warn := range warns { + sessVars.StmtCtx.AppendWarning(warn) + } + if err := setFoundInBinding(sctx, true); err != nil { + logutil.BgLogger().Warn("set tidb_found_in_binding failed", zap.Error(err)) + } if _, ok := stmtNode.(*ast.ExplainStmt); ok { - sctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("Using the bindSQL: %v", binding.BindSQL)) + sessVars.StmtCtx.AppendWarning(errors.Errorf("Using the bindSQL: %v", chosenBinding.BindSQL)) } } - return bestPlan, names, nil + // Restore the hint to avoid changing the stmt node. + hint.BindHint(stmtNode, originHints) } - bestCostAmongHints := math.MaxFloat64 - var ( - bestPlanAmongHints plannercore.Plan - bestPlanBindSQL string - ) - originHints := hint.CollectHint(stmtNode) - // Try to find the best binding. - for _, binding := range bindRecord.Bindings { - if binding.Status != bindinfo.Using { - continue - } - metrics.BindUsageCounter.WithLabelValues(scope).Inc() - hint.BindHint(stmtNode, binding.Hint) - curStmtHints, curWarns := handleStmtHints(binding.Hint.GetFirstTableHints()) - sctx.GetSessionVars().StmtCtx.StmtHints = curStmtHints - plan, _, cost, err := optimize(ctx, sctx, node, is) + // No plan found from the bindings, or the bindings are ignored. + if bestPlan == nil { + sessVars.StmtCtx.StmtHints = originStmtHints + bestPlan, names, _, err = optimize(ctx, sctx, node, is) if err != nil { - binding.Status = bindinfo.Invalid - handleInvalidBindRecord(ctx, sctx, scope, bindinfo.BindRecord{ - OriginalSQL: bindRecord.OriginalSQL, - Db: bindRecord.Db, - Bindings: []bindinfo.Binding{binding}, - }) - continue - } - if cost < bestCostAmongHints { - if sctx.GetSessionVars().UsePlanBaselines { - stmtHints, warns = curStmtHints, curWarns - } - bestCostAmongHints = cost - bestPlanAmongHints = plan - bestPlanBindSQL = binding.BindSQL + return nil, nil, err } } - if _, ok := stmtNode.(*ast.ExplainStmt); ok && bestPlanBindSQL != "" { - sctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("Using the bindSQL: %v", bestPlanBindSQL)) - } - // 1. If it is a select query. - // 2. If there is already a evolution task, we do not need to handle it again. - // 3. If the origin binding contain `read_from_storage` hint, we should ignore the evolve task. - // 4. If the best plan contain TiFlash hint, we should ignore the evolve task. - if _, ok := stmtNode.(*ast.SelectStmt); ok && - sctx.GetSessionVars().EvolvePlanBaselines && binding == nil && - !originHints.ContainTableHint(plannercore.HintReadFromStorage) && - !bindRecord.Bindings[0].Hint.ContainTableHint(plannercore.HintReadFromStorage) { - handleEvolveTasks(ctx, sctx, bindRecord, stmtNode, bestPlanHintStr) - } - // Restore the hint to avoid changing the stmt node. - hint.BindHint(stmtNode, originHints) - if sctx.GetSessionVars().UsePlanBaselines && bestPlanAmongHints != nil { - return bestPlanAmongHints, names, nil + + // Add a baseline evolution task if: + // 1. the returned plan is from bindings; + // 2. the query is a select statement; + // 3. the original binding contains no read_from_storage hint; + // 4. the plan when ignoring bindings contains no tiflash hint; + // 5. the pending verified binding has not been added already; + savedStmtHints := sessVars.StmtCtx.StmtHints + defer func() { + sessVars.StmtCtx.StmtHints = savedStmtHints + }() + if sessVars.EvolvePlanBaselines && bestPlanFromBind != nil { + // Check bestPlanFromBind firstly to avoid nil stmtNode. + if _, ok := stmtNode.(*ast.SelectStmt); ok && !bindRecord.Bindings[0].Hint.ContainTableHint(plannercore.HintReadFromStorage) { + sessVars.StmtCtx.StmtHints = originStmtHints + defPlan, _, _, err := optimize(ctx, sctx, node, is) + if err != nil { + // Ignore this evolution task. + return bestPlan, names, nil + } + defPlanHints := plannercore.GenHintsFromPhysicalPlan(defPlan) + for _, hint := range defPlanHints { + if hint.HintName.String() == plannercore.HintReadFromStorage { + return bestPlan, names, nil + } + } + // The hints generated from the plan do not contain the statement hints of the query, add them back. + for _, off := range originStmtHintsOffs { + defPlanHints = append(defPlanHints, tableHints[off]) + } + defPlanHintsStr := hint.RestoreOptimizerHints(defPlanHints) + binding := bindRecord.FindBinding(defPlanHintsStr) + if binding == nil { + handleEvolveTasks(ctx, sctx, bindRecord, stmtNode, defPlanHintsStr) + } + } } + return bestPlan, names, nil } @@ -277,7 +288,16 @@ var planBuilderPool = sync.Pool{ }, } +// optimizeCnt is a global variable only used for test. +var optimizeCnt int + func optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is infoschema.InfoSchema) (plannercore.Plan, types.NameSlice, float64, error) { + failpoint.Inject("checkOptimizeCountOne", func() { + optimizeCnt++ + if optimizeCnt > 1 { + failpoint.Return(nil, nil, 0, errors.New("gofail wrong optimizerCnt error")) + } + }) // build logical plan sctx.GetSessionVars().PlanID = 0 sctx.GetSessionVars().PlanColumnID = 0 @@ -503,31 +523,35 @@ func OptimizeExecStmt(ctx context.Context, sctx sessionctx.Context, return nil, err } -func handleStmtHints(hints []*ast.TableOptimizerHint) (stmtHints stmtctx.StmtHints, warns []error) { +func handleStmtHints(hints []*ast.TableOptimizerHint) (stmtHints stmtctx.StmtHints, offs []int, warns []error) { if len(hints) == 0 { return } - var memoryQuotaHint, useToJAHint, useCascadesHint, maxExecutionTime, forceNthPlan *ast.TableOptimizerHint + hintOffs := make(map[string]int, len(hints)) + var forceNthPlan *ast.TableOptimizerHint var memoryQuotaHintCnt, useToJAHintCnt, useCascadesHintCnt, noIndexMergeHintCnt, readReplicaHintCnt, maxExecutionTimeCnt, forceNthPlanCnt int setVars := make(map[string]string) - for _, hint := range hints { + setVarsOffs := make([]int, 0, len(hints)) + for i, hint := range hints { switch hint.HintName.L { case "memory_quota": - memoryQuotaHint = hint + hintOffs[hint.HintName.L] = i memoryQuotaHintCnt++ case "use_toja": - useToJAHint = hint + hintOffs[hint.HintName.L] = i useToJAHintCnt++ case "use_cascades": - useCascadesHint = hint + hintOffs[hint.HintName.L] = i useCascadesHintCnt++ case "no_index_merge": + hintOffs[hint.HintName.L] = i noIndexMergeHintCnt++ case "read_consistent_replica": + hintOffs[hint.HintName.L] = i readReplicaHintCnt++ case "max_execution_time": + hintOffs[hint.HintName.L] = i maxExecutionTimeCnt++ - maxExecutionTime = hint case "nth_plan": forceNthPlanCnt++ forceNthPlan = hint @@ -551,18 +575,21 @@ func handleStmtHints(hints []*ast.TableOptimizerHint) (stmtHints stmtctx.StmtHin continue } setVars[setVarHint.VarName] = setVarHint.Value + setVarsOffs = append(setVarsOffs, i) } } stmtHints.SetVars = setVars // Handle MEMORY_QUOTA if memoryQuotaHintCnt != 0 { + memoryQuotaHint := hints[hintOffs["memory_quota"]] if memoryQuotaHintCnt > 1 { - warn := errors.Errorf("MEMORY_QUOTA() s defined more than once, only the last definition takes effect: MEMORY_QUOTA(%v)", memoryQuotaHint.HintData.(int64)) + warn := errors.Errorf("MEMORY_QUOTA() is defined more than once, only the last definition takes effect: MEMORY_QUOTA(%v)", memoryQuotaHint.HintData.(int64)) warns = append(warns, warn) } // Executor use MemoryQuota <= 0 to indicate no memory limit, here use < 0 to handle hint syntax error. if memoryQuota := memoryQuotaHint.HintData.(int64); memoryQuota < 0 { + delete(hintOffs, "memory_quota") warn := errors.New("The use of MEMORY_QUOTA hint is invalid, valid usage: MEMORY_QUOTA(10 MB) or MEMORY_QUOTA(10 GB)") warns = append(warns, warn) } else { @@ -576,6 +603,7 @@ func handleStmtHints(hints []*ast.TableOptimizerHint) (stmtHints stmtctx.StmtHin } // Handle USE_TOJA if useToJAHintCnt != 0 { + useToJAHint := hints[hintOffs["use_toja"]] if useToJAHintCnt > 1 { warn := errors.Errorf("USE_TOJA() is defined more than once, only the last definition takes effect: USE_TOJA(%v)", useToJAHint.HintData.(bool)) warns = append(warns, warn) @@ -585,6 +613,7 @@ func handleStmtHints(hints []*ast.TableOptimizerHint) (stmtHints stmtctx.StmtHin } // Handle USE_CASCADES if useCascadesHintCnt != 0 { + useCascadesHint := hints[hintOffs["use_cascades"]] if useCascadesHintCnt > 1 { warn := errors.Errorf("USE_CASCADES() is defined more than once, only the last definition takes effect: USE_CASCADES(%v)", useCascadesHint.HintData.(bool)) warns = append(warns, warn) @@ -611,6 +640,7 @@ func handleStmtHints(hints []*ast.TableOptimizerHint) (stmtHints stmtctx.StmtHin } // Handle MAX_EXECUTION_TIME if maxExecutionTimeCnt != 0 { + maxExecutionTime := hints[hintOffs["max_execution_time"]] if maxExecutionTimeCnt > 1 { warn := errors.Errorf("MAX_EXECUTION_TIME() is defined more than once, only the last definition takes effect: MAX_EXECUTION_TIME(%v)", maxExecutionTime.HintData.(uint64)) warns = append(warns, warn) @@ -633,6 +663,10 @@ func handleStmtHints(hints []*ast.TableOptimizerHint) (stmtHints stmtctx.StmtHin } else { stmtHints.ForceNthPlan = -1 } + for _, off := range hintOffs { + offs = append(offs, off) + } + offs = append(offs, setVarsOffs...) return }