Skip to content

Commit

Permalink
planner: add more test cases for fuzzy binding (#50106)
Browse files Browse the repository at this point in the history
ref #48875
  • Loading branch information
qw4990 authored Jan 5, 2024
1 parent bf166d9 commit 5fe7940
Show file tree
Hide file tree
Showing 11 changed files with 154 additions and 175 deletions.
1 change: 1 addition & 0 deletions pkg/bindinfo/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ go_test(
"//pkg/testkit",
"//pkg/testkit/testsetup",
"//pkg/types",
"//pkg/util",
"//pkg/util/hack",
"//pkg/util/parser",
"//pkg/util/stmtsummary",
Expand Down
28 changes: 20 additions & 8 deletions pkg/bindinfo/binding_match.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,24 @@ func eraseLastSemicolon(stmt ast.StmtNode) {
}
}

// NormalizeStmtForBinding normalizes a statement for binding.
// Schema names will be completed automatically: `select * from t` --> `select * from db . t`.
func NormalizeStmtForBinding(stmtNode ast.StmtNode, specifiedDB string) (normalizedStmt, exactSQLDigest string) {
return normalizeStmt(stmtNode, specifiedDB, false)
}

// NormalizeStmtForFuzzyBinding normalizes a statement for fuzzy matching.
// Schema names will be eliminated automatically: `select * from db . t` --> `select * from t`.
func NormalizeStmtForFuzzyBinding(stmtNode ast.StmtNode) (normalizedStmt, fuzzySQLDigest string) {
return normalizeStmt(stmtNode, "", true)
}

// NormalizeStmtForBinding normalizes a statement for binding.
// This function skips Explain automatically, and literals in in-lists will be normalized as '...'.
// For normal bindings, DB name will be completed automatically:
//
// e.g. `select * from t where a in (1, 2, 3)` --> `select * from test.t where a in (...)`
func normalizeStmt(stmtNode ast.StmtNode, specifiedDB string, fuzzy bool) (stmt ast.StmtNode, normalizedStmt, sqlDigest string, err error) {
func normalizeStmt(stmtNode ast.StmtNode, specifiedDB string, fuzzy bool) (normalizedStmt, sqlDigest string) {
normalize := func(n ast.StmtNode) (normalizedStmt, sqlDigest string) {
eraseLastSemicolon(n)
var digest *parser.Digest
Expand All @@ -99,12 +111,12 @@ func normalizeStmt(stmtNode ast.StmtNode, specifiedDB string, fuzzy bool) (stmt
// The difference between them is whether len(x.Text()) is empty. They cannot be distinguished by stmt.restore.
// For these cases, we need return "" as normalize SQL and hash.
if len(x.Text()) == 0 {
return x.Stmt, "", "", nil
return "", ""
}
switch x.Stmt.(type) {
case *ast.SelectStmt, *ast.DeleteStmt, *ast.UpdateStmt, *ast.InsertStmt:
normalizeSQL, digest := normalize(x.Stmt)
return x.Stmt, normalizeSQL, digest, nil
return normalizeSQL, digest
case *ast.SetOprStmt:
normalizeExplainSQL, _ := normalize(x)

Expand All @@ -116,11 +128,11 @@ func normalizeStmt(stmtNode ast.StmtNode, specifiedDB string, fuzzy bool) (stmt
// If the SQL is `EXPLAIN ((VALUES ROW ()) ORDER BY 1);`, the idx will be -1.
if idx == -1 {
hash := parser.DigestNormalized(normalizeExplainSQL)
return x.Stmt, normalizeExplainSQL, hash.String(), nil
return normalizeExplainSQL, hash.String()
}
normalizeSQL := normalizeExplainSQL[idx:]
hash := parser.DigestNormalized(normalizeSQL)
return x.Stmt, normalizeSQL, hash.String(), nil
return normalizeSQL, hash.String()
}
case *ast.SelectStmt, *ast.SetOprStmt, *ast.DeleteStmt, *ast.UpdateStmt, *ast.InsertStmt:
// This function is only used to find bind record.
Expand All @@ -129,12 +141,12 @@ func normalizeStmt(stmtNode ast.StmtNode, specifiedDB string, fuzzy bool) (stmt
// The difference between them is whether len(x.Text()) is empty. They cannot be distinguished by stmt.restore.
// For these cases, we need return "" as normalize SQL and hash.
if len(x.Text()) == 0 {
return x, "", "", nil
return "", ""
}
normalizedSQL, digest := normalize(x)
return x, normalizedSQL, digest, nil
return normalizedSQL, digest
}
return nil, "", "", nil
return "", ""
}

func fuzzyMatchBindingTableName(currentDB string, stmtTableNames, bindingTableNames []*ast.TableName) (numWildcards int, matched bool) {
Expand Down
174 changes: 122 additions & 52 deletions pkg/bindinfo/fuzzy_binding_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,15 @@ package bindinfo_test

import (
"fmt"
"strings"
"testing"
"time"

"github.com/pingcap/tidb/pkg/bindinfo"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -108,40 +110,47 @@ func TestFuzzyDuplicatedBinding(t *testing.T) {
[][]interface{}{{"select * from `*` . `t`", "SELECT /*+ use_index(`t` `b`)*/ * FROM `*`.`t`", "", "enabled", "manual", "a17da0a38af0f1d75229c5cd064d5222a610c5e5ef59436be5da1564c16f1013"}})
}

func TestUniversalBindingPriority(t *testing.T) {
t.Skip("skip it temporarily")
func TestFuzzyBindingPriority(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)

tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=1`)
tk.MustExec(`use test`)
tk.MustExec(`create table t (a int, b int, c int, d int, e int, key(a), key(b), key(c), key(d), key(e))`)

tk.MustExec(`create global universal binding using select /*+ use_index(t, a) */ * from t`)
tk.MustUseIndex(`select * from t`, "a")
tk.MustQuery(`select @@last_plan_from_binding`).Check(testkit.Rows("1"))

// global normal > global universal
tk.MustExec(`create global binding using select /*+ use_index(t, b) */ * from t`)
tk.MustUseIndex(`select * from t`, "b")
tk.MustQuery(`select @@last_plan_from_binding`).Check(testkit.Rows("1"))

// session universal > global normal
tk.MustExec(`create session universal binding using select /*+ use_index(t, c) */ * from t`)
tk.MustUseIndex(`select * from t`, "c")
tk.MustQuery(`select @@last_plan_from_binding`).Check(testkit.Rows("1"))

// global normal takes effect again if disable universal bindings
tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=0`)
tk.MustExec(`create global binding using select /*+ use_index(t, b) */ * from t`)
tk.MustUseIndex(`select * from t`, "b")
tk.MustQuery(`select @@last_plan_from_binding`).Check(testkit.Rows("1"))
tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=1`)

// session normal > session universal
tk.MustExec(`create session binding using select /*+ use_index(t, d) */ * from t`)
tk.MustUseIndex(`select * from t`, "d")
tk.MustQuery(`select @@last_plan_from_binding`).Check(testkit.Rows("1"))
tk.MustExec(`create table t1 (a int)`)
tk.MustExec(`create table t2 (a int)`)
tk.MustExec(`create table t3 (a int)`)
tk.MustExec(`create table t4 (a int)`)
tk.MustExec(`create table t5 (a int)`)

// The less wildcard number, the higher priority.
tk.MustExec(`create global binding using select /*+ leading(t1, t2, t3, t4, t5) */ * from *.t1, *.t2, *.t3, *.t4, *.t5`)
tk.MustExec(`explain format='verbose' select * from t1, t2, t3, t4, t5`)
tk.MustQuery(`show warnings`).Check(testkit.Rows("Note 1105 Using the bindSQL: SELECT /*+ leading(`t1`, `t2`, `t3`, `t4`, `t5`)*/ * FROM ((((`*`.`t1`) JOIN `*`.`t2`) JOIN `*`.`t3`) JOIN `*`.`t4`) JOIN `*`.`t5`"))

tk.MustExec(`create global binding using select /*+ leading(t1, t2, t3, t4, t5) */ * from *.t1, *.t2, *.t3, *.t4, t5`)
tk.MustExec(`explain format='verbose' select * from t1, t2, t3, t4, t5`)
tk.MustQuery(`show warnings`).Check(testkit.Rows("Note 1105 Using the bindSQL: SELECT /*+ leading(`t1`, `t2`, `t3`, `t4`, `t5`)*/ * FROM ((((`*`.`t1`) JOIN `*`.`t2`) JOIN `*`.`t3`) JOIN `*`.`t4`) JOIN `test`.`t5`"))

tk.MustExec(`create global binding using select /*+ leading(t1, t2, t3, t4, t5) */ * from *.t1, *.t2, *.t3, t4, t5`)
tk.MustExec(`explain format='verbose' select * from t1, t2, t3, t4, t5`)
tk.MustQuery(`show warnings`).Check(testkit.Rows("Note 1105 Using the bindSQL: SELECT /*+ leading(`t1`, `t2`, `t3`, `t4`, `t5`)*/ * FROM ((((`*`.`t1`) JOIN `*`.`t2`) JOIN `*`.`t3`) JOIN `test`.`t4`) JOIN `test`.`t5`"))

tk.MustExec(`create global binding using select /*+ leading(t1, t2, t3, t4, t5) */ * from *.t1, *.t2, t3, t4, t5`)
tk.MustExec(`explain format='verbose' select * from t1, t2, t3, t4, t5`)
tk.MustQuery(`show warnings`).Check(testkit.Rows("Note 1105 Using the bindSQL: SELECT /*+ leading(`t1`, `t2`, `t3`, `t4`, `t5`)*/ * FROM ((((`*`.`t1`) JOIN `*`.`t2`) JOIN `test`.`t3`) JOIN `test`.`t4`) JOIN `test`.`t5`"))

tk.MustExec(`create global binding using select /*+ leading(t1, t2, t3, t4, t5) */ * from *.t1, t2, t3, t4, t5`)
tk.MustExec(`explain format='verbose' select * from t1, t2, t3, t4, t5`)
tk.MustQuery(`show warnings`).Check(testkit.Rows("Note 1105 Using the bindSQL: SELECT /*+ leading(`t1`, `t2`, `t3`, `t4`, `t5`)*/ * FROM ((((`*`.`t1`) JOIN `test`.`t2`) JOIN `test`.`t3`) JOIN `test`.`t4`) JOIN `test`.`t5`"))

tk.MustExec(`create global binding using select /*+ leading(t1, t2, t3, t4, t5) */ * from t1, t2, t3, t4, t5`)
tk.MustExec(`explain format='verbose' select * from t1, t2, t3, t4, t5`)
tk.MustQuery(`show warnings`).Check(testkit.Rows("Note 1105 Using the bindSQL: SELECT /*+ leading(`t1`, `t2`, `t3`, `t4`, `t5`)*/ * FROM ((((`test`.`t1`) JOIN `test`.`t2`) JOIN `test`.`t3`) JOIN `test`.`t4`) JOIN `test`.`t5`"))

// Session binding's priority is higher than global binding's.
tk.MustExec(`create session binding using select /*+ leading(t1, t2, t3, t4, t5) */ * from *.t1, *.t2, *.t3, *.t4, *.t5`)
tk.MustExec(`explain format='verbose' select * from t1, t2, t3, t4, t5`)
tk.MustQuery(`show warnings`).Check(testkit.Rows("Note 1105 Using the bindSQL: SELECT /*+ leading(`t1`, `t2`, `t3`, `t4`, `t5`)*/ * FROM ((((`*`.`t1`) JOIN `*`.`t2`) JOIN `*`.`t3`) JOIN `*`.`t4`) JOIN `*`.`t5`"))
}

func TestCreateUpdateFuzzyBinding(t *testing.T) {
Expand Down Expand Up @@ -241,29 +250,6 @@ func TestFuzzyBindingSetVar(t *testing.T) {
tk.MustQuery(`select @@last_plan_from_binding`).Check(testkit.Rows("1"))
}

func TestUniversalBindingDBInHints(t *testing.T) {
t.Skip("skip it temporarily")
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec(`use test`)
tk.MustExec(`create table t1 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`)
tk.MustExec(`create table t2 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`)
tk.MustExec(`create table t3 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`)

tk.MustExec(`create universal binding using select /*+ use_index(test.t, a) */ * from t`)
tk.MustExec(`create universal binding using select /*+ leading(t1, test.t2) */ * from t1, t2 where t1.a=t2.a`)
tk.MustExec(`create universal binding using select /*+ leading(test.t1, test.t2, test.t3) */ * from t1, t2 where t1.a=t2.a and t2.b=t3.b`)

// use_index(test.t, a) --> use_index(t, a)
// leading(t1, test.t2) --> leading(t1, t2)
// leading(test.t1, test.t2, test.t3) --> leading(t1, t2, t3)
rs := showBinding(tk, "show bindings")
require.Equal(t, len(rs), 3)
require.Equal(t, rs[0][1], "SELECT /*+ leading(`t1`, `t2`)*/ * FROM (`t1`) JOIN `t2` WHERE `t1`.`a` = `t2`.`a`")
require.Equal(t, rs[1][1], "SELECT /*+ leading(`t1`, `t2`, `t3`)*/ * FROM (`t1`) JOIN `t2` WHERE `t1`.`a` = `t2`.`a` AND `t2`.`b` = `t3`.`b`")
require.Equal(t, rs[2][1], "SELECT /*+ use_index(`t` `a`)*/ * FROM `t`")
}

func TestFuzzyBindingGC(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
Expand All @@ -286,6 +272,90 @@ func TestFuzzyBindingGC(t *testing.T) {
tk.MustQuery(`select bind_sql, status from mysql.bind_info where source != 'builtin'`).Check(testkit.Rows()) // empty after GC
}

func TestFuzzyBindingInList(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec(`create database test1`)
tk.MustExec(`use test1`)
tk.MustExec(`create table t1 (a int)`)
tk.MustExec(`create table t2 (a int)`)
tk.MustExec(`create database test2`)
tk.MustExec(`use test2`)
tk.MustExec(`create table t1 (a int)`)
tk.MustExec(`create table t2 (a int)`)

tk.MustExec(`use test`)
tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=1`)
tk.MustExec(`create global binding using select * from *.t1 where a in (1,2,3)`)
tk.MustExec(`explain format='verbose' select * from test1.t1 where a in (1)`)
tk.MustQuery(`show warnings`).Check(testkit.Rows("Note 1105 Using the bindSQL: SELECT * FROM `*`.`t1` WHERE `a` IN (1,2,3)"))
tk.MustExec(`explain format='verbose' select * from test2.t1 where a in (1,2,3,4,5)`)
tk.MustQuery(`show warnings`).Check(testkit.Rows("Note 1105 Using the bindSQL: SELECT * FROM `*`.`t1` WHERE `a` IN (1,2,3)"))
tk.MustExec(`use test1`)
tk.MustExec(`explain format='verbose' select * from t1 where a in (1)`)
tk.MustQuery(`show warnings`).Check(testkit.Rows("Note 1105 Using the bindSQL: SELECT * FROM `*`.`t1` WHERE `a` IN (1,2,3)"))

tk.MustExec(`create global binding using select * from *.t1, *.t2 where t1.a in (1) and t2.a in (2)`)
for _, currentDB := range []string{"test1", "test2"} {
for _, t1DB := range []string{"", "test1.", "test2."} {
for _, t2DB := range []string{"", "test1.", "test2."} {
for _, t1Cond := range []string{"(1)", "(1,2,3)", "(1,1,1,1,1,1,2,2,2)"} {
for _, t2Cond := range []string{"(1)", "(1,2,3)", "(1,1,1,1,1,1,2,2,2)"} {
tk.MustExec(`use ` + currentDB)
sql := fmt.Sprintf(`explain format='verbose' select * from %st1, %st2 where t1.a in %s and t2.a in %s`, t1DB, t2DB, t1Cond, t2Cond)
tk.MustExec(sql)
tk.MustQuery(`show warnings`).Check(testkit.Rows("Note 1105 Using the bindSQL: SELECT * FROM (`*`.`t1`) JOIN `*`.`t2` WHERE `t1`.`a` IN (1) AND `t2`.`a` IN (2)"))
}
}
}
}
}
}

func TestFuzzyBindingPlanCache(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec(`use test`)
tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=1`)
tk.MustExec(`create table t (a int, b int, c int, d int, e int, key(a), key(b), key(c), key(d))`)

hasPlan := func(operator, accessInfo string) {
tkProcess := tk.Session().ShowProcess()
ps := []*util.ProcessInfo{tkProcess}
tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
rows := tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).Rows()
flag := false
for _, row := range rows {
op := row[0].(string)
info := row[4].(string)
if strings.Contains(op, operator) && strings.Contains(info, accessInfo) {
flag = true
break
}
}
require.Equal(t, flag, true)
}

tk.MustExec(`prepare stmt from 'select * from t where e > ?'`)
tk.MustExec(`set @v=0`)
tk.MustExec(`execute stmt using @v`)
hasPlan("TableFullScan", "")

tk.MustExec(`create database test2`)
tk.MustExec(`use test2`)
tk.MustExec(`create global binding using select /*+ use_index(t, a) */ * from *.t where e > 1`)
tk.MustExec(`execute stmt using @v`)
hasPlan("IndexFullScan", "index:a(a)")

tk.MustExec(`create global binding using select /*+ use_index(t, b) */ * from *.t where e > 1`)
tk.MustExec(`execute stmt using @v`)
hasPlan("IndexFullScan", "index:b(b)")

tk.MustExec(`create global binding using select /*+ use_index(t, c) */ * from *.t where e > 1`)
tk.MustExec(`execute stmt using @v`)
hasPlan("IndexFullScan", "index:c(c)")
}

func TestFuzzyBindingHints(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
Expand Down
11 changes: 3 additions & 8 deletions pkg/bindinfo/global_handle.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,9 +189,8 @@ func buildFuzzyDigestMap(bindRecords []*BindRecord) map[string][]string {
p = parser.New()
continue
}
sqlWithoutDB := utilparser.RestoreWithoutDB(stmt)
_, fuzzyDigest := parser.NormalizeDigestForBinding(sqlWithoutDB)
m[fuzzyDigest.String()] = append(m[fuzzyDigest.String()], binding.SQLDigest)
_, fuzzyDigest := NormalizeStmtForFuzzyBinding(stmt)
m[fuzzyDigest] = append(m[fuzzyDigest], binding.SQLDigest)
}
}
return m
Expand Down Expand Up @@ -531,11 +530,7 @@ func (h *globalBindingHandle) MatchGlobalBinding(sctx sessionctx.Context, stmt a
return nil, nil
}

_, _, fuzzDigest, err := normalizeStmt(stmt, sctx.GetSessionVars().CurrentDB, true)
if err != nil {
return nil, err
}

_, fuzzDigest := NormalizeStmtForFuzzyBinding(stmt)
tableNames := CollectTableNames(stmt)
var bestBinding *BindRecord
leastWildcards := len(tableNames) + 1
Expand Down
11 changes: 2 additions & 9 deletions pkg/bindinfo/session_handle.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,11 +109,7 @@ func (h *sessionBindingHandle) MatchSessionBinding(sctx sessionctx.Context, stmt
if h.ch.Size() == 0 {
return nil, nil
}

_, _, fuzzDigest, err := normalizeStmt(stmt, sctx.GetSessionVars().CurrentDB, true)
if err != nil {
return nil, err
}
_, fuzzDigest := NormalizeStmtForFuzzyBinding(stmt)

// The current implementation is simplistic, but session binding is only for test purpose, so
// there shouldn't be many session bindings, and to keep it simple, this implementation is acceptable.
Expand All @@ -127,10 +123,7 @@ func (h *sessionBindingHandle) MatchSessionBinding(sctx sessionctx.Context, stmt
if err != nil {
return nil, err
}
_, _, bindingFuzzyDigest, err := normalizeStmt(bindingStmt, sctx.GetSessionVars().CurrentDB, true)
if err != nil {
return nil, err
}
_, bindingFuzzyDigest := NormalizeStmtForFuzzyBinding(bindingStmt)
if bindingFuzzyDigest != fuzzDigest {
continue
}
Expand Down
1 change: 0 additions & 1 deletion pkg/executor/prepared_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1086,7 +1086,6 @@ func TestPreparePC4Binding(t *testing.T) {

tk.MustExec("prepare stmt from \"select * from t\"")
require.Equal(t, 1, len(tk.Session().GetSessionVars().PreparedStmts))
require.Equal(t, "select * from `test` . `t`", tk.Session().GetSessionVars().PreparedStmts[1].(*plannercore.PlanCacheStmt).NormalizedSQL4PC)

tk.MustQuery("execute stmt")
tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("0"))
Expand Down
2 changes: 0 additions & 2 deletions pkg/planner/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ go_library(
"//pkg/infoschema",
"//pkg/kv",
"//pkg/metrics",
"//pkg/parser",
"//pkg/parser/ast",
"//pkg/planner/cascades",
"//pkg/planner/core",
Expand All @@ -25,7 +24,6 @@ go_library(
"//pkg/util/hint",
"//pkg/util/intest",
"//pkg/util/logutil",
"//pkg/util/parser",
"//pkg/util/topsql",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
Expand Down
3 changes: 2 additions & 1 deletion pkg/planner/core/plan_cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -794,12 +794,13 @@ func tryCachePointPlan(_ context.Context, sctx sessionctx.Context,
func GetBindSQL4PlanCache(sctx sessionctx.Context, stmt *PlanCacheStmt) (string, bool) {
useBinding := sctx.GetSessionVars().UsePlanBaselines
ignore := false
if !useBinding || stmt.PreparedAst.Stmt == nil || stmt.NormalizedSQL4PC == "" || stmt.SQLDigest4PC == "" {
if !useBinding || stmt.PreparedAst.Stmt == nil {
return "", ignore
}
if sctx.Value(bindinfo.SessionBindInfoKeyType) == nil {
return "", ignore
}
// TODO: qw4990, avoid normalizing stmt.PreparedAst.Stmt for binding repeatedly.
sessionHandle := sctx.Value(bindinfo.SessionBindInfoKeyType).(bindinfo.SessionBindingHandle)
bindRecord, _ := sessionHandle.MatchSessionBinding(sctx, stmt.PreparedAst.Stmt)
if bindRecord != nil {
Expand Down
Loading

0 comments on commit 5fe7940

Please sign in to comment.