Skip to content

Commit

Permalink
Add idxadv_test.go for package idxadvisor, need more test cases.
Browse files Browse the repository at this point in the history
  • Loading branch information
zouhuan1215 committed Aug 11, 2019
1 parent 25b795f commit 5cc8967
Show file tree
Hide file tree
Showing 7 changed files with 214 additions and 27 deletions.
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
github.com/grpc-ecosystem/grpc-gateway v1.5.1 // indirect
github.com/json-iterator/go v1.1.6 // indirect
github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d
github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2 // indirect
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
Expand Down Expand Up @@ -71,6 +74,7 @@ require (
golang.org/x/tools v0.0.0-20190130214255-bb1329dc71a0
google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275 // indirect
google.golang.org/grpc v1.17.0
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0
sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4
sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67
Expand Down
8 changes: 8 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,14 @@ github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d h1:hJXjZMxj0SWlMoQkzeZDLi2cmeiWKa7y1B8Rg+qaoEc=
github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 h1:UUHMLvzt/31azWTN/ifGWef4WUqvXk0iRqdhdy/2uzI=
github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY=
github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2 h1:Pp8RxiF4rSoXP9SED26WCfNB28/dwTDpPXS8XMJR8rc=
github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio=
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
Expand Down Expand Up @@ -304,6 +310,8 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
Expand Down
132 changes: 132 additions & 0 deletions idxadvisor/idxadv_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
package idxadvisor_test

import (
"encoding/json"
"fmt"
"io/ioutil"
"path/filepath"
"testing"

"github.com/juju/errors"
. "github.com/pingcap/check"
"github.com/pingcap/tidb/domain"
idxadv "github.com/pingcap/tidb/idxadvisor"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/statistics/handle"
"github.com/pingcap/tidb/store/mockstore"
"github.com/pingcap/tidb/util/testkit"
"github.com/pingcap/tidb/util/testleak"
)

var _ = Suite(&testAnalyzeSuite{})

type testAnalyzeSuite struct {
}

func TestT(t *testing.T) {
CustomVerboseFlag = true
TestingT(t)
}

func (s *testAnalyzeSuite) loadTableStats(fileName string, dom *domain.Domain) error {
statsPath := filepath.Join("testdata", fileName)
bytes, err := ioutil.ReadFile(statsPath)

if err != nil {
return err
}

statsTbl := &handle.JSONTable{}
err = json.Unmarshal(bytes, statsTbl)

if err != nil {
return err
}

statsHandle := dom.StatsHandle()
err = statsHandle.LoadStatsFromJSON(dom.InfoSchema(), statsTbl)

if err != nil {
return err
}

return nil
}

func (s *testAnalyzeSuite) TestSQLClient(c *C) {
err := idxadv.RunSqlClient("test-mode")
c.Assert(err, IsNil)
}

func (s *testAnalyzeSuite) TestIndexAdvisor(c *C) {
defer testleak.AfterTest(c)()
store, dom, err := newStoreWithBootstrap()
c.Assert(err, IsNil)

testkit := testkit.NewTestKit(c, store)
idxadv.MockNewIdxAdv()
defer func() {
dom.Close()
store.Close()
}()

testkit.MustExec("use test")
testkit.MustExec("drop table if exists t, t1")
testkit.MustExec("create table t (a int primary key, b int, c varchar(200), d datetime DEFAULT CURRENT_TIMESTAMP, e int, ts timestamp DEFAULT CURRENT_TIMESTAMP)")
testkit.MustExec("create table t1 (a int, b int)")

err = s.loadTableStats("analyzesSuiteTestIndexReadT.json", dom)
c.Assert(err, IsNil)

for i := 1; i < 16; i++ {
testkit.MustExec(fmt.Sprintf("insert into t1 values(%v, %v)", i, i))
}
testkit.MustExec("analyze table t1")

ctx := testkit.Se.(sessionctx.Context)
sessionVars := ctx.GetSessionVars()
sessionVars.HashAggFinalConcurrency = 1
sessionVars.HashAggPartialConcurrency = 1
connID := sessionVars.ConnectionID

tests := []struct {
sql []string
res string
}{
{
sql: []string{"select count(*) from t group by e",
"select * from t order by e limit 10"},
res: "t: (e)",
},
}

testkit.MustExec("set tidb_enable_index_advisor=1")
for _, tt := range tests {
for _, sql := range tt.sql {
testkit.Exec(sql)
}
res, err := idxadv.GetRecommendIdxStr(connID)
c.Assert(err, IsNil)
c.Assert(res, Equals, tt.res, Commentf("for %v", tt.sql))
}
}

func newStoreWithBootstrap() (kv.Storage, *domain.Domain, error) {
store, err := mockstore.NewMockTikvStore()
if err != nil {
return nil, nil, errors.Trace(err)
}

session.SetSchemaLease(0)
session.DisableStats4Test()

dom, err := session.BootstrapSession(store)
if err != nil {
return nil, nil, err
}

dom.SetStatsUpdating(true)
return store, dom, errors.Trace(err)
}
65 changes: 59 additions & 6 deletions idxadvisor/idxadvisor.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ func (ci CandidateIdxes) Less(i, j int) bool { return ci[i].Benefit > ci[j].Bene
func (ci CandidateIdxes) Swap(i, j int) { ci[i], ci[j] = ci[j], ci[i] }

// NewIdxAdv create a new IdxAdvisor.
// TODO: *sql.DB is not supposed to a member of IdxAdvisor.
// *sql.DB can interacts with idxadvisor by session variable
func NewIdxAdv(db *sql.DB) *IdxAdvisor {
ia := &IdxAdvisor{dbClient: db}
ia.ready.Store(false)
Expand All @@ -74,6 +76,14 @@ func NewIdxAdv(db *sql.DB) *IdxAdvisor {
return ia
}

// MockNewIdxAdv() return *IdxAdvisor without initiating dbClient member
func MockNewIdxAdv() *IdxAdvisor {
ia := &IdxAdvisor{}
ia.ready.Store(false)
idxadvPool.push(ia)
return ia
}

func GetIdxAdv(connID uint64) *IdxAdvisor {
if ia, ok := registeredIdxAdv[connID]; ok {
return ia
Expand All @@ -87,6 +97,45 @@ func GetIdxAdv(connID uint64) *IdxAdvisor {
}
}

//// GetRecommendIdx return recommend index within given connection
//func GetRecommendIdx(connID uint64) (*CandidateIdxes, error) {
// if ia, ok := registeredIdxAdv[connID]; !ok {
// return nil, errors.New(fmt.Sprintf(
// "bad attempt to get recommend index with no registered index advisor. connID: %v\n", connID))
// } else {
// return &ia.Candidate_idx, nil
// }
//}

// GetRecommendIdxStr return recommend index in string format. Used in test
func GetRecommendIdxStr(connID uint64) (string, error) {
if ia, ok := registeredIdxAdv[connID]; !ok {
return "", errors.New(fmt.Sprintf(
"bad attempt to get recommend index with no registered index advisor. connID: %v\n", connID))
} else {
idxes := ia.Candidate_idx
if len(idxes) == 0 {
return "", nil
}

var idxesStr string
for _, idx := range idxes {
var idxStr string
idxStr = fmt.Sprintf("%s: (", idx.Index.Index.Table.L)
cols := idx.Index.Index.Columns
colLen := len(cols)

for i := 0; i < len(cols)-1; i++ {
idxStr = fmt.Sprintf("%s%s ", idxStr, cols[i].Name.L)
}

idxStr = fmt.Sprintf("%s%s)", idxStr, cols[colLen-1].Name.L)
idxesStr = fmt.Sprintf("%s%s,", idxesStr, idxStr)
}
return idxesStr[:len(idxesStr)-1], nil
}
}

// Init set session variable tidb_enable_index_advisor = true
func (ia *IdxAdvisor) Init() error {
ia.queryChan = make(chan string, queryChanSize)
Expand All @@ -110,10 +159,9 @@ func (ia *IdxAdvisor) IsReady() bool {
}

// StartTask start handling queries in idxadv mode after session variable tidb_enable_index_advisor has been set
func (ia *IdxAdvisor) StartTask(query string) {
func (ia *IdxAdvisor) StartTask(sqlFile string) {
if ia.IsReady() {
sqlFile := "/tmp/queries/"
go readQuery(&sqlFile, ia.queryChan)
go readQuery(sqlFile, ia.queryChan)

cnt := 0
for {
Expand Down Expand Up @@ -173,16 +221,21 @@ func (ia *IdxAdvisor) StartTask(query string) {
// }
//}

func readQuery(sqlFile *string, queryChan chan string) {
func readQuery(sqlFile string, queryChan chan string) {
defer func() {
close(queryChan)
}()

// If readQuery is called in idxadv_test.go, return immediately
if sqlFile == "test-mode" {
return
}

for i := 1; i <= 22; i++ {
sqlfile := *sqlFile + strconv.Itoa(i) + ".sql"
sqlfile := sqlFile + strconv.Itoa(i) + ".sql"
contents, err := ioutil.ReadFile(sqlfile)
if err != nil {
panic(err)

}
sqlBegin := strings.Index(string(contents), "select")
query := contents[sqlBegin:]
Expand Down
23 changes: 6 additions & 17 deletions idxadvisor/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,12 @@ func SaveVirtualIndices(is infoschema.InfoSchema, dbname string, iwc IndicesWith
ia := GetIdxAdv(connID)
indices := iwc.Indices
ia.queryCnt++
// WriteResultToFile(connID, ia.queryCnt, origCost, iwc.Cost, indices)

idxes := make([]*model.IndexInfo, len(indices), len(indices))
for i, indice := range indices {
idxes[i] = indice.Index
}
WriteResultToFile(connID, ia.queryCnt, origCost, iwc.Cost, idxes)

fmt.Printf("***Connection id %d, virtual physical plan's cost: %f, original cost: %f \n", connID, iwc.Cost, origCost)
benefit := origCost - iwc.Cost
Expand Down Expand Up @@ -153,22 +158,6 @@ func BuildIdxOutputInfo(indices []*model.IndexInfo) string {
return vIdxesInfo
}

// WriteResult prints virtual indices and their benefit.
func WriteResult() {
fmt.Println("----------------------Result----------------------")
for _, v := range registeredIdxAdv {
for _, i := range v.Candidate_idx {
fmt.Printf("%s: ", i.Index.Index.Table.L)
fmt.Printf("(")
for _, col := range i.Index.Index.Columns {
fmt.Printf("%s ", col.Name.L)
}
fmt.Printf("\b) %f\n", i.Benefit)
}
fmt.Println("-----------------------------------------------")
}
}

// WriteFinaleResult saves virtual indices and their benefit.
func WriteFinaleResult() {
fmt.Println("----------------------Result----------------------")
Expand Down
8 changes: 4 additions & 4 deletions idxadvisor/sqlclient.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,13 @@ func getDSN(overriders ...configOverrider) string {
}

// runSqlClient runs an index advisor client
func RunSqlClient(query string) error {
func RunSqlClient(sqlFile string) error {
fmt.Println("*******************************************")
fmt.Printf("RunSqlClient\n")
fmt.Println("*******************************************")
waitUntilServerOnline(statusPort)
var defMySQLConfig configOverrider
return runSqlClient(defMySQLConfig, query)
return runSqlClient(defMySQLConfig, sqlFile)
}

func waitUntilServerOnline(statusPort uint) {
Expand Down Expand Up @@ -83,7 +83,7 @@ func waitUntilServerOnline(statusPort uint) {

}

func runSqlClient(overrider configOverrider, query string) error {
func runSqlClient(overrider configOverrider, sqlFile string) error {
db, err := sql.Open("mysql", getDSN(overrider))
defer db.Close()
if err != nil {
Expand All @@ -96,7 +96,7 @@ func runSqlClient(overrider configOverrider, query string) error {
}

fmt.Printf("===============ia.StartTask(query)===========\n")
ia.StartTask(query)
ia.StartTask(sqlFile)

}
return nil
Expand Down
1 change: 1 addition & 0 deletions idxadvisor/testdata/analyzesSuiteTestIndexReadT.json

Large diffs are not rendered by default.

0 comments on commit 5cc8967

Please sign in to comment.