From 4d117d2abaa159f0fbd5d0d09675706df717c65f Mon Sep 17 00:00:00 2001 From: IggieWang <1002203581@qq.com> Date: Sun, 11 Aug 2019 17:20:25 +0800 Subject: [PATCH] finish virtual index DDL statement --- idxadvisor/idxadvisor.go | 121 +++++++++++++++++----------------- idxadvisor/index.go | 103 +++++++++++++++++++++++------ planner/core/index_advisor.go | 2 +- 3 files changed, 145 insertions(+), 81 deletions(-) diff --git a/idxadvisor/idxadvisor.go b/idxadvisor/idxadvisor.go index 4ece6951c898f..18e31da5ff7c2 100644 --- a/idxadvisor/idxadvisor.go +++ b/idxadvisor/idxadvisor.go @@ -177,7 +177,7 @@ func readQuery(sqlFile *string, queryChan chan string) { defer func() { close(queryChan) }() - for i := 1; i <= 22; i++ { + for i := 1; i <= 7; i++ { sqlfile := *sqlFile + strconv.Itoa(i) + ".sql" contents, err := ioutil.ReadFile(sqlfile) if err != nil { @@ -275,64 +275,67 @@ func GenVirtualIndexCols(tblInfo *model.TableInfo, dbname, tblname model.CIStr, } // multi columns - // candidateCols := [][]model.CIStr{} - // eq := tblInfoSets.Eq - // o := tblInfoSets.O - // rg := tblInfoSets.Rg - // ref := tblInfoSets.Ref - // - // // EQ + O + RANGE + REF - // cols := [][]model.CIStr{} - // for i, oCols := range o { - // cols = append(cols, []model.CIStr{}) - // addToCandidateCols(eq, &cols[i], &candidateCols) - // addToCandidateCols(oCols, &cols[i], &candidateCols) - // addToCandidateCols(rg, &cols[i], &candidateCols) - // addToCandidateCols(ref, &cols[i], &candidateCols) - // } - // if len(cols) == 0 { - // cols = append(cols, []model.CIStr{}) - // addToCandidateCols(eq, &cols[0], &candidateCols) - // addToCandidateCols(rg, &cols[0], &candidateCols) - // addToCandidateCols(ref, &cols[0], &candidateCols) - // } - // - // // O + EQ + RANGE + REF - // cols = cols[:0] - // for i, oCols := range o { - // cols = append(cols, []model.CIStr{}) - // addToCandidateCols(oCols, &cols[i], &candidateCols) - // addToCandidateCols(eq, &cols[i], &candidateCols) - // addToCandidateCols(rg, &cols[i], &candidateCols) - // addToCandidateCols(ref, &cols[i], &candidateCols) - // } - // if len(cols) == 0 { - // cols = append(cols, []model.CIStr{}) - // addToCandidateCols(eq, &cols[0], &candidateCols) - // addToCandidateCols(rg, &cols[0], &candidateCols) - // addToCandidateCols(ref, &cols[0], &candidateCols) - // } - // - // candidateCols = plannercore.RemoveRepeatedColumnSet(candidateCols) - // if len(candidateCols) > 0 { - // fmt.Printf("table %s multi candidate index: ", tblname) - // fmt.Println(candidateCols) - // } - // for _, candidateColumns := range candidateCols { - // idxCols := make([]*ast.IndexColName, len(candidateColumns), len(candidateColumns)) - // for i, column := range candidateColumns { - // columnInfo := new(model.ColumnInfo) - // for _, tmpColumn := range columnInfos { - // if tmpColumn.Name.L == column.L { - // columnInfo = tmpColumn - // break - // } - // } - // idxCols[i] = BuildIdxColNameFromColInfo(columnInfo, dbname, tblname) - // } - // result = append(result, idxCols) - // - // } + candidateCols := [][]model.CIStr{} + eq := tblInfoSets.Eq + o := tblInfoSets.O + rg := tblInfoSets.Rg + ref := tblInfoSets.Ref + + // EQ + O + RANGE + REF + cols := [][]model.CIStr{} + for i, oCols := range o { + cols = append(cols, []model.CIStr{}) + addToCandidateCols(eq, &cols[i], &candidateCols) + addToCandidateCols(oCols, &cols[i], &candidateCols) + addToCandidateCols(rg, &cols[i], &candidateCols) + addToCandidateCols(ref, &cols[i], &candidateCols) + } + if len(cols) == 0 { + cols = append(cols, []model.CIStr{}) + addToCandidateCols(eq, &cols[0], &candidateCols) + addToCandidateCols(rg, &cols[0], &candidateCols) + addToCandidateCols(ref, &cols[0], &candidateCols) + } + + // O + EQ + RANGE + REF + cols = cols[:0] + for i, oCols := range o { + cols = append(cols, []model.CIStr{}) + addToCandidateCols(oCols, &cols[i], &candidateCols) + addToCandidateCols(eq, &cols[i], &candidateCols) + addToCandidateCols(rg, &cols[i], &candidateCols) + addToCandidateCols(ref, &cols[i], &candidateCols) + } + if len(cols) == 0 { + cols = append(cols, []model.CIStr{}) + addToCandidateCols(eq, &cols[0], &candidateCols) + addToCandidateCols(rg, &cols[0], &candidateCols) + addToCandidateCols(ref, &cols[0], &candidateCols) + } + + candidateCols = plannercore.RemoveRepeatedColumnSet(candidateCols) + if len(candidateCols) > 0 { + fmt.Printf("table %s multi candidate index: ", tblname) + fmt.Println(candidateCols) + } + for _, candidateColumns := range candidateCols { + idxCols := []*ast.IndexColName{} + for _, column := range candidateColumns { + columnInfo := new(model.ColumnInfo) + isExisted := false + for _, tmpColumn := range columnInfos { + if tmpColumn.Name.L == column.L { + columnInfo = tmpColumn + isExisted = true + break + } + } + if isExisted { + idxCols = append(idxCols, BuildIdxColNameFromColInfo(columnInfo, dbname, tblname)) + } + } + result = append(result, idxCols) + } return result } diff --git a/idxadvisor/index.go b/idxadvisor/index.go index 7ee3dc3e6c566..c518509e4be59 100644 --- a/idxadvisor/index.go +++ b/idxadvisor/index.go @@ -1,6 +1,9 @@ package idxadvisor import ( + "strings" + "io" + "bufio" "fmt" "os" "sort" @@ -10,8 +13,9 @@ import ( plannercore "github.com/pingcap/tidb/planner/core" ) -const outputPath string = "/tmp/indexadvisor" -const sepString string = " " +const outputPath string = "/home/iggie" +const sepString string = " " +const TopN = 3 // IndicesWithCost includes in indices and their physical plan cost. type IndicesWithCost struct { @@ -73,7 +77,11 @@ func SaveVirtualIndices(is infoschema.InfoSchema, dbname string, iwc IndicesWith ia := GetIdxAdv(connID) indices := iwc.Indices ia.queryCnt++ - // WriteResultToFile(connID, ia.queryCnt, origCost, iwc.Cost, indices) + idxes := []*model.IndexInfo{} + for _, index := range indices { + idxes = append(idxes, index.Index) + } + writeResultToFile(connID, ia.queryCnt, origCost, iwc.Cost, idxes) fmt.Printf("***Connection id %d, virtual physical plan's cost: %f, original cost: %f \n", connID, iwc.Cost, origCost) benefit := origCost - iwc.Cost @@ -108,27 +116,30 @@ func SaveVirtualIndices(is infoschema.InfoSchema, dbname string, iwc IndicesWith } } -func WriteResultToFile(connID uint64, queryCnt uint64, origCost, vcost float64, indices []*model.IndexInfo) { +func writeResultToFile(connID uint64, queryCnt uint64, origCost, vcost float64, indices []*model.IndexInfo) { origCostPrefix := fmt.Sprintf("%v_OCOST", connID) origCostOut := fmt.Sprintf("%-10d%f\n", queryCnt, origCost) - WriteToFile(origCostPrefix, origCostOut) + writeToFile(origCostPrefix, origCostOut, true) virtualCostPrefix := fmt.Sprintf("%v_OVCOST", connID) virtualCostOut := fmt.Sprintf("%-10d%f\n", queryCnt, vcost) - WriteToFile(virtualCostPrefix, virtualCostOut) + writeToFile(virtualCostPrefix, virtualCostOut, true) virtualIdxPrefix := fmt.Sprintf("%v_OINDEX", connID) - virtualIdxOut := fmt.Sprintf("%-10d{%s}\n", queryCnt, BuildIdxOutputInfo(indices)) - WriteToFile(virtualIdxPrefix, virtualIdxOut) + virtualIdxOut := fmt.Sprintf("%-10d{%s}\n", queryCnt, buildIdxOutputInfo(indices)) + writeToFile(virtualIdxPrefix, virtualIdxOut, true) origSummaryPrefix := fmt.Sprintf("%v_ORIGIN", connID) - origSummaryOut := fmt.Sprintf("%-10d%f%v%f%v{%v}\n", queryCnt, origCost, sepString, vcost, sepString, BuildIdxOutputInfo(indices)) - WriteToFile(origSummaryPrefix, origSummaryOut) + origSummaryOut := fmt.Sprintf("%-10d%f%v%f%v{%v}\n", queryCnt, origCost, sepString, vcost, sepString, buildIdxOutputInfo(indices)) + writeToFile(origSummaryPrefix, origSummaryOut, true) } -func WriteToFile(filePrefix, content string) { - fileName := fmt.Sprintf("%s/%s", outputPath, filePrefix) +func writeToFile(filename, content string, append bool) { + fileName := fmt.Sprintf("%s/%s", outputPath, filename) fd, err := os.OpenFile(fileName, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666) + if !append { + fd, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0666) + } if err != nil { panic(err) } @@ -137,7 +148,29 @@ func WriteToFile(filePrefix, content string) { fd.WriteString(content) } -func BuildIdxOutputInfo(indices []*model.IndexInfo) string { +func readFile(filename string) (lines []string, err error) { + fp, err := os.Open(filename) + if err != nil { + return nil, err + } + defer fp.Close() + + r := bufio.NewReader(fp) + for { + line, _, err := r.ReadLine() + if err != nil { + if err == io.EOF { + err = nil + break + } + } else { + lines = append(lines, string(line)) + } + } + return +} + +func buildIdxOutputInfo(indices []*model.IndexInfo) string { var vIdxesInfo string if len(indices) == 0 { return "" @@ -161,7 +194,7 @@ func WriteResult() { fmt.Printf("%s: ", i.Index.Index.Table.L) fmt.Printf("(") for _, col := range i.Index.Index.Columns { - fmt.Printf("%s ", col.Name.L) + fmt.Printf("%s,", col.Name.L) } fmt.Printf("\b) %f\n", i.Benefit) } @@ -171,17 +204,45 @@ func WriteResult() { // WriteFinaleResult saves virtual indices and their benefit. func WriteFinaleResult() { - fmt.Println("----------------------Result----------------------") - for _, v := range registeredIdxAdv { + for id, v := range registeredIdxAdv { sort.Sort(v.Candidate_idx) + resFile := fmt.Sprintf("%v_RESULT", id) + content := "" for _, i := range v.Candidate_idx { - fmt.Printf("%s: ", i.Index.Index.Table.L) - fmt.Printf("(") + content += fmt.Sprintf("%s: (", i.Index.Index.Table.L) for _, col := range i.Index.Index.Columns { - fmt.Printf("%s ", col.Name.L) + content += fmt.Sprintf("%s,", col.Name.L) } - fmt.Printf("\b) %f\n", i.Benefit) + content = content[:len(content)-1] + content += fmt.Sprintf(") %f\n", i.Benefit) } - fmt.Println("-----------------------------------------------") + writeToFile(resFile, content, false) + + res, _:= CreateTopNIndexSQL(resFile, TopN) + fmt.Println(res) } } + +// CreateTopNIndexSQL generates the SQL statements for the first N virtual indices. +func CreateTopNIndexSQL(filename string, n int) (sql []string, err error) { + fileName := fmt.Sprintf("%s/%s", outputPath, filename) + lines, err := readFile(fileName) + if err != nil { + return nil, err + } + + if len(lines) < n { + n = len(lines) + } + + for i := 0; i < n; i++ { + res := strings.Fields(lines[i]) + if len(res) != 3 { + panic("split error") + } + tbl := res[0][:len(res[0])-1] + idx := res[1] + sql = append(sql, fmt.Sprintf("create index virtual_index_%d on %s%s;", i+1, tbl, idx)) + } + return +} \ No newline at end of file diff --git a/planner/core/index_advisor.go b/planner/core/index_advisor.go index 191f6c19f6952..883e643581d80 100644 --- a/planner/core/index_advisor.go +++ b/planner/core/index_advisor.go @@ -404,7 +404,7 @@ func RemoveRepeatedColumn(columns []model.CIStr) (ret []model.CIStr) { ret = append(ret, s) } else { for i, v := range ret { - if reflect.DeepEqual(s, v) { + if s.L == v.L { break } if i == len(ret)-1 {