diff --git a/cmd/importer/stats.go b/cmd/importer/stats.go index 1eb59bc61e02e..fb547f63d319d 100644 --- a/cmd/importer/stats.go +++ b/cmd/importer/stats.go @@ -40,7 +40,7 @@ func loadStats(tblInfo *model.TableInfo, path string) (*stats.Table, error) { return nil, errors.Trace(err) } handle := stats.NewHandle(mock.NewContext(), 0) - return handle.LoadStatsFromJSON(tblInfo, jsTable) + return handle.LoadStatsFromJSONToTable(tblInfo, jsTable) } type histogram struct { diff --git a/domain/domain.go b/domain/domain.go index 0e659ea7973e8..0f1ddd60115e3 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -668,6 +668,11 @@ func (do *Domain) updateStatsWorker(ctx sessionctx.Context, owner owner.Manager) log.Error("[stats] save histogram to storage fail: ", errors.ErrorStack(err)) } } + case t := <-statsHandle.LoadMetaCh(): + err = statistics.SaveMetaToStorage(ctx, t.TableID, t.Count, t.ModifyCount) + if err != nil { + log.Error("[stats] save meta to storage fail: ", errors.ErrorStack(err)) + } case <-deltaUpdateTicker.C: err = statsHandle.DumpStatsDeltaToKV() if err != nil { diff --git a/executor/load_stats.go b/executor/load_stats.go index 2e20a074206ed..66c456dd7edfb 100644 --- a/executor/load_stats.go +++ b/executor/load_stats.go @@ -18,7 +18,6 @@ import ( "github.com/juju/errors" "github.com/pingcap/tidb/domain" - "github.com/pingcap/tidb/model" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/util/chunk" @@ -81,68 +80,10 @@ func (e *LoadStatsInfo) Update(data []byte) error { if err := json.Unmarshal(data, jsonTbl); err != nil { return errors.Trace(err) } - do := domain.GetDomain(e.Ctx) - is := do.InfoSchema() - - tableInfo, err := is.TableByName(model.NewCIStr(jsonTbl.DatabaseName), model.NewCIStr(jsonTbl.TableName)) - if err != nil { - return errors.Trace(err) - } - h := do.StatsHandle() if h == nil { return errors.New("Load Stats: handle is nil") } - - tbl, err := h.LoadStatsFromJSON(tableInfo.Meta(), jsonTbl) - if err != nil { - return errors.Trace(err) - } - - if h.Lease > 0 { - hists := make([]*statistics.Histogram, 0, len(tbl.Columns)) - cms := make([]*statistics.CMSketch, 0, len(tbl.Columns)) - for _, col := range tbl.Columns { - hists = append(hists, &col.Histogram) - cms = append(cms, col.CMSketch) - } - h.AnalyzeResultCh() <- &statistics.AnalyzeResult{ - TableID: tbl.TableID, - Hist: hists, - Cms: cms, - Count: tbl.Count, - IsIndex: 0, - Err: nil} - - hists = make([]*statistics.Histogram, 0, len(tbl.Indices)) - cms = make([]*statistics.CMSketch, 0, len(tbl.Indices)) - for _, idx := range tbl.Indices { - hists = append(hists, &idx.Histogram) - cms = append(cms, idx.CMSketch) - } - h.AnalyzeResultCh() <- &statistics.AnalyzeResult{ - TableID: tbl.TableID, - Hist: hists, - Cms: cms, - Count: tbl.Count, - IsIndex: 1, - Err: nil} - - return nil - } - for _, col := range tbl.Columns { - err = statistics.SaveStatsToStorage(e.Ctx, tbl.TableID, tbl.Count, 0, &col.Histogram, col.CMSketch) - if err != nil { - return errors.Trace(err) - } - } - for _, idx := range tbl.Indices { - err = statistics.SaveStatsToStorage(e.Ctx, tbl.TableID, tbl.Count, 1, &idx.Histogram, idx.CMSketch) - if err != nil { - return errors.Trace(err) - } - } - err = h.Update(GetInfoSchema(e.Ctx)) - return errors.Trace(err) + return errors.Trace(h.LoadStatsFromJSON(GetInfoSchema(e.Ctx), jsonTbl)) } diff --git a/server/statistics_handler_test.go b/server/statistics_handler_test.go index 5d495128b6dc5..588699fd3beb9 100644 --- a/server/statistics_handler_test.go +++ b/server/statistics_handler_test.go @@ -21,6 +21,7 @@ import ( "os" "github.com/go-sql-driver/mysql" + "github.com/gorilla/mux" . "github.com/pingcap/check" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/session" @@ -30,41 +31,17 @@ import ( type testDumpStatsSuite struct { server *Server + sh *StatsHandler } var _ = Suite(new(testDumpStatsSuite)) -func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) { - ds.startServer(c) - ds.prepareData(c) - defer ds.stopServer(c) - - resp, err := http.Get("http://127.0.0.1:10090/stats/dump/tidb/test") - c.Assert(err, IsNil) - - path := "/tmp/stats.json" - fp, err := os.Create(path) - c.Assert(err, IsNil) - c.Assert(fp, NotNil) - - defer func() { - err = fp.Close() - c.Assert(err, IsNil) - err = os.Remove(path) - c.Assert(err, IsNil) - }() - - js, err := ioutil.ReadAll(resp.Body) - c.Assert(err, IsNil) - fp.Write(js) - ds.checkData(c, path) -} - func (ds *testDumpStatsSuite) startServer(c *C) { mvccStore := mocktikv.MustNewMVCCStore() store, err := mockstore.NewMockTikvStore(mockstore.WithMVCCStore(mvccStore)) c.Assert(err, IsNil) + session.SetStatsLease(0) _, err = session.BootstrapSession(store) c.Assert(err, IsNil) @@ -81,6 +58,10 @@ func (ds *testDumpStatsSuite) startServer(c *C) { ds.server = server go server.Run() waitUntilServerOnline(cfg.Status.StatusPort) + + do, err := session.GetDomain(store) + c.Assert(err, IsNil) + ds.sh = &StatsHandler{do} } func (ds *testDumpStatsSuite) stopServer(c *C) { @@ -89,18 +70,58 @@ func (ds *testDumpStatsSuite) stopServer(c *C) { } } +func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) { + ds.startServer(c) + ds.prepareData(c) + defer ds.stopServer(c) + + router := mux.NewRouter() + router.Handle("/stats/dump/{db}/{table}", ds.sh) + + srv := &http.Server{Addr: ":10099", Handler: router} + go srv.ListenAndServe() + defer srv.Close() + + waitUntilServerOnline(10099) + + resp, err := http.Get("http://127.0.0.1:10099/stats/dump/tidb/test") + c.Assert(err, IsNil) + defer resp.Body.Close() + + path := "/tmp/stats.json" + fp, err := os.Create(path) + c.Assert(err, IsNil) + c.Assert(fp, NotNil) + defer func() { + c.Assert(fp.Close(), IsNil) + c.Assert(os.Remove(path), IsNil) + }() + + js, err := ioutil.ReadAll(resp.Body) + c.Assert(err, IsNil) + fp.Write(js) + ds.checkData(c, path) +} + func (ds *testDumpStatsSuite) prepareData(c *C) { db, err := sql.Open("mysql", getDSN()) c.Assert(err, IsNil, Commentf("Error connecting")) defer db.Close() dbt := &DBTest{c, db} + h := ds.sh.do.StatsHandle() dbt.mustExec("create database tidb") dbt.mustExec("use tidb") dbt.mustExec("create table test (a int, b varchar(20))") + h.HandleDDLEvent(<-h.DDLEventCh()) dbt.mustExec("create index c on test (a, b)") - dbt.mustExec("insert test values (1, 2)") + dbt.mustExec("insert test values (1, 's')") + c.Assert(h.DumpStatsDeltaToKV(), IsNil) dbt.mustExec("analyze table test") + dbt.mustExec("insert into test(a,b) values (1, 'v'),(3, 'vvv'),(5, 'vv')") + is := ds.sh.do.InfoSchema() + c.Assert(h.DumpStatsDeltaToKV(), IsNil) + c.Assert(h.Update(is), IsNil) } func (ds *testDumpStatsSuite) checkData(c *C, path string) { @@ -109,19 +130,29 @@ func (ds *testDumpStatsSuite) checkData(c *C, path string) { config.Strict = false })) c.Assert(err, IsNil, Commentf("Error connecting")) - defer db.Close() dbt := &DBTest{c, db} + defer func() { + dbt.mustExec("drop database tidb") + dbt.mustExec("truncate table mysql.stats_meta") + dbt.mustExec("truncate table mysql.stats_histograms") + dbt.mustExec("truncate table mysql.stats_buckets") + db.Close() + }() + dbt.mustExec("use tidb") dbt.mustExec("drop stats test") _, err = dbt.db.Exec(fmt.Sprintf("load stats '%s'", path)) c.Assert(err, IsNil) - rows := dbt.mustQuery("show stats_histograms") + rows := dbt.mustQuery("show stats_meta") dbt.Check(rows.Next(), IsTrue, Commentf("unexpected data")) var dbName, tableName string + var modifyCount, count int64 var other interface{} - err = rows.Scan(&dbName, &tableName, &other, &other, &other, &other, &other, &other) + err = rows.Scan(&dbName, &tableName, &other, &modifyCount, &count) dbt.Check(err, IsNil) dbt.Check(dbName, Equals, "tidb") dbt.Check(tableName, Equals, "test") + dbt.Check(modifyCount, Equals, int64(3)) + dbt.Check(count, Equals, int64(4)) } diff --git a/statistics/dump.go b/statistics/dump.go index 19bd55ad38044..8e855ae196dc0 100644 --- a/statistics/dump.go +++ b/statistics/dump.go @@ -15,6 +15,7 @@ package statistics import ( "github.com/juju/errors" + "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/model" "github.com/pingcap/tidb/mysql" "github.com/pingcap/tidb/sessionctx/stmtctx" @@ -30,7 +31,6 @@ type JSONTable struct { Indices map[string]*jsonColumn `json:"indices"` Count int64 `json:"count"` ModifyCount int64 `json:"modify_count"` - Version uint64 `json:"version"` } type jsonColumn struct { @@ -60,6 +60,9 @@ func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo) (*JS if err != nil { return nil, errors.Trace(err) } + if tbl == nil { + return nil, nil + } jsonTbl := &JSONTable{ DatabaseName: dbName, TableName: tableInfo.Name.L, @@ -67,7 +70,6 @@ func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo) (*JS Indices: make(map[string]*jsonColumn, len(tbl.Indices)), Count: tbl.Count, ModifyCount: tbl.ModifyCount, - Version: tbl.Version, } for _, col := range tbl.Columns { hist, err := col.ConvertTo(new(stmtctx.StatementContext), types.NewFieldType(mysql.TypeBlob)) @@ -83,17 +85,83 @@ func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo) (*JS return jsonTbl, nil } -// LoadStatsFromJSON load statistic from json. -func (h *Handle) LoadStatsFromJSON(tableInfo *model.TableInfo, jsonTbl *JSONTable) (*Table, error) { +// LoadStatsFromJSON will load statistic from JSONTable, and save it to the storage. +func (h *Handle) LoadStatsFromJSON(is infoschema.InfoSchema, jsonTbl *JSONTable) error { + tableInfo, err := is.TableByName(model.NewCIStr(jsonTbl.DatabaseName), model.NewCIStr(jsonTbl.TableName)) + if err != nil { + return errors.Trace(err) + } + tbl, err := h.LoadStatsFromJSONToTable(tableInfo.Meta(), jsonTbl) + if err != nil { + return errors.Trace(err) + } + + if h.Lease > 0 { + hists := make([]*Histogram, 0, len(tbl.Columns)) + cms := make([]*CMSketch, 0, len(tbl.Columns)) + for _, col := range tbl.Columns { + hists = append(hists, &col.Histogram) + cms = append(cms, col.CMSketch) + } + h.AnalyzeResultCh() <- &AnalyzeResult{ + TableID: tbl.TableID, + Hist: hists, + Cms: cms, + Count: tbl.Count, + IsIndex: 0, + Err: nil, + } + + hists = make([]*Histogram, 0, len(tbl.Indices)) + cms = make([]*CMSketch, 0, len(tbl.Indices)) + for _, idx := range tbl.Indices { + hists = append(hists, &idx.Histogram) + cms = append(cms, idx.CMSketch) + } + h.AnalyzeResultCh() <- &AnalyzeResult{ + TableID: tbl.TableID, + Hist: hists, + Cms: cms, + Count: tbl.Count, + IsIndex: 1, + Err: nil, + } + + h.LoadMetaCh() <- &LoadMeta{ + TableID: tbl.TableID, + Count: tbl.Count, + ModifyCount: tbl.ModifyCount, + } + return errors.Trace(err) + } + for _, col := range tbl.Columns { + err = SaveStatsToStorage(h.ctx, tbl.TableID, tbl.Count, 0, &col.Histogram, col.CMSketch) + if err != nil { + return errors.Trace(err) + } + } + for _, idx := range tbl.Indices { + err = SaveStatsToStorage(h.ctx, tbl.TableID, tbl.Count, 1, &idx.Histogram, idx.CMSketch) + if err != nil { + return errors.Trace(err) + } + } + err = SaveMetaToStorage(h.ctx, tbl.TableID, tbl.Count, tbl.ModifyCount) + if err != nil { + return errors.Trace(err) + } + return errors.Trace(h.Update(is)) +} + +// LoadStatsFromJSONToTable load statistic from JSONTable and return the Table of statistic. +func (h *Handle) LoadStatsFromJSONToTable(tableInfo *model.TableInfo, jsonTbl *JSONTable) (*Table, error) { tbl := &Table{ TableID: tableInfo.ID, Columns: make(map[int64]*Column, len(jsonTbl.Columns)), Indices: make(map[int64]*Index, len(jsonTbl.Indices)), Count: jsonTbl.Count, - Version: jsonTbl.Version, ModifyCount: jsonTbl.ModifyCount, } - for id, jsonIdx := range jsonTbl.Indices { for _, idxInfo := range tableInfo.Indices { if idxInfo.Name.L != id { @@ -132,3 +200,10 @@ func (h *Handle) LoadStatsFromJSON(tableInfo *model.TableInfo, jsonTbl *JSONTabl } return tbl, nil } + +// LoadMeta is the statistic meta loaded from json file. +type LoadMeta struct { + TableID int64 + Count int64 + ModifyCount int64 +} diff --git a/statistics/dump_test.go b/statistics/dump_test.go index e7fb09c404ef8..c631d5ef5e680 100644 --- a/statistics/dump_test.go +++ b/statistics/dump_test.go @@ -51,16 +51,24 @@ func (s *testDumpStatsSuite) TestConversion(c *C) { tk.MustExec("create index c on t(a,b)") tk.MustExec("insert into t(a,b) values (3, 1),(2, 1),(1, 10)") tk.MustExec("analyze table t") - + tk.MustExec("insert into t(a,b) values (1, 1),(3, 1),(5, 10)") is := s.do.InfoSchema() h := s.do.StatsHandle() + h.DumpStatsDeltaToKV() h.Update(is) + tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) jsonTbl, err := h.DumpStatsToJSON("test", tableInfo.Meta()) c.Assert(err, IsNil) - loadTbl, err := h.LoadStatsFromJSON(tableInfo.Meta(), jsonTbl) + loadTbl, err := h.LoadStatsFromJSONToTable(tableInfo.Meta(), jsonTbl) c.Assert(err, IsNil) + tbl := h.GetTableStats(tableInfo.Meta()) assertTableEqual(c, loadTbl, tbl) + + err = h.LoadStatsFromJSON(is, jsonTbl) + c.Assert(err, IsNil) + loadTblInStorage := h.GetTableStats(tableInfo.Meta()) + assertTableEqual(c, loadTblInStorage, tbl) } diff --git a/statistics/handle.go b/statistics/handle.go index fdb9ec9474604..4867fe18fb4ea 100644 --- a/statistics/handle.go +++ b/statistics/handle.go @@ -54,6 +54,8 @@ type Handle struct { feedback []*QueryFeedback Lease time.Duration + // loadMetaCh is a channel to notify a load stats operation has done. + loadMetaCh chan *LoadMeta } // Clear the statsCache, only for test. @@ -85,6 +87,7 @@ func NewHandle(ctx sessionctx.Context, lease time.Duration) *Handle { globalMap: make(tableDeltaMap), Lease: lease, feedback: make([]*QueryFeedback, 0, MaxQueryFeedbackCount), + loadMetaCh: make(chan *LoadMeta, 1), } handle.statsCache.Store(statsCache{}) return handle @@ -206,3 +209,8 @@ func (h *Handle) LoadNeededHistograms() error { } return nil } + +// LoadMetaCh returns loaded statistic meta channel in handle. +func (h *Handle) LoadMetaCh() chan *LoadMeta { + return h.loadMetaCh +} diff --git a/statistics/handle_test.go b/statistics/handle_test.go index 94efbf44062c6..68647c58c1c6a 100644 --- a/statistics/handle_test.go +++ b/statistics/handle_test.go @@ -111,6 +111,8 @@ func (s *testStatsCacheSuite) TestStatsCache(c *C) { } func assertTableEqual(c *C, a *statistics.Table, b *statistics.Table) { + c.Assert(a.Count, Equals, b.Count) + c.Assert(a.ModifyCount, Equals, b.ModifyCount) c.Assert(len(a.Columns), Equals, len(b.Columns)) for i := range a.Columns { c.Assert(a.Columns[i].Count, Equals, b.Columns[i].Count) diff --git a/statistics/histogram.go b/statistics/histogram.go index b4adb92e1eab7..520e683c8d863 100644 --- a/statistics/histogram.go +++ b/statistics/histogram.go @@ -33,7 +33,7 @@ import ( "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/ranger" "github.com/pingcap/tidb/util/sqlexec" - tipb "github.com/pingcap/tipb/go-tipb" + "github.com/pingcap/tipb/go-tipb" "golang.org/x/net/context" ) @@ -247,6 +247,26 @@ func SaveStatsToStorage(sctx sessionctx.Context, tableID int64, count int64, isI return errors.Trace(err) } +// SaveMetaToStorage will save stats_meta to storage. +func SaveMetaToStorage(sctx sessionctx.Context, tableID, count, modifyCount int64) error { + ctx := context.TODO() + exec := sctx.(sqlexec.SQLExecutor) + _, err := exec.Execute(ctx, "begin") + if err != nil { + return errors.Trace(err) + } + var sql string + version := sctx.Txn().StartTS() + sql = fmt.Sprintf("replace into mysql.stats_meta (version, table_id, count, modify_count) values (%d, %d, %d, %d)", version, tableID, count, modifyCount) + if _, err = exec.Execute(ctx, sql); err != nil { + _, err1 := exec.Execute(ctx, "rollback") + terror.Log(errors.Trace(err1)) + return errors.Trace(err) + } + _, err = exec.Execute(ctx, "commit") + return errors.Trace(err) +} + func histogramFromStorage(ctx sessionctx.Context, tableID int64, colID int64, tp *types.FieldType, distinct int64, isIndex int, ver uint64, nullCount int64, totColSize int64) (*Histogram, error) { selSQL := fmt.Sprintf("select count, repeats, lower_bound, upper_bound from mysql.stats_buckets where table_id = %d and is_index = %d and hist_id = %d order by bucket_id", tableID, isIndex, colID) rows, fields, err := ctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQL(ctx, selSQL) diff --git a/statistics/table.go b/statistics/table.go index bff3a43cbe0e9..6cde3d1a631b1 100644 --- a/statistics/table.go +++ b/statistics/table.go @@ -53,11 +53,13 @@ type Table struct { func (t *Table) copy() *Table { nt := &Table{ - TableID: t.TableID, - Count: t.Count, - Pseudo: t.Pseudo, - Columns: make(map[int64]*Column), - Indices: make(map[int64]*Index), + TableID: t.TableID, + Count: t.Count, + ModifyCount: t.ModifyCount, + Version: t.Version, + Pseudo: t.Pseudo, + Columns: make(map[int64]*Column), + Indices: make(map[int64]*Index), } for id, col := range t.Columns { nt.Columns[id] = col