stats: fix dump stats

cmd/importer/stats.go

@@ -40,7 +40,7 @@ func loadStats(tblInfo *model.TableInfo, path string) (*stats.Table, error) {
 		return nil, errors.Trace(err)
 	}
 	handle := stats.NewHandle(mock.NewContext(), 0)
-	return handle.LoadStatsFromJSON(tblInfo, jsTable)
+	return handle.LoadStatsFromJSONToTable(tblInfo, jsTable)
 }
 
 type histogram struct {

domain/domain.go

@@ -668,6 +668,11 @@ func (do *Domain) updateStatsWorker(ctx sessionctx.Context, owner owner.Manager)
 					log.Error("[stats] save histogram to storage fail: ", errors.ErrorStack(err))
 				}
 			}
+		case t := <-statsHandle.LoadMetaCh():
+			err = statistics.SaveMetaToStorage(ctx, t.TableID, t.Count, t.ModifyCount)
+			if err != nil {
+				log.Error("[stats] save meta to storage fail: ", errors.ErrorStack(err))
+			}
 		case <-deltaUpdateTicker.C:
 			err = statsHandle.DumpStatsDeltaToKV()
 			if err != nil {

executor/load_stats.go

@@ -18,7 +18,6 @@ import (
 
 	"github.com/juju/errors"
 	"github.com/pingcap/tidb/domain"
-	"github.com/pingcap/tidb/model"
 	"github.com/pingcap/tidb/sessionctx"
 	"github.com/pingcap/tidb/statistics"
 	"github.com/pingcap/tidb/util/chunk"
@@ -81,68 +80,10 @@ func (e *LoadStatsInfo) Update(data []byte) error {
 	if err := json.Unmarshal(data, jsonTbl); err != nil {
 		return errors.Trace(err)
 	}
-
 	do := domain.GetDomain(e.Ctx)
-	is := do.InfoSchema()
-
-	tableInfo, err := is.TableByName(model.NewCIStr(jsonTbl.DatabaseName), model.NewCIStr(jsonTbl.TableName))
-	if err != nil {
-		return errors.Trace(err)
-	}
-
 	h := do.StatsHandle()
 	if h == nil {
 		return errors.New("Load Stats: handle is nil")
 	}
-
-	tbl, err := h.LoadStatsFromJSON(tableInfo.Meta(), jsonTbl)
-	if err != nil {
-		return errors.Trace(err)
-	}
-
-	if h.Lease > 0 {
-		hists := make([]*statistics.Histogram, 0, len(tbl.Columns))
-		cms := make([]*statistics.CMSketch, 0, len(tbl.Columns))
-		for _, col := range tbl.Columns {
-			hists = append(hists, &col.Histogram)
-			cms = append(cms, col.CMSketch)
-		}
-		h.AnalyzeResultCh() <- &statistics.AnalyzeResult{
-			TableID: tbl.TableID,
-			Hist:    hists,
-			Cms:     cms,
-			Count:   tbl.Count,
-			IsIndex: 0,
-			Err:     nil}
-
-		hists = make([]*statistics.Histogram, 0, len(tbl.Indices))
-		cms = make([]*statistics.CMSketch, 0, len(tbl.Indices))
-		for _, idx := range tbl.Indices {
-			hists = append(hists, &idx.Histogram)
-			cms = append(cms, idx.CMSketch)
-		}
-		h.AnalyzeResultCh() <- &statistics.AnalyzeResult{
-			TableID: tbl.TableID,
-			Hist:    hists,
-			Cms:     cms,
-			Count:   tbl.Count,
-			IsIndex: 1,
-			Err:     nil}
-
-		return nil
-	}
-	for _, col := range tbl.Columns {
-		err = statistics.SaveStatsToStorage(e.Ctx, tbl.TableID, tbl.Count, 0, &col.Histogram, col.CMSketch)
-		if err != nil {
-			return errors.Trace(err)
-		}
-	}
-	for _, idx := range tbl.Indices {
-		err = statistics.SaveStatsToStorage(e.Ctx, tbl.TableID, tbl.Count, 1, &idx.Histogram, idx.CMSketch)
-		if err != nil {
-			return errors.Trace(err)
-		}
-	}
-	err = h.Update(GetInfoSchema(e.Ctx))
-	return errors.Trace(err)
+	return errors.Trace(h.LoadStatsFromJSON(GetInfoSchema(e.Ctx), jsonTbl))
 }

server/statistics_handler_test.go

@@ -21,6 +21,7 @@ import (
 	"os"
 
 	"github.com/go-sql-driver/mysql"
+	"github.com/gorilla/mux"
 	. "github.com/pingcap/check"
 	"github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/session"
@@ -30,41 +31,17 @@ import (
 
 type testDumpStatsSuite struct {
 	server *Server
+	sh     *StatsHandler
 }
 
 var _ = Suite(new(testDumpStatsSuite))
 
-func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) {
-	ds.startServer(c)
-	ds.prepareData(c)
-	defer ds.stopServer(c)
-
-	resp, err := http.Get("http://127.0.0.1:10090/stats/dump/tidb/test")
-	c.Assert(err, IsNil)
-
-	path := "/tmp/stats.json"
-	fp, err := os.Create(path)
-	c.Assert(err, IsNil)
-	c.Assert(fp, NotNil)
-
-	defer func() {
-		err = fp.Close()
-		c.Assert(err, IsNil)
-		err = os.Remove(path)
-		c.Assert(err, IsNil)
-	}()
-
-	js, err := ioutil.ReadAll(resp.Body)
-	c.Assert(err, IsNil)
-	fp.Write(js)
-	ds.checkData(c, path)
-}
-
 func (ds *testDumpStatsSuite) startServer(c *C) {
 	mvccStore := mocktikv.MustNewMVCCStore()
 	store, err := mockstore.NewMockTikvStore(mockstore.WithMVCCStore(mvccStore))
 	c.Assert(err, IsNil)
 
+	session.SetStatsLease(0)
 	_, err = session.BootstrapSession(store)
 	c.Assert(err, IsNil)
 
@@ -81,6 +58,10 @@ func (ds *testDumpStatsSuite) startServer(c *C) {
 	ds.server = server
 	go server.Run()
 	waitUntilServerOnline(cfg.Status.StatusPort)
+
+	do, err := session.GetDomain(store)
+	c.Assert(err, IsNil)
+	ds.sh = &StatsHandler{do}
 }
 
 func (ds *testDumpStatsSuite) stopServer(c *C) {
@@ -89,18 +70,58 @@ func (ds *testDumpStatsSuite) stopServer(c *C) {
 	}
 }
 
+func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) {
+	ds.startServer(c)
+	ds.prepareData(c)
+	defer ds.stopServer(c)
+
+	router := mux.NewRouter()
+	router.Handle("/stats/dump/{db}/{table}", ds.sh)
+
+	srv := &http.Server{Addr: ":10099", Handler: router}
+	go srv.ListenAndServe()
+	defer srv.Close()
+
+	waitUntilServerOnline(10099)
+
+	resp, err := http.Get("http://127.0.0.1:10099/stats/dump/tidb/test")
+	c.Assert(err, IsNil)
+	defer resp.Body.Close()
+
+	path := "/tmp/stats.json"
+	fp, err := os.Create(path)
+	c.Assert(err, IsNil)
+	c.Assert(fp, NotNil)
+	defer func() {
+		c.Assert(fp.Close(), IsNil)
+		c.Assert(os.Remove(path), IsNil)
+	}()
+
+	js, err := ioutil.ReadAll(resp.Body)
+	c.Assert(err, IsNil)
+	fp.Write(js)
+	ds.checkData(c, path)
+}
+
 func (ds *testDumpStatsSuite) prepareData(c *C) {
 	db, err := sql.Open("mysql", getDSN())
 	c.Assert(err, IsNil, Commentf("Error connecting"))
 	defer db.Close()
 	dbt := &DBTest{c, db}
 
+	h := ds.sh.do.StatsHandle()
 	dbt.mustExec("create database tidb")
 	dbt.mustExec("use tidb")
 	dbt.mustExec("create table test (a int, b varchar(20))")
+	h.HandleDDLEvent(<-h.DDLEventCh())
 	dbt.mustExec("create index c on test (a, b)")
-	dbt.mustExec("insert test values (1, 2)")
+	dbt.mustExec("insert test values (1, 's')")
+	c.Assert(h.DumpStatsDeltaToKV(), IsNil)
 	dbt.mustExec("analyze table test")
+	dbt.mustExec("insert into test(a,b) values (1, 'v'),(3, 'vvv'),(5, 'vv')")
+	is := ds.sh.do.InfoSchema()
+	c.Assert(h.DumpStatsDeltaToKV(), IsNil)
+	c.Assert(h.Update(is), IsNil)
 }
 
 func (ds *testDumpStatsSuite) checkData(c *C, path string) {
@@ -109,19 +130,29 @@ func (ds *testDumpStatsSuite) checkData(c *C, path string) {
 		config.Strict = false
 	}))
 	c.Assert(err, IsNil, Commentf("Error connecting"))
-	defer db.Close()
 	dbt := &DBTest{c, db}
+	defer func() {
+		dbt.mustExec("drop database tidb")
+		dbt.mustExec("truncate table mysql.stats_meta")
+		dbt.mustExec("truncate table mysql.stats_histograms")
+		dbt.mustExec("truncate table mysql.stats_buckets")
+		db.Close()
+	}()
+
 	dbt.mustExec("use tidb")
 	dbt.mustExec("drop stats test")
 	_, err = dbt.db.Exec(fmt.Sprintf("load stats '%s'", path))
 	c.Assert(err, IsNil)
 
-	rows := dbt.mustQuery("show stats_histograms")
+	rows := dbt.mustQuery("show stats_meta")
 	dbt.Check(rows.Next(), IsTrue, Commentf("unexpected data"))
 	var dbName, tableName string
+	var modifyCount, count int64
 	var other interface{}
-	err = rows.Scan(&dbName, &tableName, &other, &other, &other, &other, &other, &other)
+	err = rows.Scan(&dbName, &tableName, &other, &modifyCount, &count)
 	dbt.Check(err, IsNil)
 	dbt.Check(dbName, Equals, "tidb")
 	dbt.Check(tableName, Equals, "test")
+	dbt.Check(modifyCount, Equals, int64(3))
+	dbt.Check(count, Equals, int64(4))
 }

statistics/dump.go

@@ -15,6 +15,7 @@ package statistics
 
 import (
 	"github.com/juju/errors"
+	"github.com/pingcap/tidb/infoschema"
 	"github.com/pingcap/tidb/model"
 	"github.com/pingcap/tidb/mysql"
 	"github.com/pingcap/tidb/sessionctx/stmtctx"
@@ -30,7 +31,6 @@ type JSONTable struct {
 	Indices      map[string]*jsonColumn `json:"indices"`
 	Count        int64                  `json:"count"`
 	ModifyCount  int64                  `json:"modify_count"`
-	Version      uint64                 `json:"version"`
 }
 
 type jsonColumn struct {
@@ -60,14 +60,16 @@ func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo) (*JS
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
+	if tbl == nil {
+		return nil, nil
+	}
 	jsonTbl := &JSONTable{
 		DatabaseName: dbName,
 		TableName:    tableInfo.Name.L,
 		Columns:      make(map[string]*jsonColumn, len(tbl.Columns)),
 		Indices:      make(map[string]*jsonColumn, len(tbl.Indices)),
 		Count:        tbl.Count,
 		ModifyCount:  tbl.ModifyCount,
-		Version:      tbl.Version,
 	}
 	for _, col := range tbl.Columns {
 		hist, err := col.ConvertTo(new(stmtctx.StatementContext), types.NewFieldType(mysql.TypeBlob))
@@ -83,17 +85,83 @@ func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo) (*JS
 	return jsonTbl, nil
 }
 
-// LoadStatsFromJSON load statistic from json.
-func (h *Handle) LoadStatsFromJSON(tableInfo *model.TableInfo, jsonTbl *JSONTable) (*Table, error) {
+// LoadStatsFromJSON will load statistic from JSONTable, and save it to the storage.
+func (h *Handle) LoadStatsFromJSON(is infoschema.InfoSchema, jsonTbl *JSONTable) error {
+	tableInfo, err := is.TableByName(model.NewCIStr(jsonTbl.DatabaseName), model.NewCIStr(jsonTbl.TableName))
+	if err != nil {
+		return errors.Trace(err)
+	}
+	tbl, err := h.LoadStatsFromJSONToTable(tableInfo.Meta(), jsonTbl)
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	if h.Lease > 0 {
+		hists := make([]*Histogram, 0, len(tbl.Columns))
+		cms := make([]*CMSketch, 0, len(tbl.Columns))
+		for _, col := range tbl.Columns {
+			hists = append(hists, &col.Histogram)
+			cms = append(cms, col.CMSketch)
+		}
+		h.AnalyzeResultCh() <- &AnalyzeResult{
+			TableID: tbl.TableID,
+			Hist:    hists,
+			Cms:     cms,
+			Count:   tbl.Count,
+			IsIndex: 0,
+			Err:     nil,
+		}
+
+		hists = make([]*Histogram, 0, len(tbl.Indices))
+		cms = make([]*CMSketch, 0, len(tbl.Indices))
+		for _, idx := range tbl.Indices {
+			hists = append(hists, &idx.Histogram)
+			cms = append(cms, idx.CMSketch)
+		}
+		h.AnalyzeResultCh() <- &AnalyzeResult{
+			TableID: tbl.TableID,
+			Hist:    hists,
+			Cms:     cms,
+			Count:   tbl.Count,
+			IsIndex: 1,
+			Err:     nil,
+		}
+
+		h.LoadMetaCh() <- &LoadMeta{
+			TableID:     tbl.TableID,
+			Count:       tbl.Count,
+			ModifyCount: tbl.ModifyCount,
+		}
+		return errors.Trace(err)
+	}
+	for _, col := range tbl.Columns {
+		err = SaveStatsToStorage(h.ctx, tbl.TableID, tbl.Count, 0, &col.Histogram, col.CMSketch)
+		if err != nil {
+			return errors.Trace(err)
+		}
+	}
+	for _, idx := range tbl.Indices {
+		err = SaveStatsToStorage(h.ctx, tbl.TableID, tbl.Count, 1, &idx.Histogram, idx.CMSketch)
+		if err != nil {
+			return errors.Trace(err)
+		}
+	}
+	err = SaveMetaToStorage(h.ctx, tbl.TableID, tbl.Count, tbl.ModifyCount)
+	if err != nil {
+		return errors.Trace(err)
+	}
+	return errors.Trace(h.Update(is))
+}
+
+// LoadStatsFromJSONToTable load statistic from JSONTable and return the Table of statistic.
+func (h *Handle) LoadStatsFromJSONToTable(tableInfo *model.TableInfo, jsonTbl *JSONTable) (*Table, error) {
 	tbl := &Table{
 		TableID:     tableInfo.ID,
 		Columns:     make(map[int64]*Column, len(jsonTbl.Columns)),
 		Indices:     make(map[int64]*Index, len(jsonTbl.Indices)),
 		Count:       jsonTbl.Count,
-		Version:     jsonTbl.Version,
 		ModifyCount: jsonTbl.ModifyCount,
 	}
-
 	for id, jsonIdx := range jsonTbl.Indices {
 		for _, idxInfo := range tableInfo.Indices {
 			if idxInfo.Name.L != id {
@@ -132,3 +200,10 @@ func (h *Handle) LoadStatsFromJSON(tableInfo *model.TableInfo, jsonTbl *JSONTabl
 	}
 	return tbl, nil
 }
+
+// LoadMeta is the statistic meta loaded from json file.
+type LoadMeta struct {
+	TableID     int64
+	Count       int64
+	ModifyCount int64
+}