stats: fix dump stats

executor/load_stats.go

@@ -113,7 +113,8 @@ func (e *LoadStatsInfo) Update(data []byte) error {
 			Cms:     cms,
 			Count:   tbl.Count,
 			IsIndex: 0,
-			Err:     nil}
+			Err:     nil,
+		}
 
 		hists = make([]*statistics.Histogram, 0, len(tbl.Indices))
 		cms = make([]*statistics.CMSketch, 0, len(tbl.Indices))
@@ -127,9 +128,10 @@ func (e *LoadStatsInfo) Update(data []byte) error {
 			Cms:     cms,
 			Count:   tbl.Count,
 			IsIndex: 1,
-			Err:     nil}
-
-		return nil
+			Err:     nil,
+		}
+		err = statistics.SaveMetaToStorage(e.Ctx, tbl.TableID, tbl.Count, tbl.ModifyCount, tbl.Version)
+		return errors.Trace(err)
 	}
 	for _, col := range tbl.Columns {
 		err = statistics.SaveStatsToStorage(e.Ctx, tbl.TableID, tbl.Count, 0, &col.Histogram, col.CMSketch)
@@ -143,6 +145,10 @@ func (e *LoadStatsInfo) Update(data []byte) error {
 			return errors.Trace(err)
 		}
 	}
+	err = statistics.SaveMetaToStorage(e.Ctx, tbl.TableID, tbl.Count, tbl.ModifyCount, tbl.Version)
+	if err != nil {
+		return errors.Trace(err)
+	}
 	err = h.Update(GetInfoSchema(e.Ctx))
 	return errors.Trace(err)
 }

server/statistics_handler_test.go

@@ -23,17 +23,57 @@ import (
 	"github.com/go-sql-driver/mysql"
 	. "github.com/pingcap/check"
 	"github.com/pingcap/tidb/config"
+	"github.com/pingcap/tidb/domain"
+	"github.com/pingcap/tidb/kv"
+	"github.com/pingcap/tidb/model"
 	"github.com/pingcap/tidb/session"
+	"github.com/pingcap/tidb/statistics"
 	"github.com/pingcap/tidb/store/mockstore"
 	"github.com/pingcap/tidb/store/mockstore/mocktikv"
+	"github.com/pingcap/tidb/util/testkit"
+	"github.com/pingcap/tidb/util/testleak"
 )
 
 type testDumpStatsSuite struct {
 	server *Server
+	store  kv.Storage
+	do     *domain.Domain
+	db     *sql.DB
+	tk     *testkit.TestKit
 }
 
 var _ = Suite(new(testDumpStatsSuite))
 
+func (ds *testDumpStatsSuite) SetUpSuite(c *C) {
+	testleak.BeforeTest()
+	var err error
+	ds.store, ds.do, err = newStoreWithBootstrap()
+	c.Assert(err, IsNil)
+	ds.db, err = sql.Open("mysql", getDSN(func(config *mysql.Config) {
+		config.AllowAllFiles = true
+		config.Strict = false
+	}))
+	c.Assert(err, IsNil, Commentf("Error connecting"))
+	ds.tk = testkit.NewTestKit(c, ds.store)
+}
+
+func (ds *testDumpStatsSuite) TearDownSuite(c *C) {
+	dbt := ds.tk
+	dbt.MustExec("use tidb")
+	r := ds.tk.MustQuery("show tables")
+	for _, tb := range r.Rows() {
+		tableName := tb[0]
+		ds.tk.MustExec(fmt.Sprintf("drop table %v", tableName))
+	}
+	ds.do.StatsHandle().Clear()
+	dbt.MustExec("truncate table mysql.stats_meta")
+	dbt.MustExec("truncate table mysql.stats_histograms")
+	dbt.MustExec("truncate table mysql.stats_buckets")
+	ds.db.Close()
+	ds.store.Close()
+	testleak.AfterTest(c)()
+}
+
 func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) {
 	ds.startServer(c)
 	ds.prepareData(c)
@@ -90,38 +130,58 @@ func (ds *testDumpStatsSuite) stopServer(c *C) {
 }
 
 func (ds *testDumpStatsSuite) prepareData(c *C) {
-	db, err := sql.Open("mysql", getDSN())
-	c.Assert(err, IsNil, Commentf("Error connecting"))
-	defer db.Close()
-	dbt := &DBTest{c, db}
-
-	dbt.mustExec("create database tidb")
-	dbt.mustExec("use tidb")
-	dbt.mustExec("create table test (a int, b varchar(20))")
-	dbt.mustExec("create index c on test (a, b)")
-	dbt.mustExec("insert test values (1, 2)")
-	dbt.mustExec("analyze table test")
+	dbt := ds.tk
+	dbt.MustExec("create database tidb")
+	dbt.MustExec("use tidb")
+	dbt.MustExec("create table test (a int, b varchar(20))")
+	dbt.MustExec("create index c on test (a, b)")
+	dbt.MustExec("insert test values (1, 's')")
+	dbt.MustExec("analyze table test")
+	dbt.MustExec("insert into test(a,b) values (1, 'v'),(3, 'vvv'),(5, 'vv')")
+
+	is := ds.do.InfoSchema()
+	h := ds.do.StatsHandle()
+	h.DumpStatsDeltaToKV()
+	h.Update(is)
 }
 
 func (ds *testDumpStatsSuite) checkData(c *C, path string) {
-	db, err := sql.Open("mysql", getDSN(func(config *mysql.Config) {
-		config.AllowAllFiles = true
-		config.Strict = false
-	}))
-	c.Assert(err, IsNil, Commentf("Error connecting"))
-	defer db.Close()
-	dbt := &DBTest{c, db}
-	dbt.mustExec("use tidb")
-	dbt.mustExec("drop stats test")
-	_, err = dbt.db.Exec(fmt.Sprintf("load stats '%s'", path))
+	is := ds.do.InfoSchema()
+	h := ds.do.StatsHandle()
+	tableInfo, err := is.TableByName(model.NewCIStr("tidb"), model.NewCIStr("test"))
 	c.Assert(err, IsNil)
+	tbl := h.GetTableStats(tableInfo.Meta())
 
-	rows := dbt.mustQuery("show stats_histograms")
-	dbt.Check(rows.Next(), IsTrue, Commentf("unexpected data"))
-	var dbName, tableName string
-	var other interface{}
-	err = rows.Scan(&dbName, &tableName, &other, &other, &other, &other, &other, &other)
-	dbt.Check(err, IsNil)
-	dbt.Check(dbName, Equals, "tidb")
-	dbt.Check(tableName, Equals, "test")
+	dbt := ds.tk
+	dbt.MustExec("use tidb")
+	dbt.MustExec("drop stats test")
+	_, err = dbt.Exec(fmt.Sprintf("load stats '%s'", path))
+	c.Assert(err, IsNil)
+	loadTbl := h.GetTableStats(tableInfo.Meta())
+	assertTableEqual(c, loadTbl, tbl)
+}
+
+func assertTableEqual(c *C, a *statistics.Table, b *statistics.Table) {
+	c.Assert(a.Version, Equals, b.Version)
+	c.Assert(a.Count, Equals, b.Count)
+	c.Assert(a.ModifyCount, Equals, b.ModifyCount)
+	c.Assert(len(a.Columns), Equals, len(b.Columns))
+	for i := range a.Columns {
+		c.Assert(a.Columns[i].Count, Equals, b.Columns[i].Count)
+		c.Assert(statistics.HistogramEqual(&a.Columns[i].Histogram, &b.Columns[i].Histogram, false), IsTrue)
+		if a.Columns[i].CMSketch == nil {
+			c.Assert(b.Columns[i].CMSketch, IsNil)
+		} else {
+			c.Assert(a.Columns[i].CMSketch.Equal(b.Columns[i].CMSketch), IsTrue)
+		}
+	}
+	c.Assert(len(a.Indices), Equals, len(b.Indices))
+	for i := range a.Indices {
+		c.Assert(statistics.HistogramEqual(&a.Indices[i].Histogram, &b.Indices[i].Histogram, false), IsTrue)
+		if a.Columns[i].CMSketch == nil {
+			c.Assert(b.Columns[i].CMSketch, IsNil)
+		} else {
+			c.Assert(a.Columns[i].CMSketch.Equal(b.Columns[i].CMSketch), IsTrue)
+		}
+	}
 }

statistics/dump_test.go

@@ -51,16 +51,19 @@ func (s *testDumpStatsSuite) TestConversion(c *C) {
 	tk.MustExec("create index c on t(a,b)")
 	tk.MustExec("insert into t(a,b) values (3, 1),(2, 1),(1, 10)")
 	tk.MustExec("analyze table t")
-
+	tk.MustExec("insert into t(a,b) values (1, 1),(3, 1),(5, 10)")
 	is := s.do.InfoSchema()
 	h := s.do.StatsHandle()
+	h.DumpStatsDeltaToKV()
 	h.Update(is)
+
 	tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
 	c.Assert(err, IsNil)
 	jsonTbl, err := h.DumpStatsToJSON("test", tableInfo.Meta())
 	c.Assert(err, IsNil)
 	loadTbl, err := h.LoadStatsFromJSON(tableInfo.Meta(), jsonTbl)
 	c.Assert(err, IsNil)
+
 	tbl := h.GetTableStats(tableInfo.Meta())
 	assertTableEqual(c, loadTbl, tbl)
 }

statistics/handle_test.go

@@ -111,6 +111,9 @@ func (s *testStatsCacheSuite) TestStatsCache(c *C) {
 }
 
 func assertTableEqual(c *C, a *statistics.Table, b *statistics.Table) {
+	c.Assert(a.Version, Equals, b.Version)
+	c.Assert(a.Count, Equals, b.Count)
+	c.Assert(a.ModifyCount, Equals, b.ModifyCount)
 	c.Assert(len(a.Columns), Equals, len(b.Columns))
 	for i := range a.Columns {
 		c.Assert(a.Columns[i].Count, Equals, b.Columns[i].Count)

statistics/histogram.go

@@ -33,7 +33,7 @@ import (
 	"github.com/pingcap/tidb/util/codec"
 	"github.com/pingcap/tidb/util/ranger"
 	"github.com/pingcap/tidb/util/sqlexec"
-	tipb "github.com/pingcap/tipb/go-tipb"
+	"github.com/pingcap/tipb/go-tipb"
 	"golang.org/x/net/context"
 )
 
@@ -247,6 +247,25 @@ func SaveStatsToStorage(sctx sessionctx.Context, tableID int64, count int64, isI
 	return errors.Trace(err)
 }
 
+// SaveMetaToStorage will save stats_meta to storage.
+func SaveMetaToStorage(sctx sessionctx.Context, tableID, count, modifyCount int64, version uint64) error {
+	ctx := context.TODO()
+	exec := sctx.(sqlexec.SQLExecutor)
+	_, err := exec.Execute(ctx, "begin")
+	if err != nil {
+		return errors.Trace(err)
+	}
+	var sql string
+	sql = fmt.Sprintf("replace into mysql.stats_meta (version, table_id, count, modify_count) values (%d, %d, %d, %d)", version, tableID, count, modifyCount)
+	if _, err = exec.Execute(ctx, sql); err != nil {
+		_, err1 := exec.Execute(ctx, "rollback")
+		terror.Log(errors.Trace(err1))
+		return errors.Trace(err)
+	}
+	_, err = exec.Execute(ctx, "commit")
+	return errors.Trace(err)
+}
+
 func histogramFromStorage(ctx sessionctx.Context, tableID int64, colID int64, tp *types.FieldType, distinct int64, isIndex int, ver uint64, nullCount int64, totColSize int64) (*Histogram, error) {
 	selSQL := fmt.Sprintf("select count, repeats, lower_bound, upper_bound from mysql.stats_buckets where table_id = %d and is_index = %d and hist_id = %d order by bucket_id", tableID, isIndex, colID)
 	rows, fields, err := ctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQL(ctx, selSQL)

statistics/table.go

@@ -52,11 +52,13 @@ type Table struct {
 
 func (t *Table) copy() *Table {
 	nt := &Table{
-		TableID: t.TableID,
-		Count:   t.Count,
-		Pseudo:  t.Pseudo,
-		Columns: make(map[int64]*Column),
-		Indices: make(map[int64]*Index),
+		TableID:     t.TableID,
+		Count:       t.Count,
+		ModifyCount: t.ModifyCount,
+		Version:     t.Version,
+		Pseudo:      t.Pseudo,
+		Columns:     make(map[int64]*Column),
+		Indices:     make(map[int64]*Index),
 	}
 	for id, col := range t.Columns {
 		nt.Columns[id] = col