statistics: support dump/load correlation of histogram (#10573)

server/statistics_handler_test.go

@@ -106,6 +106,7 @@ func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) {
 	c.Assert(err, IsNil)
 	fp.Write(js)
 	ds.checkData(c, path)
+	ds.checkCorrelation(c)
 
 	// sleep for 1 seconds to ensure the existence of tidb.test
 	time.Sleep(time.Second)
@@ -179,6 +180,34 @@ func (ds *testDumpStatsSuite) prepare4DumpHistoryStats(c *C) {
 	dbt.mustExec("create table tidb.test (a int, b varchar(20))")
 }
 
+func (ds *testDumpStatsSuite) checkCorrelation(c *C) {
+	db, err := sql.Open("mysql", getDSN(nil))
+	c.Assert(err, IsNil, Commentf("Error connecting"))
+	dbt := &DBTest{c, db}
+	defer db.Close()
+
+	dbt.mustExec("use tidb")
+	rows := dbt.mustQuery("SELECT tidb_table_id FROM information_schema.tables WHERE table_name = 'test' AND table_schema = 'tidb'")
+	var tableID int64
+	if rows.Next() {
+		rows.Scan(&tableID)
+		dbt.Check(rows.Next(), IsFalse, Commentf("unexpected data"))
+	} else {
+		dbt.Error("no data")
+	}
+	rows.Close()
+	rows = dbt.mustQuery("select correlation from mysql.stats_histograms where table_id = ? and hist_id = 1 and is_index = 0", tableID)
+	if rows.Next() {
+		var corr float64
+		rows.Scan(&corr)
+		dbt.Check(corr, Equals, float64(1))
+		dbt.Check(rows.Next(), IsFalse, Commentf("unexpected data"))
+	} else {
+		dbt.Error("no data")
+	}
+	rows.Close()
+}
+
 func (ds *testDumpStatsSuite) checkData(c *C, path string) {
 	db, err := sql.Open("mysql", getDSN(func(config *mysql.Config) {
 		config.AllowAllFiles = true

statistics/handle/dump.go

@@ -44,6 +44,7 @@ type jsonColumn struct {
 	NullCount         int64           `json:"null_count"`
 	TotColSize        int64           `json:"tot_col_size"`
 	LastUpdateVersion uint64          `json:"last_update_version"`
+	Correlation       float64         `json:"correlation"`
 }
 
 func dumpJSONCol(hist *statistics.Histogram, CMSketch *statistics.CMSketch) *jsonColumn {
@@ -52,6 +53,7 @@ func dumpJSONCol(hist *statistics.Histogram, CMSketch *statistics.CMSketch) *jso
 		NullCount:         hist.NullCount,
 		TotColSize:        hist.TotColSize,
 		LastUpdateVersion: hist.LastUpdateVersion,
+		Correlation:       hist.Correlation,
 	}
 	if CMSketch != nil {
 		jsonCol.CMSketch = statistics.CMSketchToProto(CMSketch)
@@ -191,7 +193,7 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *J
 				continue
 			}
 			hist := statistics.HistogramFromProto(jsonIdx.Histogram)
-			hist.ID, hist.NullCount, hist.LastUpdateVersion = idxInfo.ID, jsonIdx.NullCount, jsonIdx.LastUpdateVersion
+			hist.ID, hist.NullCount, hist.LastUpdateVersion, hist.Correlation = idxInfo.ID, jsonIdx.NullCount, jsonIdx.LastUpdateVersion, jsonIdx.Correlation
 			idx := &statistics.Index{
 				Histogram: *hist,
 				CMSketch:  statistics.CMSketchFromProto(jsonIdx.CMSketch),
@@ -213,7 +215,7 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *J
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
-			hist.ID, hist.NullCount, hist.LastUpdateVersion, hist.TotColSize = colInfo.ID, jsonCol.NullCount, jsonCol.LastUpdateVersion, jsonCol.TotColSize
+			hist.ID, hist.NullCount, hist.LastUpdateVersion, hist.TotColSize, hist.Correlation = colInfo.ID, jsonCol.NullCount, jsonCol.LastUpdateVersion, jsonCol.TotColSize, jsonCol.Correlation
 			col := &statistics.Column{
 				PhysicalID: physicalID,
 				Histogram:  *hist,

statistics/histogram.go

@@ -162,6 +162,7 @@ func (hg *Histogram) DecodeTo(tp *types.FieldType, timeZone *time.Location) erro
 // ConvertTo converts the histogram bucket values into `Tp`.
 func (hg *Histogram) ConvertTo(sc *stmtctx.StatementContext, tp *types.FieldType) (*Histogram, error) {
 	hist := NewHistogram(hg.ID, hg.NDV, hg.NullCount, hg.LastUpdateVersion, tp, hg.Len(), hg.TotColSize)
+	hist.Correlation = hg.Correlation
 	iter := chunk.NewIterator4Chunk(hg.Bounds)
 	for row := iter.Begin(); row != iter.End(); row = iter.Next() {
 		d := row.GetDatum(0, hg.Tp)