Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

statistics: dump FMSketch to KV only for partition table with dynamic prune mode (#24453) #24539

Closed
wants to merge 9 commits into from
Closed

statistics: dump FMSketch to KV only for partition table with dynamic prune mode (#24453) #24539

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
eb7e3b5
cherry pick #24453 to release-5.0
rebelice May 11, 2021
87178eb
resolve conflicts
rebelice May 13, 2021
4852fa6
fix test
rebelice May 13, 2021
133abdf
refine test
rebelice May 13, 2021
025b20b
Merge branch 'release-5.0' into release-5.0-3fed33fc1735
rebelice May 13, 2021
c5e4140
Merge branch 'release-5.0' into release-5.0-3fed33fc1735
qw4990 Jun 21, 2021
a440cba
Merge branch 'release-5.0' into release-5.0-3fed33fc1735
ti-chi-bot Sep 1, 2021
fc5dca9
Merge branch 'release-5.0' into release-5.0-3fed33fc1735
ti-chi-bot Sep 2, 2021
2be6551
Merge branch 'release-5.0' into release-5.0-3fed33fc1735
ti-chi-bot Sep 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions executor/analyze.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error {
globalStatsMap[globalStatsID] = globalStatsInfo{result.IsIndex, hg.ID, result.StatsVer}
}
}
err1 := statsHandle.SaveStatsToStorage(statisticsID, result.Count, result.IsIndex, hg, result.Cms[i], result.TopNs[i], result.Fms[i], result.StatsVer, 1)
err1 := statsHandle.SaveStatsToStorage(statisticsID, result.Count, result.IsIndex, hg, result.Cms[i], result.TopNs[i], result.Fms[i], result.StatsVer, 1, result.TableID.IsPartitionTable() && needGlobalStats)
if err1 != nil {
err = err1
logutil.Logger(ctx).Error("save stats to storage failed", zap.Error(err))
Expand Down Expand Up @@ -181,7 +181,8 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error {
}
for i := 0; i < globalStats.Num; i++ {
hg, cms, topN, fms := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i], globalStats.Fms[i]
err = statsHandle.SaveStatsToStorage(globalStatsID.tableID, globalStats.Count, info.isIndex, hg, cms, topN, fms, info.statsVersion, 1)
// fms for global stats doesn't need to dump to kv.
err = statsHandle.SaveStatsToStorage(globalStatsID.tableID, globalStats.Count, info.isIndex, hg, cms, topN, fms, info.statsVersion, 1, false)
if err != nil {
logutil.Logger(ctx).Error("save global-level stats to storage failed", zap.Error(err))
}
Expand Down
6 changes: 4 additions & 2 deletions statistics/handle/dump.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,13 +230,15 @@ func (h *Handle) loadStatsFromJSON(tableInfo *model.TableInfo, physicalID int64,
}

for _, col := range tbl.Columns {
err = h.SaveStatsToStorage(tbl.PhysicalID, tbl.Count, 0, &col.Histogram, col.CMSketch, col.TopN, col.FMSketch, int(col.StatsVer), 1)
// loadStatsFromJSON doesn't support partition table now.
err = h.SaveStatsToStorage(tbl.PhysicalID, tbl.Count, 0, &col.Histogram, col.CMSketch, col.TopN, col.FMSketch, int(col.StatsVer), 1, false)
if err != nil {
return errors.Trace(err)
}
}
for _, idx := range tbl.Indices {
err = h.SaveStatsToStorage(tbl.PhysicalID, tbl.Count, 1, &idx.Histogram, idx.CMSketch, idx.TopN, nil, int(idx.StatsVer), 1)
// loadStatsFromJSON doesn't support partition table now.
err = h.SaveStatsToStorage(tbl.PhysicalID, tbl.Count, 1, &idx.Histogram, idx.CMSketch, idx.TopN, nil, int(idx.StatsVer), 1, false)
if err != nil {
return errors.Trace(err)
}
Expand Down
2 changes: 1 addition & 1 deletion statistics/handle/dump_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ func (s *testStatsSuite) TestDumpCMSketchWithTopN(c *C) {
cms, _, _, _ := statistics.NewCMSketchAndTopN(5, 2048, fakeData, 20, 100)

stat := h.GetTableStats(tableInfo)
err = h.SaveStatsToStorage(tableInfo.ID, 1, 0, &stat.Columns[tableInfo.Columns[0].ID].Histogram, cms, nil, nil, statistics.Version2, 1)
err = h.SaveStatsToStorage(tableInfo.ID, 1, 0, &stat.Columns[tableInfo.Columns[0].ID].Histogram, cms, nil, nil, statistics.Version2, 1, false)
c.Assert(err, IsNil)
c.Assert(h.Update(is), IsNil)

Expand Down
4 changes: 2 additions & 2 deletions statistics/handle/handle.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -954,7 +954,7 @@ func (h *Handle) extendedStatsFromStorage(reader *statsReader, table *statistics
}

// SaveStatsToStorage saves the stats to storage.
func (h *Handle) SaveStatsToStorage(tableID int64, count int64, isIndex int, hg *statistics.Histogram, cms *statistics.CMSketch, topN *statistics.TopN, fms *statistics.FMSketch, statsVersion int, isAnalyzed int64) (err error) {
func (h *Handle) SaveStatsToStorage(tableID int64, count int64, isIndex int, hg *statistics.Histogram, cms *statistics.CMSketch, topN *statistics.TopN, fms *statistics.FMSketch, statsVersion int, isAnalyzed int64, needDumpFMS bool) (err error) {
h.mu.Lock()
defer h.mu.Unlock()
ctx := context.TODO()
Expand Down Expand Up @@ -1003,7 +1003,7 @@ func (h *Handle) SaveStatsToStorage(tableID int64, count int64, isIndex int, hg
if _, err := exec.ExecuteInternal(ctx, "delete from mysql.stats_fm_sketch where table_id = %? and is_index = %? and hist_id = %?", tableID, isIndex, hg.ID); err != nil {
return err
}
if fmSketch != nil {
if fmSketch != nil && needDumpFMS {
if _, err = exec.ExecuteInternal(ctx, "insert into mysql.stats_fm_sketch (table_id, is_index, hist_id, value) values (%?, %?, %?, %?)", tableID, isIndex, hg.ID, fmSketch); err != nil {
return err
}
Expand Down
49 changes: 45 additions & 4 deletions statistics/handle/handle_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2121,17 +2121,58 @@ func (s *testStatsSuite) TestDuplicateFMSketch(c *C) {
defer cleanEnv(c, s.store, s.do)
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("create table t(a int, b int, c int)")
tk.MustExec("set @@tidb_partition_prune_mode='dynamic'")
defer tk.MustExec("set @@tidb_partition_prune_mode='static'")
tk.MustExec("create table t(a int, b int, c int) partition by hash(a) partitions 3")
tk.MustExec("insert into t values (1, 1, 1)")
tk.MustExec("analyze table t")
tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("3"))
tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("9"))
tk.MustExec("analyze table t")
tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("3"))
tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("9"))

tk.MustExec("alter table t drop column a")
s.do.StatsHandle().SetLastUpdateVersion(s.do.StatsHandle().LastUpdateVersion() + 1)
c.Assert(s.do.StatsHandle().GCStats(s.do.InfoSchema(), time.Duration(0)), IsNil)
tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("2"))
tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("6"))
}

func (s *testStatsSuite) TestIssues24401(c *C) {
defer cleanEnv(c, s.store, s.do)
testKit := testkit.NewTestKit(c, s.store)
testKit.MustExec("use test")

// normal table with static prune mode
testKit.MustExec("set @@tidb_partition_prune_mode='static'")
testKit.MustExec("set @@tidb_analyze_version=2")
defer testKit.MustExec("set @@tidb_analyze_version=1")
testKit.MustExec("create table t(a int, index(a))")
testKit.MustExec("insert into t values (1), (2), (3)")
testKit.MustExec("analyze table t")
testKit.MustQuery("select * from mysql.stats_fm_sketch").Check(testkit.Rows())

// partition table with static prune mode
testKit.MustExec("create table tp(a int, index(a)) partition by hash(a) partitions 3")
testKit.MustExec("insert into tp values (1), (2), (3)")
testKit.MustExec("analyze table tp")
testKit.MustQuery("select * from mysql.stats_fm_sketch").Check(testkit.Rows())

// normal table with dynamic prune mode
testKit.MustExec("set @@tidb_partition_prune_mode='dynamic'")
defer testKit.MustExec("set @@tidb_partition_prune_mode='static'")
testKit.MustExec("analyze table t")
testKit.MustQuery("select * from mysql.stats_fm_sketch").Check(testkit.Rows())

// partition table with dynamic prune mode
testKit.MustExec("analyze table tp")
rows := testKit.MustQuery("select * from mysql.stats_fm_sketch").Rows()
lenRows := len(rows)
c.Assert(lenRows, Equals, 3)

// check fm-sketch won't increase infinitely
testKit.MustExec("insert into tp values (10), (20), (30), (12), (23), (23), (4344)")
testKit.MustExec("analyze table tp")
rows = testKit.MustQuery("select * from mysql.stats_fm_sketch").Rows()
c.Assert(len(rows), Equals, lenRows)
}

func (s *testStatsSuite) TestStatsCacheUpdateSkip(c *C) {
Expand Down
11 changes: 6 additions & 5 deletions statistics/handle/update.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -750,11 +750,11 @@ func (h *Handle) handleSingleHistogramUpdate(is infoschema.InfoSchema, rows []ch
return nil
}
var tbl *statistics.Table
if table.Meta().GetPartitionInfo() == nil || h.CurrentPruneMode() == variable.Dynamic {
tbl = h.GetTableStats(table.Meta())
} else {
tbl = h.GetPartitionStats(table.Meta(), physicalTableID)
// feedback for partition is not ready
if table.Meta().GetPartitionInfo() != nil {
return nil
}
tbl = h.GetTableStats(table.Meta())
var cms *statistics.CMSketch
var hist *statistics.Histogram
var topN *statistics.TopN
Expand Down Expand Up @@ -822,7 +822,8 @@ func (h *Handle) deleteOutdatedFeedback(tableID, histID, isIndex int64) error {

func (h *Handle) dumpStatsUpdateToKV(tableID, isIndex int64, q *statistics.QueryFeedback, hist *statistics.Histogram, cms *statistics.CMSketch, topN *statistics.TopN, fms *statistics.FMSketch, statsVersion int64) error {
hist = statistics.UpdateHistogram(hist, q, int(statsVersion))
err := h.SaveStatsToStorage(tableID, -1, int(isIndex), hist, cms, topN, fms, int(statsVersion), 0)
// feedback for partition is not ready.
err := h.SaveStatsToStorage(tableID, -1, int(isIndex), hist, cms, topN, fms, int(statsVersion), 0, false)
metrics.UpdateStatsCounter.WithLabelValues(metrics.RetLabel(err)).Inc()
return errors.Trace(err)
}
Expand Down