From 167723cae1b9c0ac2092d14d23799481ca9594f0 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Mon, 8 Mar 2021 16:43:31 +0800 Subject: [PATCH 01/11] add test cases for auto anlyze and feedback --- statistics/handle/handle_test.go | 61 ++++++++++++++++++++++++++++++++ statistics/handle/update_test.go | 1 + 2 files changed, 62 insertions(+) diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 1986ba0b89694..285b1705a17c8 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -1228,6 +1228,67 @@ func (s *statsSerialSuite) TestGCIndexUsageInformation(c *C) { tk.MustQuery(querySQL).Check(testkit.Rows("0")) } +func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { + defer cleanEnv(c, s.store, s.do) + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t;") + tk.MustExec("set @@tidb_analyze_version = 2;") + tk.MustExec("set global tidb_auto_analyze_ratio = 0.0") + tk.MustExec(`create table t (a int, key(a)) partition by range (a) ( + partition p0 values less than (10), + partition p1 values less than (20), + partition p2 values less than (30) + );`) + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic';") + statistics.FeedbackProbability.Store(0) + tk.MustExec("insert into t values (1), (5), (6), (11), (15), (21), (25);") + tk.MustExec("analyze table t;") + result := tk.MustQuery("show stats_meta where table_name = 't'").Sort() + c.Assert(len(result.Rows()), Equals, 4) + c.Assert(result.Rows()[0][5], Equals, "7") + c.Assert(result.Rows()[1][5], Equals, "3") + c.Assert(result.Rows()[2][5], Equals, "2") + c.Assert(result.Rows()[3][5], Equals, "2") + statistics.FeedbackProbability.Store(1) + tk.MustExec("analyze table t;") + result = tk.MustQuery("show stats_meta where table_name = 't'").Sort() + c.Assert(len(result.Rows()), Equals, 4) + c.Assert(result.Rows()[0][5], Equals, "7") + c.Assert(result.Rows()[1][5], Equals, "3") + c.Assert(result.Rows()[2][5], Equals, "2") + c.Assert(result.Rows()[3][5], Equals, "2") + + for i := 0; i < 200; i++ { + tk.MustExec("insert into t values (1),(12),(4),(22),(5)") + } + h := s.do.StatsHandle() + is := s.do.InfoSchema() + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + c.Assert(err, IsNil) + tblInfo := table.Meta() + statsTblBefore := h.GetTableStats(tblInfo) + // trigger feedback + tk.MustExec("select * from t where t.a <= 5 order by a desc") + tk.MustExec("select a from t partition(p2) where t.a > 20;") + + h.UpdateStatsByLocalFeedback(s.do.InfoSchema()) + err = h.DumpStatsFeedbackToKV() + c.Assert(err, IsNil) + err = h.HandleUpdateStats(s.do.InfoSchema()) + c.Assert(err, IsNil) + statsTblAfter := h.GetTableStats(tblInfo) + // assert that statistics not changed + assertTableEqual(c, statsTblBefore, statsTblAfter) + + result = tk.MustQuery("show stats_meta where table_name = 't'").Sort() + c.Assert(len(result.Rows()), Equals, 4) + c.Assert(result.Rows()[0][5], Equals, "7") + c.Assert(result.Rows()[1][5], Equals, "3") + c.Assert(result.Rows()[2][5], Equals, "2") + c.Assert(result.Rows()[3][5], Equals, "2") +} + func (s *testStatsSuite) TestExtendedStatsPartitionTable(c *C) { defer cleanEnv(c, s.store, s.do) tk := testkit.NewTestKit(c, s.store) diff --git a/statistics/handle/update_test.go b/statistics/handle/update_test.go index 006fda24c7d45..7299a5e35d859 100644 --- a/statistics/handle/update_test.go +++ b/statistics/handle/update_test.go @@ -2050,6 +2050,7 @@ func (s *testSerialStatsSuite) TestAutoUpdatePartitionInDynamicOnlyMode(c *C) { testKit := testkit.NewTestKit(c, s.store) testkit.WithPruneMode(testKit, variable.DynamicOnly, func() { testKit.MustExec("use test") + testKit.MustExec("set @@tidb_analyze_version = 2;") testKit.MustExec("drop table if exists t") testKit.MustExec(`create table t (a int, b varchar(10), index idx_ab(a, b)) partition by range (a) ( From d866ce2eab1a7d1df197970d1d345b355b0ed1b5 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Mon, 8 Mar 2021 16:44:26 +0800 Subject: [PATCH 02/11] fix test --- statistics/handle/handle_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 285b1705a17c8..04dff4d0bbb63 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -1235,7 +1235,7 @@ func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { tk.MustExec("drop table if exists t;") tk.MustExec("set @@tidb_analyze_version = 2;") tk.MustExec("set global tidb_auto_analyze_ratio = 0.0") - tk.MustExec(`create table t (a int, key(a)) partition by range (a) ( + tk.MustExec(`create table t (a int, index idx(a)) partition by range (a) ( partition p0 values less than (10), partition p1 values less than (20), partition p2 values less than (30) @@ -1270,7 +1270,7 @@ func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { statsTblBefore := h.GetTableStats(tblInfo) // trigger feedback tk.MustExec("select * from t where t.a <= 5 order by a desc") - tk.MustExec("select a from t partition(p2) where t.a > 20;") + tk.MustExec("select a from t partition(p2) use index(idx) where t.a > 20;") h.UpdateStatsByLocalFeedback(s.do.InfoSchema()) err = h.DumpStatsFeedbackToKV() From fd5adaf58a2d9a9d8c798d81b021a36f0618c452 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Tue, 9 Mar 2021 10:19:46 +0800 Subject: [PATCH 03/11] statistics: test the auto analyze and feedback for the global-level stats --- statistics/handle/handle_test.go | 103 ++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 36 deletions(-) diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 04dff4d0bbb63..002cf15b641ed 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -1230,47 +1230,56 @@ func (s *statsSerialSuite) TestGCIndexUsageInformation(c *C) { func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { defer cleanEnv(c, s.store, s.do) - tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test") - tk.MustExec("drop table if exists t;") - tk.MustExec("set @@tidb_analyze_version = 2;") - tk.MustExec("set global tidb_auto_analyze_ratio = 0.0") - tk.MustExec(`create table t (a int, index idx(a)) partition by range (a) ( - partition p0 values less than (10), - partition p1 values less than (20), - partition p2 values less than (30) - );`) - tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic';") - statistics.FeedbackProbability.Store(0) - tk.MustExec("insert into t values (1), (5), (6), (11), (15), (21), (25);") - tk.MustExec("analyze table t;") - result := tk.MustQuery("show stats_meta where table_name = 't'").Sort() - c.Assert(len(result.Rows()), Equals, 4) - c.Assert(result.Rows()[0][5], Equals, "7") - c.Assert(result.Rows()[1][5], Equals, "3") - c.Assert(result.Rows()[2][5], Equals, "2") - c.Assert(result.Rows()[3][5], Equals, "2") + testKit := testkit.NewTestKit(c, s.store) + testKit.MustExec("use test") + testKit.MustExec("set global tidb_analyze_version = 1") + testKit.MustExec("set @@tidb_analyze_version = 1") + + oriProbability := statistics.FeedbackProbability.Load() + oriNumber := statistics.MaxNumberOfRanges + oriMinLogCount := handle.MinLogScanCount + oriErrorRate := handle.MinLogErrorRate + defer func() { + statistics.FeedbackProbability.Store(oriProbability) + statistics.MaxNumberOfRanges = oriNumber + handle.MinLogScanCount = oriMinLogCount + handle.MinLogErrorRate = oriErrorRate + }() + // Case 1: You can't set tidb_analyze_version to 2 if feedback is enabled. + // Note: if we want to set @@tidb_partition_prune_mode = 'dynamic'. We must set idb_analyze_version to 2 first. statistics.FeedbackProbability.Store(1) - tk.MustExec("analyze table t;") - result = tk.MustQuery("show stats_meta where table_name = 't'").Sort() - c.Assert(len(result.Rows()), Equals, 4) - c.Assert(result.Rows()[0][5], Equals, "7") - c.Assert(result.Rows()[1][5], Equals, "3") - c.Assert(result.Rows()[2][5], Equals, "2") - c.Assert(result.Rows()[3][5], Equals, "2") + testKit.MustQuery("select @@tidb_analyze_version").Check(testkit.Rows("1")) + testKit.MustExec("set @@tidb_analyze_version = 2") + testKit.MustQuery("show warnings").Check(testkit.Rows(`Error 1105 variable tidb_analyze_version not updated because analyze version 2 is incompatible with query feedback. Please consider setting feedback-probability to 0.0 in config file to disable query feedback`)) + testKit.MustQuery("select @@tidb_analyze_version").Check(testkit.Rows("1")) + // Case 2: Feedback wouldn't be applied on version 2 and global-level statistics. + statistics.FeedbackProbability.Store(0) + testKit.MustExec("set @@tidb_analyze_version = 2") + testKit.MustExec("set @@tidb_partition_prune_mode = 'dynamic';") + testKit.MustQuery("select @@tidb_analyze_version").Check(testkit.Rows("2")) + testKit.MustExec("create table t (a bigint(64), b bigint(64), index idx(b)) PARTITION BY HASH(a) PARTITIONS 2;") for i := 0; i < 200; i++ { - tk.MustExec("insert into t values (1),(12),(4),(22),(5)") + testKit.MustExec("insert into t values (1,2),(2,2),(4,5),(2,3),(3,4)") } + testKit.MustExec("analyze table t with 0 topn") h := s.do.StatsHandle() is := s.do.InfoSchema() table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) tblInfo := table.Meta() + testKit.MustExec("analyze table t") + err = h.Update(s.do.InfoSchema()) + c.Assert(err, IsNil) statsTblBefore := h.GetTableStats(tblInfo) + statistics.FeedbackProbability.Store(1) + // make the statistics inaccurate. + for i := 0; i < 200; i++ { + testKit.MustExec("insert into t values (3,4), (3,4), (3,4), (3,4), (3,4)") + } // trigger feedback - tk.MustExec("select * from t where t.a <= 5 order by a desc") - tk.MustExec("select a from t partition(p2) use index(idx) where t.a > 20;") + testKit.MustExec("select * from t where t.a <= 5 order by a desc") + testKit.MustExec("select b from t use index(idx) where t.b <= 5") h.UpdateStatsByLocalFeedback(s.do.InfoSchema()) err = h.DumpStatsFeedbackToKV() @@ -1281,12 +1290,34 @@ func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { // assert that statistics not changed assertTableEqual(c, statsTblBefore, statsTblAfter) - result = tk.MustQuery("show stats_meta where table_name = 't'").Sort() - c.Assert(len(result.Rows()), Equals, 4) - c.Assert(result.Rows()[0][5], Equals, "7") - c.Assert(result.Rows()[1][5], Equals, "3") - c.Assert(result.Rows()[2][5], Equals, "2") - c.Assert(result.Rows()[3][5], Equals, "2") + // Case 3: Feedback is still effective on version 1 and partition-level statistics. + testKit.MustExec("set tidb_analyze_version = 1") + testKit.MustExec("set @@tidb_partition_prune_mode = 'static';") + testKit.MustExec("create table t1 (a bigint(64), b bigint(64), index idx(b)) PARTITION BY HASH(a) PARTITIONS 2") + for i := 0; i < 200; i++ { + testKit.MustExec("insert into t1 values (1,2),(2,2),(4,5),(2,3),(3,4)") + } + testKit.MustExec("analyze table t1 with 0 topn") + // make the statistics inaccurate. + for i := 0; i < 200; i++ { + testKit.MustExec("insert into t1 values (3,4), (3,4), (3,4), (3,4), (3,4)") + } + is = s.do.InfoSchema() + table, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t1")) + c.Assert(err, IsNil) + tblInfo = table.Meta() + statsTblBefore = h.GetTableStats(tblInfo) + // trigger feedback + testKit.MustExec("select b from t1 use index(idx) where t1.b <= 5") + + h.UpdateStatsByLocalFeedback(s.do.InfoSchema()) + err = h.DumpStatsFeedbackToKV() + c.Assert(err, IsNil) + err = h.HandleUpdateStats(s.do.InfoSchema()) + c.Assert(err, IsNil) + statsTblAfter = h.GetTableStats(tblInfo) + // assert that statistics changed(feedback worked) + c.Assert(statistics.HistogramEqual(&statsTblBefore.Indices[1].Histogram, &statsTblAfter.Indices[1].Histogram, false), IsFalse) } func (s *testStatsSuite) TestExtendedStatsPartitionTable(c *C) { From d863fc16bd82d126e6f81abe7d4f2a0e3776b143 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Tue, 9 Mar 2021 10:49:39 +0800 Subject: [PATCH 04/11] fix ut and refactor the code --- statistics/handle/handle_test.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 5f44b1e18a816..f7ba29e2663aa 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -1605,7 +1605,6 @@ func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { defer cleanEnv(c, s.store, s.do) testKit := testkit.NewTestKit(c, s.store) testKit.MustExec("use test") - testKit.MustExec("set global tidb_analyze_version = 1") testKit.MustExec("set @@tidb_analyze_version = 1") oriProbability := statistics.FeedbackProbability.Load() @@ -1619,7 +1618,7 @@ func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { handle.MinLogErrorRate = oriErrorRate }() // Case 1: You can't set tidb_analyze_version to 2 if feedback is enabled. - // Note: if we want to set @@tidb_partition_prune_mode = 'dynamic'. We must set idb_analyze_version to 2 first. + // Note: if we want to set @@tidb_partition_prune_mode = 'dynamic'. We must set tidb_analyze_version to 2 first. We have already tested this. statistics.FeedbackProbability.Store(1) testKit.MustQuery("select @@tidb_analyze_version").Check(testkit.Rows("1")) testKit.MustExec("set @@tidb_analyze_version = 2") @@ -1651,8 +1650,10 @@ func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { testKit.MustExec("insert into t values (3,4), (3,4), (3,4), (3,4), (3,4)") } // trigger feedback - testKit.MustExec("select * from t where t.a <= 5 order by a desc") - testKit.MustExec("select b from t use index(idx) where t.b <= 5") + testKit.MustExec("select * from t partition(p0) where t.a <= 3;") + testKit.MustExec("select * from t partition(p1) where t.a <= 3;") + testKit.MustExec("select * from t where t.b <= 3 order by a;") + testKit.MustExec("select * from t use index(idx) where t.b <= 3;") h.UpdateStatsByLocalFeedback(s.do.InfoSchema()) err = h.DumpStatsFeedbackToKV() @@ -1681,7 +1682,10 @@ func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { tblInfo = table.Meta() statsTblBefore = h.GetTableStats(tblInfo) // trigger feedback - testKit.MustExec("select b from t1 use index(idx) where t1.b <= 5") + testKit.MustExec("select * from t partition(p0) where t.a <= 3;") + testKit.MustExec("select * from t partition(p1) where t.a <= 3;") + testKit.MustExec("select * from t where t.b <= 3 order by a;") + testKit.MustExec("select * from t use index(idx) where t.b <= 3;") h.UpdateStatsByLocalFeedback(s.do.InfoSchema()) err = h.DumpStatsFeedbackToKV() @@ -1689,8 +1693,9 @@ func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { err = h.HandleUpdateStats(s.do.InfoSchema()) c.Assert(err, IsNil) statsTblAfter = h.GetTableStats(tblInfo) - // assert that statistics changed(feedback worked) - c.Assert(statistics.HistogramEqual(&statsTblBefore.Indices[1].Histogram, &statsTblAfter.Indices[1].Histogram, false), IsFalse) + // assert that statistics not changed + // the feedback can not work for the partition table + assertTableEqual(c, statsTblBefore, statsTblAfter) } func (s *testStatsSuite) TestExtendedStatsPartitionTable(c *C) { From ad341bcd23f601a69e55c35f3cb46efdb7fa2a79 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Tue, 9 Mar 2021 11:28:37 +0800 Subject: [PATCH 05/11] forbid the feedback for the partition table --- executor/builder.go | 12 ++++++++++++ statistics/handle/handle_test.go | 19 ++++++++++--------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index 9b25a075e8e56..a0e559874a388 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -2559,6 +2559,10 @@ func buildNoRangeTableReader(b *executorBuilder, v *plannercore.PhysicalTableRea e.feedback = statistics.NewQueryFeedback(getFeedbackStatsTableID(e.ctx, tbl), ts.Hist, int64(ts.StatsCount()), ts.Desc) } collect := statistics.CollectFeedback(b.ctx.GetSessionVars().StmtCtx, e.feedback, len(ts.Ranges)) + // Do not collect the feedback when the table is the partition table. + if collect && tbl.Meta().Partition != nil { + collect = false + } if !collect { e.feedback.Invalidate() } @@ -2830,6 +2834,10 @@ func buildNoRangeIndexReader(b *executorBuilder, v *plannercore.PhysicalIndexRea e.feedback = statistics.NewQueryFeedback(tblID, is.Hist, int64(is.StatsCount()), is.Desc) } collect := statistics.CollectFeedback(b.ctx.GetSessionVars().StmtCtx, e.feedback, len(is.Ranges)) + // Do not collect the feedback when the table is the partition table. + if collect && tbl.Meta().Partition != nil { + collect = false + } if !collect { e.feedback.Invalidate() } @@ -2968,6 +2976,10 @@ func buildNoRangeIndexLookUpReader(b *executorBuilder, v *plannercore.PhysicalIn collectTable := false e.tableRequest.CollectRangeCounts = &collectTable collectIndex := statistics.CollectFeedback(b.ctx.GetSessionVars().StmtCtx, e.feedback, len(is.Ranges)) + // Do not collect the feedback when the table is the partition table. + if collectIndex && tbl.Meta().Partition != nil { + collectIndex = false + } if !collectIndex { e.feedback.Invalidate() } diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index f7ba29e2663aa..df094a40b9c42 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -1650,10 +1650,10 @@ func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { testKit.MustExec("insert into t values (3,4), (3,4), (3,4), (3,4), (3,4)") } // trigger feedback - testKit.MustExec("select * from t partition(p0) where t.a <= 3;") - testKit.MustExec("select * from t partition(p1) where t.a <= 3;") - testKit.MustExec("select * from t where t.b <= 3 order by a;") - testKit.MustExec("select * from t use index(idx) where t.b <= 3;") + testKit.MustExec("select b from t partition(p0) use index(idx) where t.b <= 3;") + testKit.MustExec("select b from t partition(p1) use index(idx) where t.b <= 3;") + testKit.MustExec("select b from t use index(idx) where t.b <= 3 order by b;") + testKit.MustExec("select b from t use index(idx) where t.b <= 3;") h.UpdateStatsByLocalFeedback(s.do.InfoSchema()) err = h.DumpStatsFeedbackToKV() @@ -1662,6 +1662,7 @@ func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { c.Assert(err, IsNil) statsTblAfter := h.GetTableStats(tblInfo) // assert that statistics not changed + // the feedback can not work for the partition table in dynamic mode assertTableEqual(c, statsTblBefore, statsTblAfter) // Case 3: Feedback is still effective on version 1 and partition-level statistics. @@ -1682,10 +1683,10 @@ func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { tblInfo = table.Meta() statsTblBefore = h.GetTableStats(tblInfo) // trigger feedback - testKit.MustExec("select * from t partition(p0) where t.a <= 3;") - testKit.MustExec("select * from t partition(p1) where t.a <= 3;") - testKit.MustExec("select * from t where t.b <= 3 order by a;") - testKit.MustExec("select * from t use index(idx) where t.b <= 3;") + testKit.MustExec("select b from t partition(p0) use index(idx) where t.b <= 3;") + testKit.MustExec("select b from t partition(p1) use index(idx) where t.b <= 3;") + testKit.MustExec("select b from t use index(idx) where t.b <= 3 order by b;") + testKit.MustExec("select b from t use index(idx) where t.b <= 3;") h.UpdateStatsByLocalFeedback(s.do.InfoSchema()) err = h.DumpStatsFeedbackToKV() @@ -1694,7 +1695,7 @@ func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { c.Assert(err, IsNil) statsTblAfter = h.GetTableStats(tblInfo) // assert that statistics not changed - // the feedback can not work for the partition table + // the feedback can not work for the partition table in static mode assertTableEqual(c, statsTblBefore, statsTblAfter) } From 80711d4019a65173a2f2b7ed8168da8b289fb487 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Tue, 9 Mar 2021 11:29:51 +0800 Subject: [PATCH 06/11] change the comment --- statistics/handle/handle_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index df094a40b9c42..aef800030406c 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -1665,7 +1665,7 @@ func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { // the feedback can not work for the partition table in dynamic mode assertTableEqual(c, statsTblBefore, statsTblAfter) - // Case 3: Feedback is still effective on version 1 and partition-level statistics. + // Case 3: Feedback is also not effective on version 1 and partition-level statistics. testKit.MustExec("set tidb_analyze_version = 1") testKit.MustExec("set @@tidb_partition_prune_mode = 'static';") testKit.MustExec("create table t1 (a bigint(64), b bigint(64), index idx(b)) PARTITION BY HASH(a) PARTITIONS 2") From e8c97e575cafa076eaea1147ff605341fc5c467e Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Tue, 9 Mar 2021 14:27:24 +0800 Subject: [PATCH 07/11] update the test --- statistics/handle/update_test.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/statistics/handle/update_test.go b/statistics/handle/update_test.go index 7299a5e35d859..26a58928b035f 100644 --- a/statistics/handle/update_test.go +++ b/statistics/handle/update_test.go @@ -773,8 +773,8 @@ func (s *testStatsSuite) TestUpdatePartitionErrorRate(c *C) { c.Assert(h.Update(is), IsNil) tbl = h.GetPartitionStats(tblInfo, pid) - // The error rate of this column is not larger than MaxErrorRate now. - c.Assert(tbl.Columns[aID].NotAccurate(), IsFalse) + // Feedback will not take effect under partition table. + c.Assert(tbl.Columns[aID].NotAccurate(), IsTrue) } func appendBucket(h *statistics.Histogram, l, r int64) { @@ -979,6 +979,7 @@ func (s *testStatsSuite) TestQueryFeedbackForPartition(c *C) { handle.MinLogErrorRate = 0 h := s.do.StatsHandle() + // Feedback will not take effect under partition table. tests := []struct { sql string hist string @@ -987,23 +988,23 @@ func (s *testStatsSuite) TestQueryFeedbackForPartition(c *C) { { // test primary key feedback sql: "select * from t where t.a <= 5", - hist: "column:1 ndv:2 totColSize:0\n" + - "num: 1 lower_bound: -9223372036854775808 upper_bound: 2 repeats: 0 ndv: 0\n" + - "num: 1 lower_bound: 2 upper_bound: 5 repeats: 0 ndv: 0", + hist: "column:1 ndv:2 totColSize:2\n" + + "num: 1 lower_bound: 1 upper_bound: 1 repeats: 1 ndv: 0\n" + + "num: 1 lower_bound: 2 upper_bound: 2 repeats: 1 ndv: 0", idxCols: 0, }, { // test index feedback by double read sql: "select * from t use index(idx) where t.b <= 5", hist: "index:1 ndv:1\n" + - "num: 2 lower_bound: -inf upper_bound: 6 repeats: 0 ndv: 0", + "num: 2 lower_bound: 2 upper_bound: 2 repeats: 2 ndv: 0", idxCols: 1, }, { // test index feedback by single read sql: "select b from t use index(idx) where t.b <= 5", hist: "index:1 ndv:1\n" + - "num: 2 lower_bound: -inf upper_bound: 6 repeats: 0 ndv: 0", + "num: 2 lower_bound: 2 upper_bound: 2 repeats: 2 ndv: 0", idxCols: 1, }, } @@ -1180,10 +1181,11 @@ func (s *testStatsSuite) TestUpdatePartitionStatsByLocalFeedback(c *C) { pid := tblInfo.Partition.Definitions[0].ID tbl := h.GetPartitionStats(tblInfo, pid) + // // Feedback will not take effect under partition table. c.Assert(tbl.Columns[tblInfo.Columns[0].ID].ToString(0), Equals, "column:1 ndv:3 totColSize:0\n"+ "num: 1 lower_bound: 1 upper_bound: 1 repeats: 1 ndv: 0\n"+ - "num: 2 lower_bound: 2 upper_bound: 4 repeats: 0 ndv: 0\n"+ - "num: 1 lower_bound: 4 upper_bound: 9223372036854775807 repeats: 0 ndv: 0") + "num: 1 lower_bound: 2 upper_bound: 2 repeats: 1 ndv: 0\n"+ + "num: 1 lower_bound: 4 upper_bound: 4 repeats: 1 ndv: 0") } func (s *testStatsSuite) TestFeedbackWithStatsVer2(c *C) { From f9148599274572d0d19c3f52b842cc384fc8387c Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Wed, 10 Mar 2021 14:47:06 +0800 Subject: [PATCH 08/11] address comments --- executor/analyze_test.go | 6 +++--- statistics/handle/handle_test.go | 2 +- statistics/handle/update.go | 4 ++++ statistics/integration_test.go | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/executor/analyze_test.go b/executor/analyze_test.go index a4a8499dd277a..556f0b339a3de 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -508,8 +508,8 @@ func (s *testFastAnalyze) TestFastAnalyze(c *C) { */ // test fast analyze in dynamic mode - tk.MustExec("set @@tidb_analyze_version = 2;") - tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic';") + tk.MustExec("set @@session.tidb_analyze_version = 2;") + tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic';") tk.MustExec("drop table if exists t4;") tk.MustExec("create table t4(a int, b int) PARTITION BY HASH(a) PARTITIONS 2;") tk.MustExec("insert into t4 values(1,1),(3,3),(4,4),(2,2),(5,5);") @@ -632,7 +632,7 @@ func (s *testSuite1) testAnalyzeIncremental(tk *testkit.TestKit, c *C) { c.Assert(tblStats.Indices[tblInfo.Indices[0].ID].QueryBytes(val), Equals, uint64(1)) // test analyzeIndexIncremental for global-level stats; - tk.MustExec("set @@tidb_analyze_version = 2;") + tk.MustExec("set @@session.tidb_analyze_version = 2;") tk.MustExec("set @@tidb_partition_prune_mode = 'static';") tk.MustExec("drop table if exists t;") tk.MustExec(`create table t (a int, b int, primary key(a), index idx(b)) partition by range (a) ( diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index e2adf9e57308d..30dc6c28e02b1 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -1658,7 +1658,7 @@ func (s *statsSerialSuite) TestGCIndexUsageInformation(c *C) { tk.MustQuery(querySQL).Check(testkit.Rows("0")) } -func (s *testStatsSuite) TestFeedbackWithGlobalStats(c *C) { +func (s *statsSerialSuite) TestFeedbackWithGlobalStats(c *C) { defer cleanEnv(c, s.store, s.do) testKit := testkit.NewTestKit(c, s.store) testKit.MustExec("use test") diff --git a/statistics/handle/update.go b/statistics/handle/update.go index 9a5aa8980f967..c772bf5e1891d 100644 --- a/statistics/handle/update.go +++ b/statistics/handle/update.go @@ -567,6 +567,10 @@ OUTER: if !ok { continue } + if table.Meta().Partition != nil { + // If the table is partition table, the feedback will not work. + continue + } tblStats := h.GetPartitionStats(table.Meta(), fb.PhysicalID) newTblStats := tblStats.Copy() if fb.Tp == statistics.IndexType { diff --git a/statistics/integration_test.go b/statistics/integration_test.go index d6a3be81f6323..1a2b1727ba8e0 100644 --- a/statistics/integration_test.go +++ b/statistics/integration_test.go @@ -230,7 +230,7 @@ func (s *testIntegrationSuite) TestGlobalStats(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t;") - tk.MustExec("set @@tidb_analyze_version = 2;") + tk.MustExec("set @@session.tidb_analyze_version = 2;") tk.MustExec(`create table t (a int, key(a)) partition by range (a) ( partition p0 values less than (10), partition p1 values less than (20), From 4027f719c060c1253cbd33af7e519ed70d8f5485 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Wed, 10 Mar 2021 15:30:00 +0800 Subject: [PATCH 09/11] address comments --- statistics/handle/update_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statistics/handle/update_test.go b/statistics/handle/update_test.go index dddf1e742d071..f2f1919d41262 100644 --- a/statistics/handle/update_test.go +++ b/statistics/handle/update_test.go @@ -1216,7 +1216,7 @@ func (s *testStatsSuite) TestUpdatePartitionStatsByLocalFeedback(c *C) { pid := tblInfo.Partition.Definitions[0].ID tbl := h.GetPartitionStats(tblInfo, pid) - // // Feedback will not take effect under partition table. + // Feedback will not take effect under partition table. c.Assert(tbl.Columns[tblInfo.Columns[0].ID].ToString(0), Equals, "column:1 ndv:3 totColSize:0\n"+ "num: 1 lower_bound: 1 upper_bound: 1 repeats: 1 ndv: 0\n"+ "num: 1 lower_bound: 2 upper_bound: 2 repeats: 1 ndv: 0\n"+ From f7e22a16ea4a69cd35e986b3de9fd4e1a5b0c7b3 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Wed, 10 Mar 2021 15:43:21 +0800 Subject: [PATCH 10/11] address comments --- statistics/handle/handle_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 30dc6c28e02b1..01a869a94ab10 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -1740,10 +1740,10 @@ func (s *statsSerialSuite) TestFeedbackWithGlobalStats(c *C) { tblInfo = table.Meta() statsTblBefore = h.GetTableStats(tblInfo) // trigger feedback - testKit.MustExec("select b from t partition(p0) use index(idx) where t.b <= 3;") - testKit.MustExec("select b from t partition(p1) use index(idx) where t.b <= 3;") - testKit.MustExec("select b from t use index(idx) where t.b <= 3 order by b;") - testKit.MustExec("select b from t use index(idx) where t.b <= 3;") + testKit.MustExec("select b from t1 partition(p0) use index(idx) where t1.b <= 3;") + testKit.MustExec("select b from t1 partition(p1) use index(idx) where t1.b <= 3;") + testKit.MustExec("select b from t1 use index(idx) where t1.b <= 3 order by b;") + testKit.MustExec("select b from t1 use index(idx) where t1.b <= 3;") h.UpdateStatsByLocalFeedback(s.do.InfoSchema()) err = h.DumpStatsFeedbackToKV() From d8e609c4f8efeb4a7d4a1e8d32746f2e0103c781 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Wed, 10 Mar 2021 16:00:05 +0800 Subject: [PATCH 11/11] address comments --- statistics/handle/handle_test.go | 38 ++++++++++++++------------------ 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 01a869a94ab10..9cc96cc33cba4 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -1682,6 +1682,21 @@ func (s *statsSerialSuite) TestFeedbackWithGlobalStats(c *C) { testKit.MustQuery("show warnings").Check(testkit.Rows(`Error 1105 variable tidb_analyze_version not updated because analyze version 2 is incompatible with query feedback. Please consider setting feedback-probability to 0.0 in config file to disable query feedback`)) testKit.MustQuery("select @@tidb_analyze_version").Check(testkit.Rows("1")) + h := s.do.StatsHandle() + var err error + // checkFeedbackOnPartitionTable is used to check whether the statistics are the same as before. + checkFeedbackOnPartitionTable := func(statsBefore *statistics.Table, tblInfo *model.TableInfo) { + h.UpdateStatsByLocalFeedback(s.do.InfoSchema()) + err = h.DumpStatsFeedbackToKV() + c.Assert(err, IsNil) + err = h.HandleUpdateStats(s.do.InfoSchema()) + c.Assert(err, IsNil) + statsTblAfter := h.GetTableStats(tblInfo) + // assert that statistics not changed + // the feedback can not work for the partition table in both static and dynamic mode + assertTableEqual(c, statsBefore, statsTblAfter) + } + // Case 2: Feedback wouldn't be applied on version 2 and global-level statistics. statistics.FeedbackProbability.Store(0) testKit.MustExec("set @@tidb_analyze_version = 2") @@ -1692,7 +1707,6 @@ func (s *statsSerialSuite) TestFeedbackWithGlobalStats(c *C) { testKit.MustExec("insert into t values (1,2),(2,2),(4,5),(2,3),(3,4)") } testKit.MustExec("analyze table t with 0 topn") - h := s.do.StatsHandle() is := s.do.InfoSchema() table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) @@ -1711,16 +1725,7 @@ func (s *statsSerialSuite) TestFeedbackWithGlobalStats(c *C) { testKit.MustExec("select b from t partition(p1) use index(idx) where t.b <= 3;") testKit.MustExec("select b from t use index(idx) where t.b <= 3 order by b;") testKit.MustExec("select b from t use index(idx) where t.b <= 3;") - - h.UpdateStatsByLocalFeedback(s.do.InfoSchema()) - err = h.DumpStatsFeedbackToKV() - c.Assert(err, IsNil) - err = h.HandleUpdateStats(s.do.InfoSchema()) - c.Assert(err, IsNil) - statsTblAfter := h.GetTableStats(tblInfo) - // assert that statistics not changed - // the feedback can not work for the partition table in dynamic mode - assertTableEqual(c, statsTblBefore, statsTblAfter) + checkFeedbackOnPartitionTable(statsTblBefore, tblInfo) // Case 3: Feedback is also not effective on version 1 and partition-level statistics. testKit.MustExec("set tidb_analyze_version = 1") @@ -1744,16 +1749,7 @@ func (s *statsSerialSuite) TestFeedbackWithGlobalStats(c *C) { testKit.MustExec("select b from t1 partition(p1) use index(idx) where t1.b <= 3;") testKit.MustExec("select b from t1 use index(idx) where t1.b <= 3 order by b;") testKit.MustExec("select b from t1 use index(idx) where t1.b <= 3;") - - h.UpdateStatsByLocalFeedback(s.do.InfoSchema()) - err = h.DumpStatsFeedbackToKV() - c.Assert(err, IsNil) - err = h.HandleUpdateStats(s.do.InfoSchema()) - c.Assert(err, IsNil) - statsTblAfter = h.GetTableStats(tblInfo) - // assert that statistics not changed - // the feedback can not work for the partition table in static mode - assertTableEqual(c, statsTblBefore, statsTblAfter) + checkFeedbackOnPartitionTable(statsTblBefore, tblInfo) } func (s *testStatsSuite) TestExtendedStatsPartitionTable(c *C) {