From 089124e42a03a7a3b7270b476518bcd7e7f15f7d Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Wed, 24 Feb 2021 18:21:30 +0800 Subject: [PATCH 01/13] 1.fix the wrong result for the stats.Count 2.make fast analyze work in dynamic mode --- executor/analyze.go | 13 +++++++++++-- executor/analyze_test.go | 15 +++++++++++++++ planner/core/planbuilder.go | 10 ---------- statistics/handle/handle.go | 16 ++++++++-------- statistics/table.go | 6 +++--- 5 files changed, 37 insertions(+), 23 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index 22021bc49fa18..be55a36867a0e 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -18,6 +18,7 @@ import ( "context" "math" "math/rand" + "regexp" "runtime" "sort" "strconv" @@ -171,6 +172,15 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { sc := e.ctx.GetSessionVars().StmtCtx globalStats, err := statsHandle.MergePartitionStats2GlobalStats(sc, infoschema.GetInfoSchema(e.ctx), globalStatsID.tableID, info.isIndex, info.idxID) if err != nil { + errMessage := err.Error() + match, err1 := regexp.MatchString("^\\[stats\\] build global-level stats failed due to missing partition-level stats.", errMessage) + if err1 != nil { + return err1 + } + if match { + sc.AppendWarning(err) + continue + } return err } for i := 0; i < globalStats.Num; i++ { @@ -809,8 +819,7 @@ func (e *AnalyzeFastExec) calculateEstimateSampleStep() (err error) { sql := new(strings.Builder) sqlexec.MustFormatSQL(sql, "select count(*) from %n.%n", dbInfo.Name.L, e.tblInfo.Name.L) - pruneMode := variable.PartitionPruneMode(e.ctx.GetSessionVars().PartitionPruneMode.Load()) - if pruneMode != variable.DynamicOnly && e.tblInfo.ID != e.tableID.GetStatisticsID() { + if e.tblInfo.ID != e.tableID.GetStatisticsID() { for _, definition := range e.tblInfo.Partition.Definitions { if definition.ID == e.tableID.GetStatisticsID() { sqlexec.MustFormatSQL(sql, " partition(%n)", definition.Name.L) diff --git a/executor/analyze_test.go b/executor/analyze_test.go index 60bdc0ac9972c..7048af8bc1def 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -473,6 +473,21 @@ func (s *testFastAnalyze) TestFastAnalyze(c *C) { "└─IndexRangeScan 2.00 cop[tikv] table:t3, partition:p1, index:k(v) range:[3,3], keep order:false", )) tk.MustExec(`set @@tidb_partition_prune_mode='` + string(variable.DynamicOnly) + `'`) + + // test fast analyze in dynamic mode + tk.MustExec("drop table if exists t4;") + tk.MustExec("create table t4(a int, b int) PARTITION BY HASH(a) PARTITIONS 2;") + tk.MustExec("insert into t4 values(1,1),(3,3),(4,4),(2,2),(5,5);") + tk.MustExec(`set @@tidb_partition_prune_mode='` + string(variable.DynamicOnly) + `'`) + // Because the statistics of partition p1 are missing, the construction of global-level stats will fail. + tk.MustExec("analyze table t4 partition p0;") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 [stats] build global-level stats failed due to missing partition-level stats.(tableID: 63, partitionID: 64)")) + tk.MustExec("analyze table t4 partition p1;") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 [stats] build global-level stats failed due to missing partition-level stats.(tableID: 63, partitionID: 65)")) + //tk.MustQuery(`explain format = 'brief' select * from t4 where a = 3`).Check(testkit.Rows( + // "IndexReader 2.00 root index:IndexRangeScan", + // "└─IndexRangeScan 2.00 cop[tikv] table:t3, partition:p1, index:k(v) range:[3,3], keep order:false", + //)) } func (s *testSuite1) TestIssue15993(c *C) { diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 492d3ac4fb382..80290c1e07b5a 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1677,11 +1677,6 @@ func getPhysicalIDsAndPartitionNames(tblInfo *model.TableInfo, partitionNames [] func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt, opts map[ast.AnalyzeOptionType]uint64, version int) (Plan, error) { p := &Analyze{Opts: opts} - pruneMode := variable.PartitionPruneMode(b.ctx.GetSessionVars().PartitionPruneMode.Load()) - if len(as.PartitionNames) > 0 && pruneMode == variable.DynamicOnly { - logutil.BgLogger().Info("analyze partition didn't affect in dynamic-prune-mode", zap.String("partitions", as.PartitionNames[0].L)) - return p, nil - } for _, tbl := range as.TableNames { if tbl.TableInfo.IsView() { return nil, errors.Errorf("analyze view %s is not supported now.", tbl.Name.O) @@ -1754,11 +1749,6 @@ func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt, opts map[ast.A func (b *PlanBuilder) buildAnalyzeIndex(as *ast.AnalyzeTableStmt, opts map[ast.AnalyzeOptionType]uint64, version int) (Plan, error) { p := &Analyze{Opts: opts} tblInfo := as.TableNames[0].TableInfo - pruneMode := variable.PartitionPruneMode(b.ctx.GetSessionVars().PartitionPruneMode.Load()) - if len(as.PartitionNames) > 0 && pruneMode == variable.DynamicOnly { - logutil.BgLogger().Info("analyze partition didn't affect in dynamic-prune-mode", zap.String("table", tblInfo.Name.L), zap.String("partitions", as.PartitionNames[0].L)) - return p, nil - } physicalIDs, names, err := getPhysicalIDsAndPartitionNames(tblInfo, as.PartitionNames) if err != nil { return nil, err diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 28c3400e6efa7..c414f9a514df3 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -350,22 +350,22 @@ func (h *Handle) MergePartitionStats2GlobalStats(sc *stmtctx.StatementContext, i } tableInfo := partitionTable.Meta() var partitionStats *statistics.Table - partitionStats, err = h.TableStatsFromStorage(tableInfo, partitionID, true, 0) - if err != nil { - return - } - if partitionStats == nil { - err = errors.Errorf("[stats] error occurred when read partition-level stats of the table with tableID %d and partitionID %d", physicalID, partitionID) + partitionStats = h.GetPartitionStats(tableInfo, partitionID) + if partitionStats.Pseudo { + err = errors.Errorf("[stats] build global-level stats failed due to missing partition-level stats.(tableID: %d, partitionID: %d)", physicalID, partitionID) return } - globalStats.Count += partitionStats.Count for i := 0; i < globalStats.Num; i++ { ID := tableInfo.Columns[i].ID if isIndex != 0 { // If the statistics is the index stats, we should use the index ID to replace the column ID. ID = idxID } - hg, cms, topN, fms := partitionStats.GetStatsInfo(ID, isIndex == 1) + count, hg, cms, topN, fms := partitionStats.GetStatsInfo(ID, isIndex == 1) + if i == 0 { + // In a partition, we will only update globalStats.Count once + globalStats.Count += count + } allHg[i] = append(allHg[i], hg) allCms[i] = append(allCms[i], cms) allTopN[i] = append(allTopN[i], topN) diff --git a/statistics/table.go b/statistics/table.go index 94f41879f06f0..69a71ee44801e 100644 --- a/statistics/table.go +++ b/statistics/table.go @@ -198,13 +198,13 @@ func (t *Table) ColumnByName(colName string) *Column { } // GetStatsInfo returns their statistics according to the ID of the column or index, including histogram, CMSketch, TopN and FMSketch. -func (t *Table) GetStatsInfo(ID int64, isIndex bool) (*Histogram, *CMSketch, *TopN, *FMSketch) { +func (t *Table) GetStatsInfo(ID int64, isIndex bool) (int64, *Histogram, *CMSketch, *TopN, *FMSketch) { if isIndex { idxStatsInfo := t.Indices[ID] - return idxStatsInfo.Histogram.Copy(), idxStatsInfo.CMSketch.Copy(), idxStatsInfo.TopN.Copy(), nil + return int64(idxStatsInfo.TotalRowCount()), idxStatsInfo.Histogram.Copy(), idxStatsInfo.CMSketch.Copy(), idxStatsInfo.TopN.Copy(), nil } colStatsInfo := t.Columns[ID] - return colStatsInfo.Histogram.Copy(), colStatsInfo.CMSketch.Copy(), colStatsInfo.TopN.Copy(), colStatsInfo.FMSketch.Copy() + return int64(colStatsInfo.TotalRowCount()), colStatsInfo.Histogram.Copy(), colStatsInfo.CMSketch.Copy(), colStatsInfo.TopN.Copy(), colStatsInfo.FMSketch.Copy() } type tableColumnID struct { From adc8c9a4a57e8880a345d80fe1494caa06cfadb1 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Thu, 25 Feb 2021 15:11:07 +0800 Subject: [PATCH 02/13] fix ut --- executor/analyze.go | 8 ++------ executor/analyze_test.go | 6 +++--- statistics/handle/handle.go | 9 ++++++--- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index be55a36867a0e..474d455037f35 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -18,7 +18,6 @@ import ( "context" "math" "math/rand" - "regexp" "runtime" "sort" "strconv" @@ -173,11 +172,8 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { globalStats, err := statsHandle.MergePartitionStats2GlobalStats(sc, infoschema.GetInfoSchema(e.ctx), globalStatsID.tableID, info.isIndex, info.idxID) if err != nil { errMessage := err.Error() - match, err1 := regexp.MatchString("^\\[stats\\] build global-level stats failed due to missing partition-level stats.", errMessage) - if err1 != nil { - return err1 - } - if match { + if errMessage == "[stats] build global-level stats failed due to missing partition-level stats" { + // When we find some partition-level stats are missing, we need to report warning. sc.AppendWarning(err) continue } diff --git a/executor/analyze_test.go b/executor/analyze_test.go index 7048af8bc1def..278ac3f620ca2 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -480,10 +480,10 @@ func (s *testFastAnalyze) TestFastAnalyze(c *C) { tk.MustExec("insert into t4 values(1,1),(3,3),(4,4),(2,2),(5,5);") tk.MustExec(`set @@tidb_partition_prune_mode='` + string(variable.DynamicOnly) + `'`) // Because the statistics of partition p1 are missing, the construction of global-level stats will fail. - tk.MustExec("analyze table t4 partition p0;") - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 [stats] build global-level stats failed due to missing partition-level stats.(tableID: 63, partitionID: 64)")) tk.MustExec("analyze table t4 partition p1;") - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 [stats] build global-level stats failed due to missing partition-level stats.(tableID: 63, partitionID: 65)")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 [stats] build global-level stats failed due to missing partition-level stats")) + tk.MustExec("analyze table t4 partition p0;") + tk.MustQuery("show warnings").Check(testkit.Rows("")) //tk.MustQuery(`explain format = 'brief' select * from t4 where a = 3`).Check(testkit.Rows( // "IndexReader 2.00 root index:IndexRangeScan", // "└─IndexRangeScan 2.00 cop[tikv] table:t3, partition:p1, index:k(v) range:[3,3], keep order:false", diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index c414f9a514df3..2532393b1119c 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -350,9 +350,8 @@ func (h *Handle) MergePartitionStats2GlobalStats(sc *stmtctx.StatementContext, i } tableInfo := partitionTable.Meta() var partitionStats *statistics.Table - partitionStats = h.GetPartitionStats(tableInfo, partitionID) - if partitionStats.Pseudo { - err = errors.Errorf("[stats] build global-level stats failed due to missing partition-level stats.(tableID: %d, partitionID: %d)", physicalID, partitionID) + partitionStats, err = h.TableStatsFromStorage(tableInfo, partitionID, true, 0) + if err != nil { return } for i := 0; i < globalStats.Num; i++ { @@ -366,6 +365,10 @@ func (h *Handle) MergePartitionStats2GlobalStats(sc *stmtctx.StatementContext, i // In a partition, we will only update globalStats.Count once globalStats.Count += count } + if cms == nil && topN == nil && fms == nil { + err = errors.Errorf("[stats] build global-level stats failed due to missing partition-level stats") + return + } allHg[i] = append(allHg[i], hg) allCms[i] = append(allCms[i], cms) allTopN[i] = append(allTopN[i], topN) From a0fce3afe171f76568d55eb44a93b78b1db282a4 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Thu, 25 Feb 2021 15:30:53 +0800 Subject: [PATCH 03/13] fix ut --- executor/analyze_test.go | 17 +++++++++++------ statistics/handle/handle.go | 8 ++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/executor/analyze_test.go b/executor/analyze_test.go index 278ac3f620ca2..c7455053686fd 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -478,16 +478,21 @@ func (s *testFastAnalyze) TestFastAnalyze(c *C) { tk.MustExec("drop table if exists t4;") tk.MustExec("create table t4(a int, b int) PARTITION BY HASH(a) PARTITIONS 2;") tk.MustExec("insert into t4 values(1,1),(3,3),(4,4),(2,2),(5,5);") - tk.MustExec(`set @@tidb_partition_prune_mode='` + string(variable.DynamicOnly) + `'`) // Because the statistics of partition p1 are missing, the construction of global-level stats will fail. tk.MustExec("analyze table t4 partition p1;") tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 [stats] build global-level stats failed due to missing partition-level stats")) + // Although the global-level stats build failed, we build partition-level stats for partition p1 success. + result := tk.MustQuery("show stats_meta where table_name = 't4'").Sort() + c.Assert(len(result.Rows()), Equals, 1) + c.Assert(result.Rows()[0][5], Equals, "3") + // Now, we have the partition-level stats for partition p0. We need get the stats for partition p1. And build the global-level stats. tk.MustExec("analyze table t4 partition p0;") - tk.MustQuery("show warnings").Check(testkit.Rows("")) - //tk.MustQuery(`explain format = 'brief' select * from t4 where a = 3`).Check(testkit.Rows( - // "IndexReader 2.00 root index:IndexRangeScan", - // "└─IndexRangeScan 2.00 cop[tikv] table:t3, partition:p1, index:k(v) range:[3,3], keep order:false", - //)) + tk.MustQuery("show warnings").Check(testkit.Rows()) + result = tk.MustQuery("show stats_meta where table_name = 't4'").Sort() + c.Assert(len(result.Rows()), Equals, 3) + c.Assert(result.Rows()[0][5], Equals, "5") + c.Assert(result.Rows()[1][5], Equals, "2") + c.Assert(result.Rows()[2][5], Equals, "3") } func (s *testSuite1) TestIssue15993(c *C) { diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 2532393b1119c..08085300255c1 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -354,6 +354,10 @@ func (h *Handle) MergePartitionStats2GlobalStats(sc *stmtctx.StatementContext, i if err != nil { return } + if partitionStats == nil { + err = errors.Errorf("[stats] build global-level stats failed due to missing partition-level stats") + return + } for i := 0; i < globalStats.Num; i++ { ID := tableInfo.Columns[i].ID if isIndex != 0 { @@ -365,10 +369,6 @@ func (h *Handle) MergePartitionStats2GlobalStats(sc *stmtctx.StatementContext, i // In a partition, we will only update globalStats.Count once globalStats.Count += count } - if cms == nil && topN == nil && fms == nil { - err = errors.Errorf("[stats] build global-level stats failed due to missing partition-level stats") - return - } allHg[i] = append(allHg[i], hg) allCms[i] = append(allCms[i], cms) allTopN[i] = append(allTopN[i], topN) From be6f8f9d8d026d0e191d98d420be69005e854470 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Thu, 25 Feb 2021 15:43:26 +0800 Subject: [PATCH 04/13] add some comments --- statistics/handle/handle.go | 1 + 1 file changed, 1 insertion(+) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 08085300255c1..39fc6a6c46171 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -354,6 +354,7 @@ func (h *Handle) MergePartitionStats2GlobalStats(sc *stmtctx.StatementContext, i if err != nil { return } + // if the err == nil && partitionStats == nil, it means we lack the stats about partitionID. if partitionStats == nil { err = errors.Errorf("[stats] build global-level stats failed due to missing partition-level stats") return From 8e4227605cff36c3ebfaa7a26ba7193db38492f2 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Thu, 25 Feb 2021 18:50:28 +0800 Subject: [PATCH 05/13] update some comments --- statistics/handle/handle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 39fc6a6c46171..23741679beed2 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -354,7 +354,7 @@ func (h *Handle) MergePartitionStats2GlobalStats(sc *stmtctx.StatementContext, i if err != nil { return } - // if the err == nil && partitionStats == nil, it means we lack the stats about partitionID. + // if the err == nil && partitionStats == nil, it means we lack the partition-level stats which the physicalID is equal to partitionID. if partitionStats == nil { err = errors.Errorf("[stats] build global-level stats failed due to missing partition-level stats") return From 50d73960a3dffd9d2a08b46bcd69c31d35bdc684 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Fri, 26 Feb 2021 12:17:21 +0800 Subject: [PATCH 06/13] address comments --- errno/errcode.go | 1 + errno/errname.go | 3 ++- errors.toml | 4 ++++ executor/analyze.go | 3 +-- executor/errors.go | 1 + statistics/handle/handle.go | 3 ++- 6 files changed, 11 insertions(+), 4 deletions(-) diff --git a/errno/errcode.go b/errno/errcode.go index 20837dad18498..24dc9824df518 100644 --- a/errno/errcode.go +++ b/errno/errcode.go @@ -991,6 +991,7 @@ const ( ErrInvalidTableSample = 8128 ErrJSONObjectKeyTooLong = 8129 ErrMultiStatementDisabled = 8130 + ErrBuildGlobalLevelStatsFailed = 8131 // Error codes used by TiDB ddl package ErrUnsupportedDDLOperation = 8200 diff --git a/errno/errname.go b/errno/errname.go index 76e385c255abb..bef12b1ef6e83 100644 --- a/errno/errname.go +++ b/errno/errname.go @@ -1032,7 +1032,8 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{ ErrInvalidTableSample: mysql.Message("Invalid TABLESAMPLE: %s", nil), - ErrJSONObjectKeyTooLong: mysql.Message("TiDB does not yet support JSON objects with the key length >= 65536", nil), + ErrJSONObjectKeyTooLong: mysql.Message("TiDB does not yet support JSON objects with the key length >= 65536", nil), + ErrBuildGlobalLevelStatsFailed: mysql.Message("Build global-level stats failed due to missing partition-level stats", nil), ErrInvalidPlacementSpec: mysql.Message("Invalid placement policy '%s': %s", nil), ErrPlacementPolicyCheck: mysql.Message("Placement policy didn't meet the constraint, reason: %s", nil), diff --git a/errors.toml b/errors.toml index 1848a80016c7a..23ade8fc3f925 100644 --- a/errors.toml +++ b/errors.toml @@ -1570,6 +1570,10 @@ invalid week mode format: '%v' error = ''' TiDB does not yet support JSON objects with the key length >= 65536 ''' +["types:8131"] +error = ''' +Build global-level stats failed due to missing partition-level stats +''' ["variable:1193"] error = ''' diff --git a/executor/analyze.go b/executor/analyze.go index 474d455037f35..ef8eb43411ca1 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -171,8 +171,7 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { sc := e.ctx.GetSessionVars().StmtCtx globalStats, err := statsHandle.MergePartitionStats2GlobalStats(sc, infoschema.GetInfoSchema(e.ctx), globalStatsID.tableID, info.isIndex, info.idxID) if err != nil { - errMessage := err.Error() - if errMessage == "[stats] build global-level stats failed due to missing partition-level stats" { + if ErrBuildGlobalLevelStatsFailed.Equal(err) { // When we find some partition-level stats are missing, we need to report warning. sc.AppendWarning(err) continue diff --git a/executor/errors.go b/executor/errors.go index afc64276900a7..6bb8a0bb473b1 100644 --- a/executor/errors.go +++ b/executor/errors.go @@ -44,6 +44,7 @@ var ( ErrRoleNotGranted = dbterror.ClassPrivilege.NewStd(mysql.ErrRoleNotGranted) ErrDeadlock = dbterror.ClassExecutor.NewStd(mysql.ErrLockDeadlock) ErrQueryInterrupted = dbterror.ClassExecutor.NewStd(mysql.ErrQueryInterrupted) + ErrBuildGlobalLevelStatsFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBuildGlobalLevelStatsFailed) ErrBRIEBackupFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEBackupFailed) ErrBRIERestoreFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIERestoreFailed) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 23741679beed2..eab7b0e5d0747 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -30,6 +30,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/ddl/util" + "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx" @@ -356,7 +357,7 @@ func (h *Handle) MergePartitionStats2GlobalStats(sc *stmtctx.StatementContext, i } // if the err == nil && partitionStats == nil, it means we lack the partition-level stats which the physicalID is equal to partitionID. if partitionStats == nil { - err = errors.Errorf("[stats] build global-level stats failed due to missing partition-level stats") + err = executor.ErrBuildGlobalLevelStatsFailed return } for i := 0; i < globalStats.Num; i++ { From b26ada97afca9204d71b159d76508e44c4afd282 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Fri, 26 Feb 2021 14:59:01 +0800 Subject: [PATCH 07/13] fix ut --- executor/analyze.go | 2 +- executor/errors.go | 1 - statistics/handle/handle.go | 3 +-- types/errors.go | 3 +++ 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index ef8eb43411ca1..c94130c5da7b1 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -171,7 +171,7 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { sc := e.ctx.GetSessionVars().StmtCtx globalStats, err := statsHandle.MergePartitionStats2GlobalStats(sc, infoschema.GetInfoSchema(e.ctx), globalStatsID.tableID, info.isIndex, info.idxID) if err != nil { - if ErrBuildGlobalLevelStatsFailed.Equal(err) { + if types.ErrBuildGlobalLevelStatsFailed.Equal(err) { // When we find some partition-level stats are missing, we need to report warning. sc.AppendWarning(err) continue diff --git a/executor/errors.go b/executor/errors.go index 6bb8a0bb473b1..afc64276900a7 100644 --- a/executor/errors.go +++ b/executor/errors.go @@ -44,7 +44,6 @@ var ( ErrRoleNotGranted = dbterror.ClassPrivilege.NewStd(mysql.ErrRoleNotGranted) ErrDeadlock = dbterror.ClassExecutor.NewStd(mysql.ErrLockDeadlock) ErrQueryInterrupted = dbterror.ClassExecutor.NewStd(mysql.ErrQueryInterrupted) - ErrBuildGlobalLevelStatsFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBuildGlobalLevelStatsFailed) ErrBRIEBackupFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEBackupFailed) ErrBRIERestoreFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIERestoreFailed) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index eab7b0e5d0747..4cc8535ab8451 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -30,7 +30,6 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/ddl/util" - "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx" @@ -357,7 +356,7 @@ func (h *Handle) MergePartitionStats2GlobalStats(sc *stmtctx.StatementContext, i } // if the err == nil && partitionStats == nil, it means we lack the partition-level stats which the physicalID is equal to partitionID. if partitionStats == nil { - err = executor.ErrBuildGlobalLevelStatsFailed + err = types.ErrBuildGlobalLevelStatsFailed return } for i := 0; i < globalStats.Num; i++ { diff --git a/types/errors.go b/types/errors.go index 21d6a88a25dbb..2fbf4e1e5f61d 100644 --- a/types/errors.go +++ b/types/errors.go @@ -83,4 +83,7 @@ var ( ErrWrongValue = dbterror.ClassTypes.NewStdErr(mysql.ErrTruncatedWrongValue, mysql.MySQLErrName[mysql.ErrWrongValue]) // ErrWrongValueForType is returned when the input value is in wrong format for function. ErrWrongValueForType = dbterror.ClassTypes.NewStdErr(mysql.ErrWrongValueForType, mysql.MySQLErrName[mysql.ErrWrongValueForType]) + // ErrBuildGlobalLevelStatsFailed is returned when the partition-level stats is missing and the build global-level stats fails. + // Put this error here is to prevent `import cycle not allowed`. + ErrBuildGlobalLevelStatsFailed = dbterror.ClassTypes.NewStd(mysql.ErrBuildGlobalLevelStatsFailed) ) From fb1c0488354897bc7bbde279ca7bc1ef0857b64d Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Mon, 1 Mar 2021 14:28:11 +0800 Subject: [PATCH 08/13] fix ut --- executor/analyze_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/analyze_test.go b/executor/analyze_test.go index c7455053686fd..4aff17daa7a2e 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -480,7 +480,7 @@ func (s *testFastAnalyze) TestFastAnalyze(c *C) { tk.MustExec("insert into t4 values(1,1),(3,3),(4,4),(2,2),(5,5);") // Because the statistics of partition p1 are missing, the construction of global-level stats will fail. tk.MustExec("analyze table t4 partition p1;") - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 [stats] build global-level stats failed due to missing partition-level stats")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 8131 Build global-level stats failed due to missing partition-level stats")) // Although the global-level stats build failed, we build partition-level stats for partition p1 success. result := tk.MustQuery("show stats_meta where table_name = 't4'").Sort() c.Assert(len(result.Rows()), Equals, 1) From 93218c11cb0684a698314e559dc806383fae9d62 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Mon, 1 Mar 2021 14:33:46 +0800 Subject: [PATCH 09/13] revert the change for `solve conflicts` --- executor/analyze_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/executor/analyze_test.go b/executor/analyze_test.go index 4aff17daa7a2e..55e8d1621fa12 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -52,7 +52,7 @@ var _ = Suite(&testFastAnalyze{}) func (s *testSuite1) TestAnalyzePartition(c *C) { tk := testkit.NewTestKit(c, s.store) - testkit.WithPruneMode(tk, variable.StaticOnly, func() { + testkit.WithPruneMode(tk, variable.Static, func() { tk.MustExec("use test") tk.MustExec("drop table if exists t") createTable := `CREATE TABLE t (a int, b int, c varchar(10), primary key(a), index idx(b)) @@ -464,7 +464,7 @@ func (s *testFastAnalyze) TestFastAnalyze(c *C) { "test t2 a 0 0 1 1 0 0 0", "test t2 a 0 1 2 1 18446744073709551615 18446744073709551615 0")) - tk.MustExec(`set @@tidb_partition_prune_mode='` + string(variable.StaticOnly) + `'`) + tk.MustExec(`set @@tidb_partition_prune_mode='` + string(variable.Static) + `'`) tk.MustExec(`create table t3 (id int, v int, primary key(id), index k(v)) partition by hash (id) partitions 4`) tk.MustExec(`insert into t3 values(1, 1), (2, 2), (5, 1), (9, 3), (13, 3), (17, 5), (3, 0)`) tk.MustExec(`analyze table t3`) @@ -472,7 +472,7 @@ func (s *testFastAnalyze) TestFastAnalyze(c *C) { "IndexReader 2.00 root index:IndexRangeScan", "└─IndexRangeScan 2.00 cop[tikv] table:t3, partition:p1, index:k(v) range:[3,3], keep order:false", )) - tk.MustExec(`set @@tidb_partition_prune_mode='` + string(variable.DynamicOnly) + `'`) + tk.MustExec(`set @@tidb_partition_prune_mode='` + string(variable.Dynamic) + `'`) // test fast analyze in dynamic mode tk.MustExec("drop table if exists t4;") From 0b720c993bb2ac3493cd1f6371752d068c2f724c Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Mon, 1 Mar 2021 14:45:51 +0800 Subject: [PATCH 10/13] fix ut --- errors.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/errors.toml b/errors.toml index 23ade8fc3f925..1848a80016c7a 100644 --- a/errors.toml +++ b/errors.toml @@ -1570,10 +1570,6 @@ invalid week mode format: '%v' error = ''' TiDB does not yet support JSON objects with the key length >= 65536 ''' -["types:8131"] -error = ''' -Build global-level stats failed due to missing partition-level stats -''' ["variable:1193"] error = ''' From 70e5beee4820590020140c75322a533735b2d60f Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Mon, 1 Mar 2021 15:10:29 +0800 Subject: [PATCH 11/13] fix ut --- errors.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/errors.toml b/errors.toml index 1848a80016c7a..42a91c57c36b1 100644 --- a/errors.toml +++ b/errors.toml @@ -1571,6 +1571,11 @@ error = ''' TiDB does not yet support JSON objects with the key length >= 65536 ''' +["types:8131"] +error = ''' +Build global-level stats failed due to missing partition-level stats +''' + ["variable:1193"] error = ''' Unknown system variable '%-.64s' From 66d62efcc97c80fcc9a8d5ba7304d09bd6e4f6e7 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Mon, 1 Mar 2021 15:39:14 +0800 Subject: [PATCH 12/13] fix ut --- planner/core/planbuilder.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index b470b7985071c..8e413424ccb5a 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1636,11 +1636,6 @@ func getPhysicalIDsAndPartitionNames(tblInfo *model.TableInfo, partitionNames [] func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt, opts map[ast.AnalyzeOptionType]uint64, version int) (Plan, error) { p := &Analyze{Opts: opts} - pruneMode := variable.PartitionPruneMode(b.ctx.GetSessionVars().PartitionPruneMode.Load()) - if len(as.PartitionNames) > 0 && pruneMode == variable.Dynamic { - logutil.BgLogger().Info("analyze partition didn't affect in dynamic-prune-mode", zap.String("partitions", as.PartitionNames[0].L)) - return p, nil - } for _, tbl := range as.TableNames { if tbl.TableInfo.IsView() { return nil, errors.Errorf("analyze view %s is not supported now.", tbl.Name.O) @@ -1713,11 +1708,6 @@ func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt, opts map[ast.A func (b *PlanBuilder) buildAnalyzeIndex(as *ast.AnalyzeTableStmt, opts map[ast.AnalyzeOptionType]uint64, version int) (Plan, error) { p := &Analyze{Opts: opts} tblInfo := as.TableNames[0].TableInfo - pruneMode := variable.PartitionPruneMode(b.ctx.GetSessionVars().PartitionPruneMode.Load()) - if len(as.PartitionNames) > 0 && pruneMode == variable.Dynamic { - logutil.BgLogger().Info("analyze partition didn't affect in dynamic-prune-mode", zap.String("table", tblInfo.Name.L), zap.String("partitions", as.PartitionNames[0].L)) - return p, nil - } physicalIDs, names, err := getPhysicalIDsAndPartitionNames(tblInfo, as.PartitionNames) if err != nil { return nil, err @@ -1778,11 +1768,6 @@ func (b *PlanBuilder) buildAnalyzeIndex(as *ast.AnalyzeTableStmt, opts map[ast.A func (b *PlanBuilder) buildAnalyzeAllIndex(as *ast.AnalyzeTableStmt, opts map[ast.AnalyzeOptionType]uint64, version int) (Plan, error) { p := &Analyze{Opts: opts} tblInfo := as.TableNames[0].TableInfo - pruneMode := variable.PartitionPruneMode(b.ctx.GetSessionVars().PartitionPruneMode.Load()) - if len(as.PartitionNames) > 0 && pruneMode == variable.Dynamic { - logutil.BgLogger().Info("analyze partition didn't affect in dynamic-prune-mode", zap.String("table", tblInfo.Name.L), zap.String("partitions", as.PartitionNames[0].L)) - return p, nil - } physicalIDs, names, err := getPhysicalIDsAndPartitionNames(tblInfo, as.PartitionNames) if err != nil { return nil, err From 352fa6eb06abaef6bb302474eddb364462d65245 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Mon, 1 Mar 2021 15:59:49 +0800 Subject: [PATCH 13/13] fix ut --- executor/analyze.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index 1bb8dc00b9270..870140bbc2744 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -171,6 +171,11 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { sc := e.ctx.GetSessionVars().StmtCtx globalStats, err := statsHandle.MergePartitionStats2GlobalStats(sc, infoschema.GetInfoSchema(e.ctx), globalStatsID.tableID, info.isIndex, info.idxID) if err != nil { + if types.ErrBuildGlobalLevelStatsFailed.Equal(err) { + // When we find some partition-level stats are missing, we need to report warning. + sc.AppendWarning(err) + continue + } return err } for i := 0; i < globalStats.Num; i++ { @@ -809,8 +814,7 @@ func (e *AnalyzeFastExec) calculateEstimateSampleStep() (err error) { sql := new(strings.Builder) sqlexec.MustFormatSQL(sql, "select count(*) from %n.%n", dbInfo.Name.L, e.tblInfo.Name.L) - pruneMode := variable.PartitionPruneMode(e.ctx.GetSessionVars().PartitionPruneMode.Load()) - if pruneMode != variable.Dynamic && e.tblInfo.ID != e.tableID.GetStatisticsID() { + if e.tblInfo.ID != e.tableID.GetStatisticsID() { for _, definition := range e.tblInfo.Partition.Definitions { if definition.ID == e.tableID.GetStatisticsID() { sqlexec.MustFormatSQL(sql, " partition(%n)", definition.Name.L)