From 135e7c75fa68d5cd4491135817a4e8317cd4e765 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 3 Jun 2023 14:26:45 +0800 Subject: [PATCH 01/16] planner: enable revive Signed-off-by: Weizhen Wang --- build/nogo_config.json | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/build/nogo_config.json b/build/nogo_config.json index 50806eef4fe08..6843fbd30bd01 100644 --- a/build/nogo_config.json +++ b/build/nogo_config.json @@ -490,30 +490,6 @@ "dumpling/export/sql_type.go": "please fix it", ".*_test\\.go$": "ignore generated code", ".*_generated\\.go$": "ignore generated code", - "planner/core/resolve_indices.go": "planner/core/resolve_indices.go", - "planner/core/rule_result_reorder.go": "planner/core/rule_result_reorder.go", - "planner/core/rule_join_reorder_dp.go": "planner/core/rule_join_reorder_dp.go", - "planner/core/plan_cache_param.go": "planner/core/plan_cache_param.go", - "planner/core/rule_predicate_push_down.go": "planner/core/rule_predicate_push_down.go", - "planner/core/rule_aggregation_push_down.go": "planner/core/rule_aggregation_push_down.go", - "planner/core/rule_max_min_eliminate.go": "planner/core/rule_max_min_eliminate.go", - "planner/core/rule_predicate_simplification.go": "planner/core/rule_predicate_simplification.go", - "planner/core/indexmerge_path.go": "planner/core/indexmerge_path.go", - "planner/core/rule_join_reorder.go": "planner/core/rule_join_reorder.go", - "planner/core/plan_cacheable_checker.go": "planner/core/plan_cacheable_checker.go", - "planner/core/rule_decorrelate.go": "planner/core/rule_decorrelate.go", - "planner/core/plan_cache_utils.go": "planner/core/plan_cache_utils.go", - "planner/core/rule_aggregation_skew_rewrite.go": "planner/core/rule_aggregation_skew_rewrite.go", - "planner/core/rule_topn_push_down.go": "planner/core/rule_topn_push_down.go", - "planner/core/plan_cost_ver1.go": "planner/core/plan_cost_ver1.go", - "planner/core/expression_rewriter.go": "planner/core/expression_rewriter.go", - "planner/core/find_best_task.go": "planner/core/find_best_task.go", - "planner/core/task.go": "planner/core/task.go", - "planner/core/preprocess.go": "planner/core/preprocess.go", - "planner/core/rule_partition_processor.go": "planner/core/rule_partition_processor.go", - "planner/core/exhaust_physical_plans.go": "planner/core/exhaust_physical_plans.go", - "planner/core/plan_cache_lru.go": "planner/core/plan_cache_lru.go", - "planner/core/common_plans.go": "planner/core/common_plans.go", "plugin/conn_ip_example/": "plugin/conn_ip_example/" }, "only_files": { @@ -530,7 +506,6 @@ "server/conn.go": "server/conn.go", "server/conn_stmt.go": "server/conn_stmt.go", "server/conn_test.go": "server/conn_test.go", - "planner/core/rule_partition_eliminate.go": "planner/core/rule_partition_eliminate code", "distsql/": "ignore distsql code", "disttask": "disttask code", "dumpling/export": "dumpling/export code", From 95e2d04b65c21cd28e0700ef7084b54c670cf342 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 3 Jun 2023 19:22:29 +0800 Subject: [PATCH 02/16] update Signed-off-by: Weizhen Wang --- planner/core/exhaust_physical_plans.go | 87 +++--------------------- planner/core/plan_cache_param.go | 8 +-- planner/core/plan_cache_param_test.go | 10 +-- planner/core/plan_cost_ver1.go | 25 +++---- planner/core/rule_predicate_push_down.go | 2 +- 5 files changed, 32 insertions(+), 100 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 6a59daa6af892..4a86327cc8b33 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -1022,7 +1022,7 @@ func (ijHelper *indexJoinBuildHelper) buildRangeDecidedByInformation(idxCols []* func (p *LogicalJoin) constructInnerTableScanTask( wrapper *indexJoinInnerChildWrapper, ranges ranger.Ranges, - outerJoinKeys []*expression.Column, + _ []*expression.Column, rangeInfo string, keepOrder bool, desc bool, @@ -1153,76 +1153,13 @@ func (p *LogicalJoin) constructInnerUnionScan(us *LogicalUnionScan, reader Physi return physicalUnionScan } -func getColsNDVLowerBoundFromHistColl(cols []*expression.Column, histColl *statistics.HistColl) int64 { - if len(cols) == 0 || histColl == nil { - return -1 - } - colUIDs := make([]int64, len(cols)) - for i, col := range cols { - colUIDs[i] = col.UniqueID - } - - // Note that we don't need to specially handle prefix index in this function, because the NDV of a prefix index is - // equal or less than the corresponding normal index, and that's safe here since we want a lower bound. - - // 1. Try to get NDV from column stats if it's a single column. - if len(colUIDs) == 1 && histColl.Columns != nil { - uid := colUIDs[0] - if colStats, ok := histColl.Columns[uid]; ok && colStats != nil { - return colStats.NDV - } - } - - slices.Sort(colUIDs) - if histColl.Indices == nil || histColl.Idx2ColumnIDs == nil { - return -1 - } - - // 2. Try to get NDV from index stats. - for idxID, idxCols := range histColl.Idx2ColumnIDs { - if len(idxCols) != len(colUIDs) { - continue - } - orderedIdxCols := make([]int64, len(idxCols)) - copy(orderedIdxCols, idxCols) - slices.Sort(orderedIdxCols) - if !slices.Equal(orderedIdxCols, colUIDs) { - continue - } - if idxStats, ok := histColl.Indices[idxID]; ok && idxStats != nil { - return idxStats.NDV - } - } - - // TODO: if there's an index that contains the expected columns, we can also make use of its NDV. - // For example, NDV(a,b,c) / NDV(c) is a safe lower bound of NDV(a,b). - - // 3. If we still haven't got an NDV, we use the minimal NDV in the column stats as a lower bound. - // This would happen when len(cols) > 1 and no proper index stats are available. - minNDV := int64(-1) - for _, colStats := range histColl.Columns { - if colStats == nil || colStats.Info == nil { - continue - } - col := colStats.Info - if col.IsGenerated() && !col.GeneratedStored { - continue - } - if (colStats.NDV > 0 && minNDV <= 0) || - colStats.NDV < minNDV { - minNDV = colStats.NDV - } - } - return minNDV -} - // constructInnerIndexScanTask is specially used to construct the inner plan for PhysicalIndexJoin. func (p *LogicalJoin) constructInnerIndexScanTask( wrapper *indexJoinInnerChildWrapper, path *util.AccessPath, ranges ranger.Ranges, filterConds []expression.Expression, - innerJoinKeys []*expression.Column, + _ []*expression.Column, rangeInfo string, keepOrder bool, desc bool, @@ -1319,7 +1256,7 @@ func (p *LogicalJoin) constructInnerIndexScanTask( // Because we are estimating an average row count of the inner side corresponding to each row from the outer side, // the estimated row count of the IndexScan should be no larger than (total row count / NDV of join key columns). // We use it as an upper bound here. - rowCountUpperBound := -1.0 + //rowCountUpperBound := -1.0 //if ds.tableStats != nil { // joinKeyNDV := getColsNDVLowerBoundFromHistColl(innerJoinKeys, ds.tableStats.HistColl) // if joinKeyNDV > 0 { @@ -1327,9 +1264,9 @@ func (p *LogicalJoin) constructInnerIndexScanTask( // } //} - if rowCountUpperBound > 0 { - rowCount = math.Min(rowCount, rowCountUpperBound) - } + //if rowCountUpperBound > 0 { + // rowCount = math.Min(rowCount, rowCountUpperBound) + //} if maxOneRow { // Theoretically, this line is unnecessary because row count estimation of join should guarantee rowCount is not larger // than 1.0; however, there may be rowCount larger than 1.0 in reality, e.g, pseudo statistics cases, which does not reflect @@ -1352,9 +1289,6 @@ func (p *LogicalJoin) constructInnerIndexScanTask( // rowCount is computed from result row count of join, which has already accounted the filters on DataSource, // i.e, rowCount equals to `countAfterIndex * selectivity`. cnt := rowCount / selectivity - if rowCountUpperBound > 0 { - cnt = math.Min(cnt, rowCountUpperBound) - } if maxOneRow { cnt = math.Min(cnt, 1.0) } @@ -1368,9 +1302,6 @@ func (p *LogicalJoin) constructInnerIndexScanTask( selectivity = SelectionFactor } cnt := tmpPath.CountAfterIndex / selectivity - if rowCountUpperBound > 0 { - cnt = math.Min(cnt, rowCountUpperBound) - } if maxOneRow { cnt = math.Min(cnt, 1.0) } @@ -2092,14 +2023,14 @@ func checkChildFitBC(p Plan) bool { return p.SCtx().GetSessionVars().BroadcastJoinThresholdSize == -1 || sz < float64(p.SCtx().GetSessionVars().BroadcastJoinThresholdSize) } -func calcBroadcastExchangeSize(p Plan, mppStoreCnt int) (float64, float64, bool) { +func calcBroadcastExchangeSize(p Plan, mppStoreCnt int) (row float64, size float64, hasSize bool) { s := p.statsInfo() - row := float64(s.Count()) * float64(mppStoreCnt-1) + row = float64(s.Count()) * float64(mppStoreCnt-1) if s.HistColl == nil { return row, 0, false } avg := s.HistColl.GetAvgRowSize(p.SCtx(), p.Schema().Columns, false, false) - size := avg * row + size = avg * row return row, size, true } diff --git a/planner/core/plan_cache_param.go b/planner/core/plan_cache_param.go index 9beae0a88e457..2d40bac6e0fc0 100644 --- a/planner/core/plan_cache_param.go +++ b/planner/core/plan_cache_param.go @@ -96,7 +96,7 @@ func (pr *paramReplacer) Reset() { // paramVals are copied from this AST. func GetParamSQLFromAST(ctx context.Context, sctx sessionctx.Context, stmt ast.StmtNode) (paramSQL string, paramVals []types.Datum, err error) { var params []*driver.ValueExpr - paramSQL, params, err = ParameterizeAST(ctx, sctx, stmt) + paramSQL, params, err = ParameterizeAST(stmt) if err != nil { return "", nil, err } @@ -105,14 +105,14 @@ func GetParamSQLFromAST(ctx context.Context, sctx sessionctx.Context, stmt ast.S p.Datum.Copy(¶mVals[i]) } - err = RestoreASTWithParams(ctx, sctx, stmt, params) + err = RestoreASTWithParams(sctx, stmt, params) return } // ParameterizeAST parameterizes this StmtNode. // e.g. `select * from t where a<10 and b<23` --> `select * from t where a `select * from t where a<10 and b<23`. -func RestoreASTWithParams(ctx context.Context, _ sessionctx.Context, stmt ast.StmtNode, params []*driver.ValueExpr) error { +func RestoreASTWithParams(_ sessionctx.Context, stmt ast.StmtNode, params []*driver.ValueExpr) error { pr := paramRestorerPool.Get().(*paramRestorer) defer func() { pr.Reset() diff --git a/planner/core/plan_cache_param_test.go b/planner/core/plan_cache_param_test.go index cd98b3ac683d0..b60c917b4dbdf 100644 --- a/planner/core/plan_cache_param_test.go +++ b/planner/core/plan_cache_param_test.go @@ -112,7 +112,7 @@ func TestParameterize(t *testing.T) { for _, c := range cases { stmt, err := parser.New().ParseOneStmt(c.sql, "", "") require.Nil(t, err) - paramSQL, params, err := ParameterizeAST(context.Background(), sctx, stmt) + paramSQL, params, err := ParameterizeAST(stmt) require.Nil(t, err) require.Equal(t, c.paramSQL, paramSQL) require.Equal(t, len(c.params), len(params)) @@ -160,12 +160,12 @@ c_credit, c_credit_lim, c_discount, c_balance, c_since FROM customer WHERE c_w_i stmt, err := parser.New().ParseOneStmt(paymentSelectCustomerForUpdate, "", "") require.Nil(b, err) sctx := MockContext() - _, _, err = ParameterizeAST(context.Background(), sctx, stmt) + _, _, err = ParameterizeAST(stmt) require.Nil(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { - ParameterizeAST(context.Background(), sctx, stmt) + ParameterizeAST(stmt) } } @@ -174,12 +174,12 @@ func BenchmarkParameterizeInsert(b *testing.B) { stmt, err := parser.New().ParseOneStmt(paymentInsertHistory, "", "") require.Nil(b, err) sctx := MockContext() - _, _, err = ParameterizeAST(context.Background(), sctx, stmt) + _, _, err = ParameterizeAST(stmt) require.Nil(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { - ParameterizeAST(context.Background(), sctx, stmt) + ParameterizeAST(stmt) } } diff --git a/planner/core/plan_cost_ver1.go b/planner/core/plan_cost_ver1.go index 8c5300574116b..2ddf8964c5ab0 100644 --- a/planner/core/plan_cost_ver1.go +++ b/planner/core/plan_cost_ver1.go @@ -172,7 +172,7 @@ func (p *PhysicalIndexLookUpReader) GetCost(costFlag uint64) (cost float64) { } // getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexLookUpReader) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { +func (p *PhysicalIndexLookUpReader) getPlanCostVer1(_ property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil @@ -190,8 +190,9 @@ func (p *PhysicalIndexLookUpReader) getPlanCostVer1(taskType property.TaskType, // to keep compatible with the previous cost implementation, re-calculate table-scan cost by using index stats-count again (see copTask.finishIndexPlan). // TODO: amend table-side cost here later - var tmp PhysicalPlan - for tmp = p.tablePlan; len(tmp.Children()) > 0; tmp = tmp.Children()[0] { + var tmp = p.tablePlan + for len(tmp.Children()) > 0 { + tmp = tmp.Children()[0] } ts := tmp.(*PhysicalTableScan) tblCost, err := ts.getPlanCostVer1(property.CopMultiReadTaskType, option) @@ -226,7 +227,7 @@ func (p *PhysicalIndexLookUpReader) getPlanCostVer1(taskType property.TaskType, } // getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexReader) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { +func (p *PhysicalIndexReader) getPlanCostVer1(_ property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil @@ -268,7 +269,7 @@ func (p *PhysicalIndexReader) GetNetDataSize() float64 { } // getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalTableReader) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { +func (p *PhysicalTableReader) getPlanCostVer1(_ property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil @@ -351,7 +352,7 @@ func (p *PhysicalTableReader) GetNetDataSize() float64 { } // getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexMergeReader) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { +func (p *PhysicalIndexMergeReader) getPlanCostVer1(_ property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil @@ -405,7 +406,7 @@ func (p *PhysicalIndexMergeReader) GetPartialReaderNetDataSize(plan PhysicalPlan } // getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalTableScan) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { +func (p *PhysicalTableScan) getPlanCostVer1(_ property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil @@ -430,7 +431,7 @@ func (p *PhysicalTableScan) getPlanCostVer1(taskType property.TaskType, option * } // getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexScan) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { +func (p *PhysicalIndexScan) getPlanCostVer1(_ property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil @@ -841,7 +842,7 @@ func (p *PhysicalMergeJoin) getPlanCostVer1(taskType property.TaskType, option * } // GetCost computes cost of hash join operator itself. -func (p *PhysicalHashJoin) GetCost(lCnt, rCnt float64, isMPP bool, costFlag uint64, op *physicalOptimizeOp) float64 { +func (p *PhysicalHashJoin) GetCost(lCnt, rCnt float64, _ bool, costFlag uint64, op *physicalOptimizeOp) float64 { buildCnt, probeCnt := lCnt, rCnt build := p.children[0] // Taking the right as the inner for right join or using the outer to build a hash table. @@ -957,7 +958,7 @@ func (p *PhysicalHashJoin) getPlanCostVer1(taskType property.TaskType, option *P } // GetCost computes cost of stream aggregation considering CPU/memory. -func (p *PhysicalStreamAgg) GetCost(inputRows float64, isRoot, isMPP bool, costFlag uint64) float64 { +func (p *PhysicalStreamAgg) GetCost(inputRows float64, isRoot, _ bool, costFlag uint64) float64 { aggFuncFactor := p.getAggFuncCostFactor(false) var cpuCost float64 sessVars := p.ctx.GetSessionVars() @@ -1144,7 +1145,7 @@ func (p *BatchPointGetPlan) GetCost(opt *physicalOptimizeOp) float64 { } // getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *BatchPointGetPlan) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { +func (p *BatchPointGetPlan) getPlanCostVer1(_ property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil @@ -1192,7 +1193,7 @@ func (p *PointGetPlan) GetCost(opt *physicalOptimizeOp) float64 { } // getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PointGetPlan) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { +func (p *PointGetPlan) getPlanCostVer1(_ property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil diff --git a/planner/core/rule_predicate_push_down.go b/planner/core/rule_predicate_push_down.go index e3df3c6245f18..ac00d7486e5a7 100644 --- a/planner/core/rule_predicate_push_down.go +++ b/planner/core/rule_predicate_push_down.go @@ -42,7 +42,7 @@ type exprPrefixAdder struct { lengths []int } -func (s *ppdSolver) optimize(_ context.Context, lp LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (*ppdSolver) optimize(_ context.Context, lp LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { _, p := lp.PredicatePushDown(nil, opt) return p, nil } From 567bf8f1b6e830641196b7ed1a3555504f493ed7 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 3 Jun 2023 19:25:54 +0800 Subject: [PATCH 03/16] update Signed-off-by: Weizhen Wang --- planner/core/find_best_task.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index edd0f4a094daa..1ea59bd78fc05 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -1123,9 +1123,9 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter if allRangeIsPoint { var pointGetTask task if len(path.Ranges) == 1 { - pointGetTask = ds.convertToPointGet(prop, candidate, opt) + pointGetTask = ds.convertToPointGet(prop, candidate) } else { - pointGetTask = ds.convertToBatchPointGet(prop, candidate, hashPartColName, opt) + pointGetTask = ds.convertToBatchPointGet(prop, candidate, hashPartColName) } // Batch/PointGet plans may be over-optimized, like `a>=1(?) and a<=1(?)` --> `a=1` --> PointGet(a=1). @@ -2307,7 +2307,7 @@ func (ds *DataSource) convertToSampleTable(prop *property.PhysicalProperty, }, nil } -func (ds *DataSource) convertToPointGet(prop *property.PhysicalProperty, candidate *candidatePath, opt *physicalOptimizeOp) (task task) { +func (ds *DataSource) convertToPointGet(prop *property.PhysicalProperty, candidate *candidatePath) (task task) { if !prop.IsSortItemEmpty() && !candidate.isMatchProp { return invalidTask } @@ -2384,8 +2384,7 @@ func (ds *DataSource) convertToPointGet(prop *property.PhysicalProperty, candida return rTsk } -func (ds *DataSource) convertToBatchPointGet(prop *property.PhysicalProperty, - candidate *candidatePath, hashPartColName *model.CIStr, opt *physicalOptimizeOp) (task task) { +func (ds *DataSource) convertToBatchPointGet(prop *property.PhysicalProperty, candidate *candidatePath, hashPartColName *model.CIStr) (task task) { if !prop.IsSortItemEmpty() && !candidate.isMatchProp { return invalidTask } From 68eac2195ce36d2ab4cc7e24b257c97cfef4e012 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 3 Jun 2023 19:30:50 +0800 Subject: [PATCH 04/16] update Signed-off-by: Weizhen Wang --- planner/core/exhaust_physical_plans.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 4a86327cc8b33..fdae3615b7d97 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -1256,7 +1256,7 @@ func (p *LogicalJoin) constructInnerIndexScanTask( // Because we are estimating an average row count of the inner side corresponding to each row from the outer side, // the estimated row count of the IndexScan should be no larger than (total row count / NDV of join key columns). // We use it as an upper bound here. - //rowCountUpperBound := -1.0 + rowCountUpperBound := -1.0 //if ds.tableStats != nil { // joinKeyNDV := getColsNDVLowerBoundFromHistColl(innerJoinKeys, ds.tableStats.HistColl) // if joinKeyNDV > 0 { @@ -1264,9 +1264,9 @@ func (p *LogicalJoin) constructInnerIndexScanTask( // } //} - //if rowCountUpperBound > 0 { - // rowCount = math.Min(rowCount, rowCountUpperBound) - //} + if rowCountUpperBound > 0 { + rowCount = math.Min(rowCount, rowCountUpperBound) + } if maxOneRow { // Theoretically, this line is unnecessary because row count estimation of join should guarantee rowCount is not larger // than 1.0; however, there may be rowCount larger than 1.0 in reality, e.g, pseudo statistics cases, which does not reflect @@ -1289,6 +1289,9 @@ func (p *LogicalJoin) constructInnerIndexScanTask( // rowCount is computed from result row count of join, which has already accounted the filters on DataSource, // i.e, rowCount equals to `countAfterIndex * selectivity`. cnt := rowCount / selectivity + if rowCountUpperBound > 0 { + cnt = math.Min(cnt, rowCountUpperBound) + } if maxOneRow { cnt = math.Min(cnt, 1.0) } @@ -1302,6 +1305,9 @@ func (p *LogicalJoin) constructInnerIndexScanTask( selectivity = SelectionFactor } cnt := tmpPath.CountAfterIndex / selectivity + if rowCountUpperBound > 0 { + cnt = math.Min(cnt, rowCountUpperBound) + } if maxOneRow { cnt = math.Min(cnt, 1.0) } @@ -2023,14 +2029,14 @@ func checkChildFitBC(p Plan) bool { return p.SCtx().GetSessionVars().BroadcastJoinThresholdSize == -1 || sz < float64(p.SCtx().GetSessionVars().BroadcastJoinThresholdSize) } -func calcBroadcastExchangeSize(p Plan, mppStoreCnt int) (row float64, size float64, hasSize bool) { +func calcBroadcastExchangeSize(p Plan, mppStoreCnt int) (float64, float64, bool) { s := p.statsInfo() - row = float64(s.Count()) * float64(mppStoreCnt-1) + row := float64(s.Count()) * float64(mppStoreCnt-1) if s.HistColl == nil { return row, 0, false } avg := s.HistColl.GetAvgRowSize(p.SCtx(), p.Schema().Columns, false, false) - size = avg * row + size := avg * row return row, size, true } From bc0af7c958600a07ba4d8d0ff043a0dd1c373825 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 3 Jun 2023 19:41:25 +0800 Subject: [PATCH 05/16] update Signed-off-by: Weizhen Wang --- planner/core/rule_join_reorder_dp.go | 2 +- planner/core/rule_predicate_simplification.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/planner/core/rule_join_reorder_dp.go b/planner/core/rule_join_reorder_dp.go index dc13c59633146..276228693426b 100644 --- a/planner/core/rule_join_reorder_dp.go +++ b/planner/core/rule_join_reorder_dp.go @@ -138,7 +138,7 @@ func (s *joinReorderDPSolver) solve(joinGroup []LogicalPlan, tracer *joinReorder } // bfsGraph bfs a sub graph starting at startPos. And relabel its label for future use. -func (s *joinReorderDPSolver) bfsGraph(startNode int, visited []bool, adjacents [][]int, nodeID2VisitID []int) []int { +func (*joinReorderDPSolver) bfsGraph(startNode int, visited []bool, adjacents [][]int, nodeID2VisitID []int) []int { queue := []int{startNode} visited[startNode] = true var visitID2NodeID []int diff --git a/planner/core/rule_predicate_simplification.go b/planner/core/rule_predicate_simplification.go index 87231ccd35fa0..c234bf7c2d026 100644 --- a/planner/core/rule_predicate_simplification.go +++ b/planner/core/rule_predicate_simplification.go @@ -65,7 +65,7 @@ func findPredicateType(expr expression.Expression) (*expression.Column, predicat return nil, otherPredicate } -func (s *predicateSimplification) optimize(_ context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (*predicateSimplification) optimize(_ context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { return p.predicateSimplification(opt), nil } From 0e9a4100c7eca18122a91dab558f7e4e7811f683 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 3 Jun 2023 19:44:21 +0800 Subject: [PATCH 06/16] update Signed-off-by: Weizhen Wang --- planner/core/rule_predicate_simplification.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/rule_predicate_simplification.go b/planner/core/rule_predicate_simplification.go index c234bf7c2d026..e0b238da2d74c 100644 --- a/planner/core/rule_predicate_simplification.go +++ b/planner/core/rule_predicate_simplification.go @@ -154,7 +154,7 @@ func applyPredicateSimplification(sctx sessionctx.Context, predicates []expressi return newValues } -func (s *DataSource) predicateSimplification(opt *logicalOptimizeOp) LogicalPlan { +func (s *DataSource) predicateSimplification(*logicalOptimizeOp) LogicalPlan { p := s.self.(*DataSource) p.pushedDownConds = applyPredicateSimplification(p.ctx, p.pushedDownConds) p.allConds = applyPredicateSimplification(p.ctx, p.allConds) From db82362045879cdab551558fde0b0b87b27cd643 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 3 Jun 2023 19:49:22 +0800 Subject: [PATCH 07/16] update Signed-off-by: Weizhen Wang --- planner/core/rule_result_reorder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planner/core/rule_result_reorder.go b/planner/core/rule_result_reorder.go index a182501cc05e1..afb17b487d123 100644 --- a/planner/core/rule_result_reorder.go +++ b/planner/core/rule_result_reorder.go @@ -93,7 +93,7 @@ func (rs *resultReorder) injectSort(lp LogicalPlan) LogicalPlan { return sort } -func (rs *resultReorder) isInputOrderKeeper(lp LogicalPlan) bool { +func (*resultReorder) isInputOrderKeeper(lp LogicalPlan) bool { switch lp.(type) { case *LogicalSelection, *LogicalProjection, *LogicalLimit: return true @@ -123,6 +123,6 @@ func (rs *resultReorder) extractHandleCol(lp LogicalPlan) *expression.Column { return nil } -func (rs *resultReorder) name() string { +func (*resultReorder) name() string { return "result_reorder" } From a5c2bc4b93b0b6ad522baeae790e95e8c502cfc7 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 3 Jun 2023 19:57:41 +0800 Subject: [PATCH 08/16] update Signed-off-by: Weizhen Wang --- planner/core/rule_partition_processor.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index 57225e53368af..1fef8143aede0 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -125,8 +125,7 @@ func generateHashPartitionExpr(ctx sessionctx.Context, pi *model.PartitionInfo, return exprs[0], nil } -func getPartColumnsForHashPartition(ctx sessionctx.Context, hashExpr expression.Expression, - columns []*expression.Column, names types.NameSlice) ([]*expression.Column, []int) { +func getPartColumnsForHashPartition(hashExpr expression.Expression) ([]*expression.Column, []int) { partCols := expression.ExtractColumns(hashExpr) colLen := make([]int, 0, len(partCols)) for i := 0; i < len(partCols); i++ { @@ -144,7 +143,7 @@ func (s *partitionProcessor) getUsedHashPartitions(ctx sessionctx.Context, if err != nil { return nil, nil, err } - partCols, colLen := getPartColumnsForHashPartition(ctx, hashExpr, columns, names) + partCols, colLen := getPartColumnsForHashPartition(hashExpr) detachedResult, err := ranger.DetachCondAndBuildRangeForPartition(ctx, conds, partCols, colLen, ctx.GetSessionVars().RangeMaxSize) if err != nil { return nil, nil, err @@ -245,7 +244,7 @@ func (s *partitionProcessor) getUsedHashPartitions(ctx sessionctx.Context, func (s *partitionProcessor) getUsedKeyPartitions(ctx sessionctx.Context, tbl table.Table, partitionNames []model.CIStr, columns []*expression.Column, - conds []expression.Expression, names types.NameSlice) ([]int, []expression.Expression, error) { + conds []expression.Expression, _ types.NameSlice) ([]int, []expression.Expression, error) { pi := tbl.Meta().Partition partExpr := tbl.(partitionTable).PartitionExpr() partCols, colLen := partExpr.GetPartColumnsForKeyPartition(columns) @@ -486,8 +485,7 @@ type listPartitionPruner struct { listPrune *tables.ForListPruning } -func newListPartitionPruner(ctx sessionctx.Context, tbl table.Table, partitionNames []model.CIStr, - s *partitionProcessor, conds []expression.Expression, pruneList *tables.ForListPruning, columns []*expression.Column) *listPartitionPruner { +func newListPartitionPruner(ctx sessionctx.Context, tbl table.Table, partitionNames []model.CIStr, s *partitionProcessor, pruneList *tables.ForListPruning, columns []*expression.Column) *listPartitionPruner { pruneList = pruneList.Clone() for i := range pruneList.PruneExprCols { for j := range columns { @@ -723,7 +721,7 @@ func (s *partitionProcessor) findUsedListPartitions(ctx sessionctx.Context, tbl pi := tbl.Meta().Partition partExpr := tbl.(partitionTable).PartitionExpr() - listPruner := newListPartitionPruner(ctx, tbl, partitionNames, s, conds, partExpr.ForListPruning, columns) + listPruner := newListPartitionPruner(ctx, tbl, partitionNames, s, partExpr.ForListPruning, columns) var used map[int]struct{} var err error if partExpr.ForListPruning.ColPrunes == nil { From 18db2be03dc06cfea0715a41ac528d3314c0c120 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 3 Jun 2023 20:07:47 +0800 Subject: [PATCH 09/16] update Signed-off-by: Weizhen Wang --- planner/core/rule_partition_processor.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index 1fef8143aede0..ec25067d4dd52 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -405,7 +405,7 @@ func (s *partitionProcessor) pruneHashOrKeyPartition(ctx sessionctx.Context, tbl // reconstructTableColNames reconstructs FieldsNames according to ds.TblCols. // ds.names may not match ds.TblCols since ds.names is pruned while ds.TblCols contains all original columns. // please see https://github.com/pingcap/tidb/issues/22635 for more details. -func (s *partitionProcessor) reconstructTableColNames(ds *DataSource) ([]*types.FieldName, error) { +func (*partitionProcessor) reconstructTableColNames(ds *DataSource) ([]*types.FieldName, error) { names := make([]*types.FieldName, 0, len(ds.TblCols)) // Use DeletableCols to get all the columns. colsInfo := ds.table.DeletableCols() @@ -779,7 +779,7 @@ func (s *partitionProcessor) prune(ds *DataSource, opt *logicalOptimizeOp) (Logi } // findByName checks whether object name exists in list. -func (s *partitionProcessor) findByName(partitionNames []model.CIStr, partitionName string) bool { +func (*partitionProcessor) findByName(partitionNames []model.CIStr, partitionName string) bool { for _, s := range partitionNames { if s.L == partitionName { return true @@ -1584,7 +1584,7 @@ func pruneUseBinarySearch(lessThan lessThanDataInt, data dataForPrune, unsigned return start, end } -func (s *partitionProcessor) resolveAccessPaths(ds *DataSource) error { +func (*partitionProcessor) resolveAccessPaths(ds *DataSource) error { possiblePaths, err := getPossibleAccessPaths( ds.ctx, &tableHintInfo{indexMergeHintList: ds.indexMergeHints, indexHintList: ds.IndexHints}, ds.astIndexHints, ds.table, ds.DBName, ds.tableInfo.Name, ds.isForUpdateRead, true) @@ -1684,7 +1684,7 @@ func appendWarnForUnknownPartitions(ctx sessionctx.Context, hintName string, unk ctx.GetSessionVars().StmtCtx.AppendWarning(warning) } -func (s *partitionProcessor) checkHintsApplicable(ds *DataSource, partitionSet set.StringSet) { +func (*partitionProcessor) checkHintsApplicable(ds *DataSource, partitionSet set.StringSet) { for _, idxHint := range ds.IndexHints { unknownPartitions := checkTableHintsApplicableForPartition(idxHint.partitions, partitionSet) appendWarnForUnknownPartitions(ds.ctx, restore2IndexHint(idxHint.hintTypeString(), idxHint), unknownPartitions) From c2be5d85c1f22e1e1a5b9af5b229935f50cb450e Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sun, 4 Jun 2023 16:09:48 +0800 Subject: [PATCH 10/16] update Signed-off-by: Weizhen Wang --- build/nogo_config.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/build/nogo_config.json b/build/nogo_config.json index 6843fbd30bd01..06d746067620f 100644 --- a/build/nogo_config.json +++ b/build/nogo_config.json @@ -490,6 +490,21 @@ "dumpling/export/sql_type.go": "please fix it", ".*_test\\.go$": "ignore generated code", ".*_generated\\.go$": "ignore generated code", + "planner/core/resolve_indices.go": "planner/core/resolve_indices.go", + "planner/core/rule_aggregation_push_down.go": "planner/core/rule_aggregation_push_down.go", + "planner/core/rule_max_min_eliminate.go": "planner/core/rule_max_min_eliminate.go", + "planner/core/indexmerge_path.go": "planner/core/indexmerge_path.go", + "planner/core/rule_join_reorder.go": "planner/core/rule_join_reorder.go", + "planner/core/plan_cacheable_checker.go": "planner/core/plan_cacheable_checker.go", + "planner/core/rule_decorrelate.go": "planner/core/rule_decorrelate.go", + "planner/core/plan_cache_utils.go": "planner/core/plan_cache_utils.go", + "planner/core/rule_aggregation_skew_rewrite.go": "planner/core/rule_aggregation_skew_rewrite.go", + "planner/core/rule_topn_push_down.go": "planner/core/rule_topn_push_down.go", + "planner/core/expression_rewriter.go": "planner/core/expression_rewriter.go", + "planner/core/task.go": "planner/core/task.go", + "planner/core/preprocess.go": "planner/core/preprocess.go", + "planner/core/plan_cache_lru.go": "planner/core/plan_cache_lru.go", + "planner/core/common_plans.go": "planner/core/common_plans.go", "plugin/conn_ip_example/": "plugin/conn_ip_example/" }, "only_files": { @@ -506,6 +521,7 @@ "server/conn.go": "server/conn.go", "server/conn_stmt.go": "server/conn_stmt.go", "server/conn_test.go": "server/conn_test.go", + "planner/core/rule_partition_eliminate.go": "planner/core/rule_partition_eliminate code", "distsql/": "ignore distsql code", "disttask": "disttask code", "dumpling/export": "dumpling/export code", From eaf72e1d0647e2b2555500142c250c99016b93fa Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sun, 4 Jun 2023 16:25:35 +0800 Subject: [PATCH 11/16] update Signed-off-by: Weizhen Wang --- planner/core/plan_cache_param_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/planner/core/plan_cache_param_test.go b/planner/core/plan_cache_param_test.go index b60c917b4dbdf..7983b02459867 100644 --- a/planner/core/plan_cache_param_test.go +++ b/planner/core/plan_cache_param_test.go @@ -28,7 +28,6 @@ import ( ) func TestParameterize(t *testing.T) { - sctx := MockContext() cases := []struct { sql string paramSQL string @@ -159,7 +158,6 @@ func BenchmarkParameterizeSelect(b *testing.B) { c_credit, c_credit_lim, c_discount, c_balance, c_since FROM customer WHERE c_w_id = ? AND c_d_id = ?AND c_id = ? FOR UPDATE` stmt, err := parser.New().ParseOneStmt(paymentSelectCustomerForUpdate, "", "") require.Nil(b, err) - sctx := MockContext() _, _, err = ParameterizeAST(stmt) require.Nil(b, err) @@ -173,7 +171,6 @@ func BenchmarkParameterizeInsert(b *testing.B) { paymentInsertHistory := `INSERT INTO history (h_c_d_id, h_c_w_id, h_c_id, h_d_id, h_w_id, h_date, h_amount, h_data) VALUES (1, 2, 3, 4, 5, 6, 7, 8)` stmt, err := parser.New().ParseOneStmt(paymentInsertHistory, "", "") require.Nil(b, err) - sctx := MockContext() _, _, err = ParameterizeAST(stmt) require.Nil(b, err) From 80dac55db716c42a3951c0a87d6e6c7c1dad2df7 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sun, 4 Jun 2023 16:28:32 +0800 Subject: [PATCH 12/16] update Signed-off-by: Weizhen Wang --- planner/core/exhaust_physical_plans.go | 4 ++-- planner/core/plan_cache_param.go | 5 ++--- planner/core/plan_cache_param_test.go | 5 ++--- planner/core/rule_join_reorder_dp.go | 2 +- planner/optimize.go | 2 +- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index fdae3615b7d97..83e628aca356e 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -139,7 +139,7 @@ func (p *PhysicalMergeJoin) tryToGetChildReqProp(prop *property.PhysicalProperty return []*property.PhysicalProperty{lProp, rProp}, true } -func (p *LogicalJoin) checkJoinKeyCollation(leftKeys, rightKeys []*expression.Column) bool { +func (*LogicalJoin) checkJoinKeyCollation(leftKeys, rightKeys []*expression.Column) bool { // if a left key and its corresponding right key have different collation, don't use MergeJoin since // the their children may sort their records in different ways for i := range leftKeys { @@ -1115,7 +1115,7 @@ func (p *LogicalJoin) constructInnerByWrapper(wrapper *indexJoinInnerChildWrappe return child } -func (p *LogicalJoin) constructInnerSel(sel *LogicalSelection, child PhysicalPlan) PhysicalPlan { +func (*LogicalJoin) constructInnerSel(sel *LogicalSelection, child PhysicalPlan) PhysicalPlan { if sel == nil { return child } diff --git a/planner/core/plan_cache_param.go b/planner/core/plan_cache_param.go index 2d40bac6e0fc0..e95d17a6a88d4 100644 --- a/planner/core/plan_cache_param.go +++ b/planner/core/plan_cache_param.go @@ -16,7 +16,6 @@ package core import ( "bytes" - "context" "errors" "sync" @@ -83,7 +82,7 @@ func (pr *paramReplacer) Enter(in ast.Node) (out ast.Node, skipChildren bool) { return in, false } -func (pr *paramReplacer) Leave(in ast.Node) (out ast.Node, ok bool) { +func (*paramReplacer) Leave(in ast.Node) (out ast.Node, ok bool) { return in, true } @@ -94,7 +93,7 @@ func (pr *paramReplacer) Reset() { // GetParamSQLFromAST returns the parameterized SQL of this AST. // NOTICE: this function does not modify the original AST. // paramVals are copied from this AST. -func GetParamSQLFromAST(ctx context.Context, sctx sessionctx.Context, stmt ast.StmtNode) (paramSQL string, paramVals []types.Datum, err error) { +func GetParamSQLFromAST(sctx sessionctx.Context, stmt ast.StmtNode) (paramSQL string, paramVals []types.Datum, err error) { var params []*driver.ValueExpr paramSQL, params, err = ParameterizeAST(stmt) if err != nil { diff --git a/planner/core/plan_cache_param_test.go b/planner/core/plan_cache_param_test.go index 7983b02459867..4009a9fa2c02a 100644 --- a/planner/core/plan_cache_param_test.go +++ b/planner/core/plan_cache_param_test.go @@ -15,7 +15,6 @@ package core import ( - "context" "fmt" "math/rand" "sync" @@ -139,7 +138,7 @@ func TestGetParamSQLFromASTConcurrently(t *testing.T) { wg.Add(1) go func(id int) { for i := 0; i < 1000; i++ { - _, vals, err := GetParamSQLFromAST(context.Background(), MockContext(), stmts[id]) + _, vals, err := GetParamSQLFromAST(MockContext(), stmts[id]) require.Nil(t, err) require.Equal(t, len(vals), 3) require.Equal(t, vals[0].GetValue(), int64(id*3+0)) @@ -188,6 +187,6 @@ func BenchmarkGetParamSQL(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - GetParamSQLFromAST(context.Background(), sctx, stmt) + GetParamSQLFromAST(sctx, stmt) } } diff --git a/planner/core/rule_join_reorder_dp.go b/planner/core/rule_join_reorder_dp.go index 276228693426b..9e87ced883fe3 100644 --- a/planner/core/rule_join_reorder_dp.go +++ b/planner/core/rule_join_reorder_dp.go @@ -211,7 +211,7 @@ func (s *joinReorderDPSolver) dpGraph(visitID2NodeID, nodeID2VisitID []int, _ [] return bestPlan[(1< Date: Sun, 4 Jun 2023 16:32:35 +0800 Subject: [PATCH 13/16] update Signed-off-by: Weizhen Wang --- planner/core/exhaust_physical_plans.go | 2 +- planner/core/find_best_task.go | 2 +- planner/core/rule_partition_processor.go | 2 +- planner/core/rule_predicate_simplification.go | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 83e628aca356e..98db0d74b4ca1 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -2804,7 +2804,7 @@ func (lw *LogicalWindow) exhaustPhysicalPlans(prop *property.PhysicalProperty) ( } // exhaustPhysicalPlans is only for implementing interface. DataSource and Dual generate task in `findBestTask` directly. -func (p *baseLogicalPlan) exhaustPhysicalPlans(_ *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { +func (*baseLogicalPlan) exhaustPhysicalPlans(*property.PhysicalProperty) ([]PhysicalPlan, bool, error) { panic("baseLogicalPlan.exhaustPhysicalPlans() should never be called.") } diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 1ea59bd78fc05..9ea4207060b75 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -783,7 +783,7 @@ func (ds *DataSource) getIndexCandidate(path *util.AccessPath, prop *property.Ph return candidate } -func (ds *DataSource) getIndexMergeCandidate(path *util.AccessPath) *candidatePath { +func (*DataSource) getIndexMergeCandidate(path *util.AccessPath) *candidatePath { candidate := &candidatePath{path: path} return candidate } diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index ec25067d4dd52..cf8ae39fe3232 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -1754,7 +1754,7 @@ func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.Part return unionAll, nil } -func (s *partitionProcessor) pruneRangeColumnsPartition(ctx sessionctx.Context, conds []expression.Expression, pi *model.PartitionInfo, pe *tables.PartitionExpr, columns []*expression.Column) (partitionRangeOR, error) { +func (*partitionProcessor) pruneRangeColumnsPartition(ctx sessionctx.Context, conds []expression.Expression, pi *model.PartitionInfo, pe *tables.PartitionExpr, columns []*expression.Column) (partitionRangeOR, error) { result := fullRange(len(pi.Definitions)) if len(pi.Columns) < 1 { diff --git a/planner/core/rule_predicate_simplification.go b/planner/core/rule_predicate_simplification.go index e0b238da2d74c..403b212885fcc 100644 --- a/planner/core/rule_predicate_simplification.go +++ b/planner/core/rule_predicate_simplification.go @@ -154,8 +154,8 @@ func applyPredicateSimplification(sctx sessionctx.Context, predicates []expressi return newValues } -func (s *DataSource) predicateSimplification(*logicalOptimizeOp) LogicalPlan { - p := s.self.(*DataSource) +func (ds *DataSource) predicateSimplification(*logicalOptimizeOp) LogicalPlan { + p := ds.self.(*DataSource) p.pushedDownConds = applyPredicateSimplification(p.ctx, p.pushedDownConds) p.allConds = applyPredicateSimplification(p.ctx, p.allConds) return p From 8e2c8be89eacee8fbd689020195657ce6abb86fb Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sun, 4 Jun 2023 16:37:36 +0800 Subject: [PATCH 14/16] update Signed-off-by: Weizhen Wang --- planner/core/exhaust_physical_plans.go | 16 ++++++++-------- planner/core/find_best_task.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 98db0d74b4ca1..8f4d8bfdf25dd 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -2029,18 +2029,18 @@ func checkChildFitBC(p Plan) bool { return p.SCtx().GetSessionVars().BroadcastJoinThresholdSize == -1 || sz < float64(p.SCtx().GetSessionVars().BroadcastJoinThresholdSize) } -func calcBroadcastExchangeSize(p Plan, mppStoreCnt int) (float64, float64, bool) { +func calcBroadcastExchangeSize(p Plan, mppStoreCnt int) (row float64, size float64, hasSize bool) { s := p.statsInfo() - row := float64(s.Count()) * float64(mppStoreCnt-1) + row = float64(s.Count()) * float64(mppStoreCnt-1) if s.HistColl == nil { return row, 0, false } avg := s.HistColl.GetAvgRowSize(p.SCtx(), p.Schema().Columns, false, false) - size := avg * row + size = avg * row return row, size, true } -func calcBroadcastExchangeSizeByChild(p1 Plan, p2 Plan, mppStoreCnt int) (float64, float64, bool) { +func calcBroadcastExchangeSizeByChild(p1 Plan, p2 Plan, mppStoreCnt int) (row float64, size float64, hasSize bool) { row1, size1, hasSize1 := calcBroadcastExchangeSize(p1, mppStoreCnt) row2, size2, hasSize2 := calcBroadcastExchangeSize(p2, mppStoreCnt) @@ -2056,18 +2056,18 @@ func calcBroadcastExchangeSizeByChild(p1 Plan, p2 Plan, mppStoreCnt int) (float6 return math.Min(row1, row2), 0, false } -func calcHashExchangeSize(p Plan, mppStoreCnt int) (float64, float64, bool) { +func calcHashExchangeSize(p Plan, mppStoreCnt int) (row float64, sz float64, hasSize bool) { s := p.statsInfo() - row := float64(s.Count()) * float64(mppStoreCnt-1) / float64(mppStoreCnt) + row = float64(s.Count()) * float64(mppStoreCnt-1) / float64(mppStoreCnt) if s.HistColl == nil { return row, 0, false } avg := s.HistColl.GetAvgRowSize(p.SCtx(), p.Schema().Columns, false, false) - sz := avg * row + sz = avg * row return row, sz, true } -func calcHashExchangeSizeByChild(p1 Plan, p2 Plan, mppStoreCnt int) (float64, float64, bool) { +func calcHashExchangeSizeByChild(p1 Plan, p2 Plan, mppStoreCnt int) (row float64, size float64, hasSize bool) { row1, size1, hasSize1 := calcHashExchangeSize(p1, mppStoreCnt) row2, size2, hasSize2 := calcHashExchangeSize(p2, mppStoreCnt) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 9ea4207060b75..aefe19864fc2a 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -1213,7 +1213,7 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter return } -func (ds *DataSource) isSafePointGetPlan4PlanCache(path *util.AccessPath) bool { +func (*DataSource) isSafePointGetPlan4PlanCache(path *util.AccessPath) bool { // PointGet might contain some over-optimized assumptions, like `a>=1 and a<=1` --> `a=1`, but // these assumptions may be broken after parameters change. From 518c1f0d9cb0e9db8e5560ea62d043a65a269898 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sun, 4 Jun 2023 16:49:20 +0800 Subject: [PATCH 15/16] update Signed-off-by: Weizhen Wang --- planner/core/plan_cache_param.go | 5 ++--- planner/core/plan_cost_ver1.go | 4 ++-- planner/core/rule_partition_processor.go | 7 +++---- planner/core/rule_predicate_push_down.go | 8 ++++---- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/planner/core/plan_cache_param.go b/planner/core/plan_cache_param.go index e95d17a6a88d4..ca66199aa3c43 100644 --- a/planner/core/plan_cache_param.go +++ b/planner/core/plan_cache_param.go @@ -134,8 +134,7 @@ type paramRestorer struct { } func (pr *paramRestorer) Enter(in ast.Node) (out ast.Node, skipChildren bool) { - switch n := in.(type) { - case *driver.ParamMarkerExpr: + if n, ok := in.(*driver.ParamMarkerExpr); ok { if n.Offset >= len(pr.params) { pr.err = errors.New("failed to restore ast.Node") return nil, true @@ -151,7 +150,7 @@ func (pr *paramRestorer) Enter(in ast.Node) (out ast.Node, skipChildren bool) { return in, false } -func (pr *paramRestorer) Leave(in ast.Node) (out ast.Node, ok bool) { +func (*paramRestorer) Leave(in ast.Node) (out ast.Node, ok bool) { return in, true } diff --git a/planner/core/plan_cost_ver1.go b/planner/core/plan_cost_ver1.go index 2ddf8964c5ab0..156ac820837f9 100644 --- a/planner/core/plan_cost_ver1.go +++ b/planner/core/plan_cost_ver1.go @@ -817,8 +817,8 @@ func (p *PhysicalMergeJoin) GetCost(lCnt, rCnt float64, costFlag uint64) float64 cpuCost += probeCost // For merge join, only one group of rows with same join key(not null) are cached, // we compute average memory cost using estimated group size. - NDV, _ := getColsNDVWithMatchedLen(innerKeys, innerSchema, innerStats) - memoryCost := (innerCnt / NDV) * sessVars.GetMemoryFactor() + ndv, _ := getColsNDVWithMatchedLen(innerKeys, innerSchema, innerStats) + memoryCost := (innerCnt / ndv) * sessVars.GetMemoryFactor() return cpuCost + memoryCost } diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index cf8ae39fe3232..39c5d7140e230 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -930,8 +930,7 @@ func (or partitionRangeOR) intersection(x partitionRangeOR) partitionRangeOR { } // intersectionRange calculate the intersection of [start, end) and [newStart, newEnd) -func intersectionRange(start, end, newStart, newEnd int) (int, int) { - var s, e int +func intersectionRange(start, end, newStart, newEnd int) (s int, e int) { if start > newStart { s = start } else { @@ -1266,7 +1265,7 @@ type rangePruner struct { monotonous monotoneMode } -func (p *rangePruner) partitionRangeForExpr(sctx sessionctx.Context, expr expression.Expression) (int, int, bool) { +func (p *rangePruner) partitionRangeForExpr(sctx sessionctx.Context, expr expression.Expression) (start int, end int, ok bool) { if constExpr, ok := expr.(*expression.Constant); ok { if b, err := constExpr.Value.ToBool(sctx.GetSessionVars().StmtCtx); err == nil && b == 0 { // A constant false expression. @@ -1280,7 +1279,7 @@ func (p *rangePruner) partitionRangeForExpr(sctx sessionctx.Context, expr expres } unsigned := mysql.HasUnsignedFlag(p.col.RetType.GetFlag()) - start, end := pruneUseBinarySearch(p.lessThan, dataForPrune, unsigned) + start, end = pruneUseBinarySearch(p.lessThan, dataForPrune, unsigned) return start, end, true } diff --git a/planner/core/rule_predicate_push_down.go b/planner/core/rule_predicate_push_down.go index ac00d7486e5a7..6e08aa65c91af 100644 --- a/planner/core/rule_predicate_push_down.go +++ b/planner/core/rule_predicate_push_down.go @@ -896,17 +896,17 @@ func (ds *DataSource) AddPrefix4ShardIndexes(sc sessionctx.Context, conds []expr func (ds *DataSource) addExprPrefixCond(sc sessionctx.Context, path *util.AccessPath, conds []expression.Expression) ([]expression.Expression, error) { - IdxCols, IdxColLens := + idxCols, idxColLens := expression.IndexInfo2PrefixCols(ds.Columns, ds.schema.Columns, path.Index) - if len(IdxCols) == 0 { + if len(idxCols) == 0 { return conds, nil } adder := &exprPrefixAdder{ sctx: sc, OrigConds: conds, - cols: IdxCols, - lengths: IdxColLens, + cols: idxCols, + lengths: idxColLens, } return adder.addExprPrefix4ShardIndex() From 3d175ca263d01331b51a39b3d1d13fa8916daa74 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sun, 4 Jun 2023 16:50:38 +0800 Subject: [PATCH 16/16] update Signed-off-by: Weizhen Wang --- planner/core/exhaust_physical_plans.go | 4 ++-- planner/core/rule_partition_processor.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 8f4d8bfdf25dd..119c6497ac8e2 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -1126,7 +1126,7 @@ func (*LogicalJoin) constructInnerSel(sel *LogicalSelection, child PhysicalPlan) return physicalSel } -func (p *LogicalJoin) constructInnerProj(proj *LogicalProjection, child PhysicalPlan) PhysicalPlan { +func (*LogicalJoin) constructInnerProj(proj *LogicalProjection, child PhysicalPlan) PhysicalPlan { if proj == nil { return child } @@ -1139,7 +1139,7 @@ func (p *LogicalJoin) constructInnerProj(proj *LogicalProjection, child Physical return physicalProj } -func (p *LogicalJoin) constructInnerUnionScan(us *LogicalUnionScan, reader PhysicalPlan) PhysicalPlan { +func (*LogicalJoin) constructInnerUnionScan(us *LogicalUnionScan, reader PhysicalPlan) PhysicalPlan { if us == nil { return reader } diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index 39c5d7140e230..5c8537ea286a0 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -1813,7 +1813,7 @@ func (p *rangeColumnsPruner) getPartCol(colID int64) *expression.Column { return nil } -func (p *rangeColumnsPruner) partitionRangeForExpr(sctx sessionctx.Context, expr expression.Expression) (int, int, bool) { +func (p *rangeColumnsPruner) partitionRangeForExpr(sctx sessionctx.Context, expr expression.Expression) (start int, end int, ok bool) { op, ok := expr.(*expression.ScalarFunction) if !ok { return 0, len(p.lessThan), false @@ -1864,7 +1864,7 @@ func (p *rangeColumnsPruner) partitionRangeForExpr(sctx sessionctx.Context, expr if exprColl != colColl && (opName != ast.EQ || !collate.IsBinCollation(exprColl)) { return 0, len(p.lessThan), true } - start, end := p.pruneUseBinarySearch(sctx, opName, con) + start, end = p.pruneUseBinarySearch(sctx, opName, con) return start, end, true }