Skip to content

Commit

Permalink
planner: revise the optimize trace output (#30882)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Yisaer
Yisaer authored Dec 22, 2021
1 parent 9d1e29d commit beb5451
Show file tree
Hide file tree
Showing 11 changed files with 172 additions and 119 deletions.
70 changes: 35 additions & 35 deletions planner/core/logical_plan_trace_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,8 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "partition_processor",
assertRuleSteps: []assertTraceStep{
{
assertReason: "Datasource[1] has multiple needed partitions[p1,p2] after pruning",
assertAction: "Datasource[1] becomes PartitionUnion[6] with children[TableScan[1],TableScan[1]]",
assertReason: "DataSource_1 has multiple needed partitions[p1,p2] after pruning",
assertAction: "DataSource_1 becomes PartitionUnion_6 with children[TableScan_1,TableScan_1]",
},
},
},
Expand All @@ -149,8 +149,8 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "partition_processor",
assertRuleSteps: []assertTraceStep{
{
assertReason: "Datasource[1] has one needed partition[p1] after pruning",
assertAction: "Datasource[1] becomes TableScan[1]",
assertReason: "DataSource_1 has one needed partition[p1] after pruning",
assertAction: "DataSource_1 becomes TableScan_1",
},
},
},
Expand All @@ -160,8 +160,8 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "partition_processor",
assertRuleSteps: []assertTraceStep{
{
assertReason: "Datasource[1] has multiple needed partitions[p1,p2] after pruning",
assertAction: "Datasource[1] becomes PartitionUnion[7] with children[TableScan[1],TableScan[1]]",
assertReason: "DataSource_1 has multiple needed partitions[p1,p2] after pruning",
assertAction: "DataSource_1 becomes PartitionUnion_7 with children[TableScan_1,TableScan_1]",
},
},
},
Expand All @@ -171,8 +171,8 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "partition_processor",
assertRuleSteps: []assertTraceStep{
{
assertReason: "Datasource[1] has one needed partition[p2] after pruning",
assertAction: "Datasource[1] becomes TableScan[1]",
assertReason: "DataSource_1 has one needed partition[p2] after pruning",
assertAction: "DataSource_1 becomes TableScan_1",
},
},
},
Expand All @@ -182,8 +182,8 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "partition_processor",
assertRuleSteps: []assertTraceStep{
{
assertReason: "Datasource[1] doesn't have needed partition table after pruning",
assertAction: "Datasource[1] becomes TableDual[5]",
assertReason: "DataSource_1 doesn't have needed partition table after pruning",
assertAction: "DataSource_1 becomes TableDual_5",
},
},
},
Expand All @@ -193,8 +193,8 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "partition_processor",
assertRuleSteps: []assertTraceStep{
{
assertReason: "Datasource[1] has multiple needed partitions[p1,p2] after pruning",
assertAction: "Datasource[1] becomes PartitionUnion[7] with children[TableScan[1],TableScan[1]]",
assertReason: "DataSource_1 has multiple needed partitions[p1,p2] after pruning",
assertAction: "DataSource_1 becomes PartitionUnion_7 with children[TableScan_1,TableScan_1]",
},
},
},
Expand All @@ -204,8 +204,8 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "partition_processor",
assertRuleSteps: []assertTraceStep{
{
assertReason: "Datasource[1] has one needed partition[p1] after pruning",
assertAction: "Datasource[1] becomes TableScan[1]",
assertReason: "DataSource_1 has one needed partition[p1] after pruning",
assertAction: "DataSource_1 becomes TableScan_1",
},
},
},
Expand Down Expand Up @@ -242,7 +242,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
},
{
assertReason: "[test.t.a] is a unique key",
assertAction: "aggregation is simplified to a projection",
assertAction: "Aggregation_2 is simplified to a Projection_4",
},
},
},
Expand All @@ -252,8 +252,8 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "projection_eliminate",
assertRuleSteps: []assertTraceStep{
{
assertAction: "Proj[2] is eliminated, Proj[3]'s expressions changed into[plus(1, plus(1, test.t.a))]",
assertReason: "Proj[3]'s child proj[2] is redundant",
assertAction: "Projection_2 is eliminated, Projection_3's expressions changed into[plus(1, plus(1, test.t.a))]",
assertReason: "Projection_3's child Projection_2 is redundant",
},
},
},
Expand All @@ -263,8 +263,8 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "aggregation_push_down",
assertRuleSteps: []assertTraceStep{
{
assertAction: "agg[6] pushed down across join[5], and join right path becomes agg[8]",
assertReason: "agg[6]'s functions[count(Column#38)] are decomposable with join",
assertAction: "Aggregation_6 pushed down across Join_5, and Join_5 right path becomes Aggregation_8",
assertReason: "Aggregation_6's functions[count(Column#38)] are decomposable with join",
},
},
},
Expand All @@ -274,16 +274,16 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "aggregation_push_down",
assertRuleSteps: []assertTraceStep{
{
assertAction: "agg[8] pushed down, and union[5]'s children changed into[[id:11,tp:Aggregation],[id:12,tp:Aggregation]]",
assertReason: "agg[8] functions[sum(Column#28)] are decomposable with union",
assertAction: "Aggregation_8 pushed down, and Union_5's children changed into[Aggregation_11,Aggregation_12]",
assertReason: "Aggregation_8 functions[sum(Column#28)] are decomposable with Union_5",
},
{
assertAction: "proj[6] is eliminated, and agg[11]'s functions changed into[sum(test.t.c),firstrow(test.t.d)]",
assertReason: "Proj[6] is directly below an agg[11] and has no side effects",
assertAction: "Projection_6 is eliminated, and Aggregation_11's functions changed into[sum(test.t.c),firstrow(test.t.d)]",
assertReason: "Projection_6 is directly below an Aggregation_11 and has no side effects",
},
{
assertAction: "proj[7] is eliminated, and agg[12]'s functions changed into[sum(test.t.a),firstrow(test.t.b)]",
assertReason: "Proj[7] is directly below an agg[12] and has no side effects",
assertAction: "Projection_7 is eliminated, and Aggregation_12's functions changed into[sum(test.t.a),firstrow(test.t.b)]",
assertReason: "Projection_7 is directly below an Aggregation_12 and has no side effects",
},
},
},
Expand All @@ -293,16 +293,16 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "max_min_eliminate",
assertRuleSteps: []assertTraceStep{
{
assertAction: "add sort[8],add limit[9] during eliminating agg[4] max function",
assertReason: "agg[4] has only one function[max] without group by, the columns in agg[4] should be sorted",
assertAction: "add Sort_8,add Limit_9 during eliminating Aggregation_4 max function",
assertReason: "Aggregation_4 has only one function[max] without group by, the columns in Aggregation_4 should be sorted",
},
{
assertAction: "add sort[10],add limit[11] during eliminating agg[6] min function",
assertReason: "agg[6] has only one function[min] without group by, the columns in agg[6] should be sorted",
assertAction: "add Sort_10,add Limit_11 during eliminating Aggregation_6 min function",
assertReason: "Aggregation_6 has only one function[min] without group by, the columns in Aggregation_6 should be sorted",
},
{
assertAction: "agg[2] splited into aggs[4,6], and add joins[12] to connect them during eliminating agg[2] multi min/max functions",
assertReason: "each column is sorted and can benefit from index/primary key in agg[4,6] and none of them has group by clause",
assertAction: "Aggregation_2 splited into [Aggregation_4,Aggregation_6], and add [Join_12] to connect them during eliminating Aggregation_2 multi min/max functions",
assertReason: "each column is sorted and can benefit from index/primary key in [Aggregation_4,Aggregation_6] and none of them has group by clause",
},
},
},
Expand All @@ -312,8 +312,8 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "max_min_eliminate",
assertRuleSteps: []assertTraceStep{
{
assertAction: "add selection[4],add sort[5],add limit[6] during eliminating agg[2] max function",
assertReason: "agg[2] has only one function[max] without group by, the columns in agg[2] shouldn't be NULL and needs NULL to be filtered out, the columns in agg[2] should be sorted",
assertAction: "add Selection_4,add Sort_5,add Limit_6 during eliminating Aggregation_2 max function",
assertReason: "Aggregation_2 has only one function[max] without group by, the columns in Aggregation_2 shouldn't be NULL and needs NULL to be filtered out, the columns in Aggregation_2 should be sorted",
},
},
},
Expand All @@ -323,7 +323,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "outer_join_eliminate",
assertRuleSteps: []assertTraceStep{
{
assertAction: "Outer join[3] is eliminated and become DataSource[1]",
assertAction: "Outer Join_3 is eliminated and become DataSource_1",
assertReason: "The columns[test.t.b,test.t.c] are from outer table, and the inner join keys[test.t.a] are unique",
},
},
Expand All @@ -334,7 +334,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
assertRuleName: "outer_join_eliminate",
assertRuleSteps: []assertTraceStep{
{
assertAction: "Outer join[3] is eliminated and become DataSource[1]",
assertAction: "Outer Join_3 is eliminated and become DataSource_1",
assertReason: "The columns[test.t.a,test.t.b] in agg are from outer table, and the agg functions are duplicate agnostic",
},
},
Expand Down
8 changes: 4 additions & 4 deletions planner/core/optimizer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,21 +107,21 @@ func (op *logicalOptimizeOp) appendBeforeRuleOptimize(index int, name string, be
if op.tracer == nil {
return
}
op.tracer.AppendRuleTracerBeforeRuleOptimize(index, name, before.buildLogicalPlanTrace(before))
op.tracer.AppendRuleTracerBeforeRuleOptimize(index, name, before.buildLogicalPlanTrace())
}

func (op *logicalOptimizeOp) appendStepToCurrent(id int, tp, reason, action string) {
func (op *logicalOptimizeOp) appendStepToCurrent(id int, tp string, reason, action func() string) {
if op.tracer == nil {
return
}
op.tracer.AppendRuleTracerStepToCurrent(id, tp, reason, action)
op.tracer.AppendRuleTracerStepToCurrent(id, tp, reason(), action())
}

func (op *logicalOptimizeOp) recordFinalLogicalPlan(final LogicalPlan) {
if op.tracer == nil {
return
}
op.tracer.RecordFinalLogicalPlan(final.buildLogicalPlanTrace(final))
op.tracer.RecordFinalLogicalPlan(final.buildLogicalPlanTrace())
}

// logicalOptRule means a logical optimizing rule, which contains decorrelate, ppd, column pruning, etc.
Expand Down
8 changes: 4 additions & 4 deletions planner/core/plan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ type LogicalPlan interface {
canPushToCop(store kv.StoreType) bool

// buildLogicalPlanTrace clone necessary information from LogicalPlan
buildLogicalPlanTrace(p Plan) *tracing.LogicalPlanTrace
buildLogicalPlanTrace() *tracing.LogicalPlanTrace
}

// PhysicalPlan is a tree of the physical operators.
Expand Down Expand Up @@ -382,10 +382,10 @@ func (p *baseLogicalPlan) ExplainInfo() string {
}

// buildLogicalPlanTrace implements LogicalPlan
func (p *baseLogicalPlan) buildLogicalPlanTrace(plan Plan) *tracing.LogicalPlanTrace {
planTrace := &tracing.LogicalPlanTrace{ID: p.ID(), TP: p.TP(), ExplainInfo: plan.ExplainInfo()}
func (p *baseLogicalPlan) buildLogicalPlanTrace() *tracing.LogicalPlanTrace {
planTrace := &tracing.LogicalPlanTrace{ID: p.ID(), TP: p.TP(), ExplainInfo: p.self.ExplainInfo()}
for _, child := range p.Children() {
planTrace.Children = append(planTrace.Children, child.buildLogicalPlanTrace(child))
planTrace.Children = append(planTrace.Children, child.buildLogicalPlanTrace())
}
return planTrace
}
Expand Down
25 changes: 17 additions & 8 deletions planner/core/rule_aggregation_elimination.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func (a *aggregationEliminateChecker) tryToEliminateAggregation(agg *LogicalAggr
// GroupByCols has unique key, so this aggregation can be removed.
if ok, proj := ConvertAggToProj(agg, agg.schema); ok {
proj.SetChildren(agg.children[0])
appendAggregationEliminateTraceStep(agg, uniqueKey, opt)
appendAggregationEliminateTraceStep(agg, proj, uniqueKey, opt)
return proj
}
}
Expand Down Expand Up @@ -116,17 +116,26 @@ func (a *aggregationEliminateChecker) tryToEliminateDistinct(agg *LogicalAggrega
}
}

func appendAggregationEliminateTraceStep(agg *LogicalAggregation, uniqueKey expression.KeyInfo, opt *logicalOptimizeOp) {
opt.appendStepToCurrent(agg.ID(), agg.TP(),
fmt.Sprintf("%s is a unique key", uniqueKey.String()),
"aggregation is simplified to a projection")
func appendAggregationEliminateTraceStep(agg *LogicalAggregation, proj *LogicalProjection, uniqueKey expression.KeyInfo, opt *logicalOptimizeOp) {
reason := func() string {
return fmt.Sprintf("%s is a unique key", uniqueKey.String())
}
action := func() string {
return fmt.Sprintf("%v_%v is simplified to a %v_%v", agg.TP(), agg.ID(), proj.TP(), proj.ID())
}

opt.appendStepToCurrent(agg.ID(), agg.TP(), reason, action)
}

func appendDistinctEliminateTraceStep(agg *LogicalAggregation, uniqueKey expression.KeyInfo, af *aggregation.AggFuncDesc,
opt *logicalOptimizeOp) {
opt.appendStepToCurrent(agg.ID(), agg.TP(),
fmt.Sprintf("%s is a unique key", uniqueKey.String()),
fmt.Sprintf("%s(distinct ...) is simplified to %s(...)", af.Name, af.Name))
reason := func() string {
return fmt.Sprintf("%s is a unique key", uniqueKey.String())
}
action := func() string {
return fmt.Sprintf("%s(distinct ...) is simplified to %s(...)", af.Name, af.Name)
}
opt.appendStepToCurrent(agg.ID(), agg.TP(), reason, action)
}

// ConvertAggToProj convert aggregation to projection.
Expand Down
32 changes: 17 additions & 15 deletions planner/core/rule_aggregation_push_down.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,7 @@ func (*aggregationPushDownSolver) name() string {
func appendAggPushDownAcrossJoinTraceStep(oldAgg, newAgg *LogicalAggregation, aggFuncs []*aggregation.AggFuncDesc, join *LogicalJoin,
childIdx int, opt *logicalOptimizeOp) {
reason := func() string {
buffer := bytes.NewBufferString(fmt.Sprintf("agg[%v]'s functions[", oldAgg.ID()))
buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v's functions[", oldAgg.TP(), oldAgg.ID()))
for i, aggFunc := range aggFuncs {
if i > 0 {
buffer.WriteString(",")
Expand All @@ -526,23 +526,23 @@ func appendAggPushDownAcrossJoinTraceStep(oldAgg, newAgg *LogicalAggregation, ag
}
buffer.WriteString("] are decomposable with join")
return buffer.String()
}()
}
action := func() string {
buffer := bytes.NewBufferString(fmt.Sprintf("agg[%v] pushed down across join[%v], ", oldAgg.ID(), join.ID()))
buffer.WriteString(fmt.Sprintf("and join %v path becomes agg[%v]", func() string {
buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v pushed down across %v_%v, ", oldAgg.TP(), oldAgg.ID(), join.TP(), join.ID()))
buffer.WriteString(fmt.Sprintf("and %v_%v %v path becomes %v_%v", join.TP(), join.ID(), func() string {
if childIdx == 0 {
return "left"
}
return "right"
}(), newAgg.ID()))
}(), newAgg.TP(), newAgg.ID()))
return buffer.String()
}()
}
opt.appendStepToCurrent(join.ID(), join.TP(), reason, action)
}

func appendAggPushDownAcrossProjTraceStep(agg *LogicalAggregation, proj *LogicalProjection, opt *logicalOptimizeOp) {
action := func() string {
buffer := bytes.NewBufferString(fmt.Sprintf("proj[%v] is eliminated, and agg[%v]'s functions changed into[", proj.ID(), agg.ID()))
buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v is eliminated, and %v_%v's functions changed into[", proj.TP(), proj.ID(), agg.TP(), agg.ID()))
for i, aggFunc := range agg.AggFuncs {
if i > 0 {
buffer.WriteString(",")
Expand All @@ -551,33 +551,35 @@ func appendAggPushDownAcrossProjTraceStep(agg *LogicalAggregation, proj *Logical
}
buffer.WriteString("]")
return buffer.String()
}()
reason := fmt.Sprintf("Proj[%v] is directly below an agg[%v] and has no side effects", proj.ID(), agg.ID())
}
reason := func() string {
return fmt.Sprintf("%v_%v is directly below an %v_%v and has no side effects", proj.TP(), proj.ID(), agg.TP(), agg.ID())
}
opt.appendStepToCurrent(agg.ID(), agg.TP(), reason, action)
}

func appendAggPushDownAcrossUnionTraceStep(union *LogicalUnionAll, agg *LogicalAggregation, opt *logicalOptimizeOp) {
reason := func() string {
buffer := bytes.NewBufferString(fmt.Sprintf("agg[%v] functions[", agg.ID()))
buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v functions[", agg.TP(), agg.ID()))
for i, aggFunc := range agg.AggFuncs {
if i > 0 {
buffer.WriteString(",")
}
buffer.WriteString(aggFunc.String())
}
buffer.WriteString("] are decomposable with union")
buffer.WriteString(fmt.Sprintf("] are decomposable with %v_%v", union.TP(), union.ID()))
return buffer.String()
}()
}
action := func() string {
buffer := bytes.NewBufferString(fmt.Sprintf("agg[%v] pushed down, and union[%v]'s children changed into[", agg.ID(), union.ID()))
buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v pushed down, and %v_%v's children changed into[", agg.TP(), agg.ID(), union.TP(), union.ID()))
for i, child := range union.Children() {
if i > 0 {
buffer.WriteString(",")
}
buffer.WriteString(fmt.Sprintf("[id:%v,tp:%s]", child.ID(), child.TP()))
buffer.WriteString(fmt.Sprintf("%v_%v", child.TP(), child.ID()))
}
buffer.WriteString("]")
return buffer.String()
}()
}
opt.appendStepToCurrent(union.ID(), union.TP(), reason, action)
}
16 changes: 11 additions & 5 deletions planner/core/rule_eliminate_projection.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ func (*projectionEliminator) name() string {
func appendDupProjEliminateTraceStep(parent, child *LogicalProjection, opt *logicalOptimizeOp) {
action := func() string {
buffer := bytes.NewBufferString(
fmt.Sprintf("Proj[%v] is eliminated, Proj[%v]'s expressions changed into[", child.ID(), parent.ID()))
fmt.Sprintf("%v_%v is eliminated, %v_%v's expressions changed into[", child.TP(), child.ID(), parent.TP(), parent.ID()))
for i, expr := range parent.Exprs {
if i > 0 {
buffer.WriteString(",")
Expand All @@ -309,13 +309,19 @@ func appendDupProjEliminateTraceStep(parent, child *LogicalProjection, opt *logi
}
buffer.WriteString("]")
return buffer.String()
}()
reason := fmt.Sprintf("Proj[%v]'s child proj[%v] is redundant", parent.ID(), child.ID())
}
reason := func() string {
return fmt.Sprintf("%v_%v's child %v_%v is redundant", parent.TP(), parent.ID(), child.TP(), child.ID())
}
opt.appendStepToCurrent(child.ID(), child.TP(), reason, action)
}

func appendProjEliminateTraceStep(proj *LogicalProjection, opt *logicalOptimizeOp) {
reason := fmt.Sprintf("Proj[%v]'s Exprs are all Columns", proj.ID())
action := fmt.Sprintf("Proj[%v] is eliminated", proj.ID())
reason := func() string {
return fmt.Sprintf("%v_%v's Exprs are all Columns", proj.TP(), proj.ID())
}
action := func() string {
return fmt.Sprintf("%v_%v is eliminated", proj.TP(), proj.ID())
}
opt.appendStepToCurrent(proj.ID(), proj.TP(), reason, action)
}
Loading

0 comments on commit beb5451

Please sign in to comment.