diff --git a/pkg/sql/opt/memo/best_expr.go b/pkg/sql/opt/memo/best_expr.go index 1542508e1fcb..bf0efa735b25 100644 --- a/pkg/sql/opt/memo/best_expr.go +++ b/pkg/sql/opt/memo/best_expr.go @@ -82,6 +82,15 @@ func (be *BestExpr) Operator() opt.Operator { return be.op } +// Expr returns the memo expression referenced by this best expression. Note +// that if the best expression is an enforcer (like a Sort), then the memo +// expression is wrapped by the enforcer (maybe even by multiple enforcers). +// This means that the same ExprID can be returned by different best expressions +// in the same group, each of which would have a different Operator type. +func (be *BestExpr) Expr() ExprID { + return be.eid +} + // Group returns the memo group which contains this best expression. func (be *BestExpr) Group() GroupID { return be.eid.Group diff --git a/pkg/sql/opt/memo/expr.go b/pkg/sql/opt/memo/expr.go index 98e86605deb9..016aa5d52fcf 100644 --- a/pkg/sql/opt/memo/expr.go +++ b/pkg/sql/opt/memo/expr.go @@ -39,6 +39,10 @@ type ExprID struct { Expr ExprOrdinal } +// InvalidExprID is the uninitialized ExprID that never points to a valid +// expression. +var InvalidExprID = ExprID{} + // MakeNormExprID returns the id of the normalized expression for the given // group. func MakeNormExprID(group GroupID) ExprID { diff --git a/pkg/sql/opt/memo/expr_view.go b/pkg/sql/opt/memo/expr_view.go index ea3637f95802..a842747cce1a 100644 --- a/pkg/sql/opt/memo/expr_view.go +++ b/pkg/sql/opt/memo/expr_view.go @@ -120,7 +120,7 @@ func (ev ExprView) Physical() *PhysicalProps { if ev.best == normBestOrdinal { panic("physical properties are not available when traversing the normalized tree") } - return ev.mem.LookupPhysicalProps(ev.lookupBestExpr().required) + return ev.mem.LookupPhysicalProps(ev.bestExpr().required) } // Group returns the memo group containing this expression. @@ -138,7 +138,7 @@ func (ev ExprView) Child(nth int) ExprView { group := ev.ChildGroup(nth) return MakeNormExprView(ev.mem, group) } - return MakeExprView(ev.mem, ev.lookupBestExpr().Child(nth)) + return MakeExprView(ev.mem, ev.bestExpr().Child(nth)) } // ChildCount returns the number of expressions that are inputs to this @@ -147,7 +147,7 @@ func (ev ExprView) ChildCount() int { if ev.best == normBestOrdinal { return ev.mem.NormExpr(ev.group).ChildCount() } - return ev.lookupBestExpr().ChildCount() + return ev.bestExpr().ChildCount() } // ChildGroup returns the memo group containing the nth child of this parent @@ -156,7 +156,7 @@ func (ev ExprView) ChildGroup(nth int) GroupID { if ev.best == normBestOrdinal { return ev.mem.NormExpr(ev.group).ChildGroup(ev.mem, nth) } - return ev.lookupBestExpr().Child(nth).group + return ev.bestExpr().Child(nth).group } // Private returns any private data associated with this expression, or nil if @@ -165,7 +165,7 @@ func (ev ExprView) Private() interface{} { if ev.best == normBestOrdinal { return ev.mem.NormExpr(ev.group).Private(ev.mem) } - return ev.mem.Expr(ev.lookupBestExpr().eid).Private(ev.mem) + return ev.mem.Expr(ev.bestExpr().eid).Private(ev.mem) } // Metadata returns the metadata that's specific to this expression tree. Some @@ -175,11 +175,21 @@ func (ev ExprView) Metadata() *opt.Metadata { return ev.mem.metadata } -func (ev ExprView) lookupChildGroup(nth int) *group { +// Cost returns the cost of executing this expression tree, as estimated by the +// optimizer. It is not available when the ExprView is traversing the normalized +// expression tree. +func (ev ExprView) Cost() Cost { + if ev.best == normBestOrdinal { + panic("Cost is not available when traversing the normalized tree") + } + return ev.mem.bestExpr(BestExprID{group: ev.group, ordinal: ev.best}).cost +} + +func (ev ExprView) childGroup(nth int) *group { return ev.mem.group(ev.ChildGroup(nth)) } -func (ev ExprView) lookupBestExpr() *BestExpr { +func (ev ExprView) bestExpr() *BestExpr { return ev.mem.group(ev.group).bestExpr(ev.best) } @@ -328,7 +338,7 @@ func (ev ExprView) formatRelational(tp treeprinter.Node, flags ExprFmtFlags) { } if !flags.HasFlags(ExprFmtHideCost) && ev.best != normBestOrdinal { - tp.Childf("cost: %.2f", ev.lookupBestExpr().cost) + tp.Childf("cost: %.2f", ev.bestExpr().cost) } // Format weak keys. diff --git a/pkg/sql/opt/memo/logical_props_factory.go b/pkg/sql/opt/memo/logical_props_factory.go index 78c823616354..3be2e111dfe5 100644 --- a/pkg/sql/opt/memo/logical_props_factory.go +++ b/pkg/sql/opt/memo/logical_props_factory.go @@ -116,7 +116,7 @@ func (f logicalPropsFactory) constructScanProps(ev ExprView) LogicalProps { func (f logicalPropsFactory) constructSelectProps(ev ExprView) LogicalProps { props := LogicalProps{Relational: &RelationalProps{}} - inputProps := ev.lookupChildGroup(0).logical.Relational + inputProps := ev.childGroup(0).logical.Relational // Inherit input properties as starting point. *props.Relational = *inputProps @@ -129,7 +129,7 @@ func (f logicalPropsFactory) constructSelectProps(ev ExprView) LogicalProps { func (f logicalPropsFactory) constructProjectProps(ev ExprView) LogicalProps { props := LogicalProps{Relational: &RelationalProps{}} - inputProps := ev.lookupChildGroup(0).logical.Relational + inputProps := ev.childGroup(0).logical.Relational // Use output columns from projection list. props.Relational.OutputCols = opt.ColListToSet(ev.Child(1).Private().(opt.ColList)) @@ -154,8 +154,8 @@ func (f logicalPropsFactory) constructProjectProps(ev ExprView) LogicalProps { func (f logicalPropsFactory) constructJoinProps(ev ExprView) LogicalProps { props := LogicalProps{Relational: &RelationalProps{}} - leftProps := ev.lookupChildGroup(0).logical.Relational - rightProps := ev.lookupChildGroup(1).logical.Relational + leftProps := ev.childGroup(0).logical.Relational + rightProps := ev.childGroup(1).logical.Relational // Output columns are union of columns from left and right inputs, except // in case of semi and anti joins, which only project the left columns. @@ -199,7 +199,7 @@ func (f logicalPropsFactory) constructJoinProps(ev ExprView) LogicalProps { func (f logicalPropsFactory) constructGroupByProps(ev ExprView) LogicalProps { props := LogicalProps{Relational: &RelationalProps{}} - inputProps := ev.lookupChildGroup(0).logical.Relational + inputProps := ev.childGroup(0).logical.Relational // Output columns are the union of grouping columns with columns from the // aggregate projection list. @@ -241,8 +241,8 @@ func (f logicalPropsFactory) constructGroupByProps(ev ExprView) LogicalProps { func (f logicalPropsFactory) constructSetProps(ev ExprView) LogicalProps { props := LogicalProps{Relational: &RelationalProps{}} - leftProps := ev.lookupChildGroup(0).logical.Relational - rightProps := ev.lookupChildGroup(1).logical.Relational + leftProps := ev.childGroup(0).logical.Relational + rightProps := ev.childGroup(1).logical.Relational colMap := *ev.Private().(*SetOpColMap) if len(colMap.Out) != len(colMap.Left) || len(colMap.Out) != len(colMap.Right) { panic(fmt.Errorf("lists in SetOpColMap are not all the same length. new:%d, left:%d, right:%d", @@ -311,7 +311,7 @@ func (f logicalPropsFactory) constructMax1RowProps(ev ExprView) LogicalProps { func (f logicalPropsFactory) passThroughRelationalProps(ev ExprView, childIdx int) LogicalProps { // Properties are immutable after construction, so just inherit relational // props pointer from child. - return LogicalProps{Relational: ev.lookupChildGroup(childIdx).logical.Relational} + return LogicalProps{Relational: ev.childGroup(childIdx).logical.Relational} } func (f logicalPropsFactory) constructScalarProps(ev ExprView) LogicalProps { @@ -326,7 +326,7 @@ func (f logicalPropsFactory) constructScalarProps(ev ExprView) LogicalProps { // By default, union outer cols from all children, both relational and scalar. for i := 0; i < ev.ChildCount(); i++ { - logical := &ev.lookupChildGroup(i).logical + logical := &ev.childGroup(i).logical if logical.Scalar != nil { props.Scalar.OuterCols.UnionWith(logical.Scalar.OuterCols) } else { diff --git a/pkg/sql/opt/memo/memo.go b/pkg/sql/opt/memo/memo.go index 9e341e88eff6..e53ce4e38aa3 100644 --- a/pkg/sql/opt/memo/memo.go +++ b/pkg/sql/opt/memo/memo.go @@ -188,8 +188,8 @@ func (m *Memo) GroupProperties(group GroupID) *LogicalProps { return &m.groups[group].logical } -// GroupByFingerprint returns the group of the expression that has the -// given fingerprint. +// GroupByFingerprint returns the group of the expression that has the given +// fingerprint. func (m *Memo) GroupByFingerprint(f Fingerprint) GroupID { return m.exprMap[f] } @@ -252,7 +252,6 @@ func (m *Memo) MemoizeNormExpr(evalCtx *tree.EvalContext, norm Expr) GroupID { if m.exprMap[norm.Fingerprint()] != 0 { panic("normalized expression has been entered into the memo more than once") } - mgrp := m.newGroup(norm) ev := MakeNormExprView(m, mgrp.id) logPropsFactory := logicalPropsFactory{evalCtx: evalCtx} @@ -274,8 +273,7 @@ func (m *Memo) MemoizeDenormExpr(group GroupID, denorm Expr) { } } else { // Add the denormalized expression to the memo. - mgrp := m.group(group) - mgrp.addExpr(denorm) + m.group(group).addExpr(denorm) m.exprMap[denorm.Fingerprint()] = group } } diff --git a/pkg/sql/opt/norm/factory.go b/pkg/sql/opt/norm/factory.go index b4af4f73a514..7777fa5706da 100644 --- a/pkg/sql/opt/norm/factory.go +++ b/pkg/sql/opt/norm/factory.go @@ -25,17 +25,18 @@ import ( ) // MatchedRuleFunc defines the callback function for the NotifyOnMatchedRule -// event. It is invoked each time a normalization rule has been matched by the -// optimizer. The name of the matched rule is passed as a parameter. If the -// function returns false, then the rule is not applied (i.e. it's skipped). +// event supported by the optimizer and factory. It is invoked each time an +// optimization rule (Normalize or Explore) has been matched. The name of the +// matched rule is passed as a parameter. If the function returns false, then +// the rule is not applied (i.e. skipped). type MatchedRuleFunc func(ruleName opt.RuleName) bool -// AppliedRuleFunc defines the callback function for the AppliedRuleFunc event -// supported by the Optimizer and Factory. It is invoked each time an -// optimization rule (Normalize or Explore) has been applied by the optimizer. -// The function is called with the name of the rule and the memo group it -// affected. If the rule was an exploration rule, then the added parameter -// gives the number of expressions added to the group by the rule. +// AppliedRuleFunc defines the callback function for the NotifyOnAppliedRule +// event supported by the optimizer and factory. It is invoked each time an +// optimization rule (Normalize or Explore) has been applied. The function is +// called with the name of the rule and the memo group it affected. If the rule +// was an exploration rule, then the added parameter gives the number of +// expressions added to the group by the rule. type AppliedRuleFunc func(ruleName opt.RuleName, group memo.GroupID, added int) //go:generate optgen -out factory.og.go factory ../ops/*.opt rules/*.opt @@ -101,9 +102,9 @@ func (f *Factory) DisableOptimizations() { // NotifyOnMatchedRule sets a callback function which is invoked each time a // normalize rule has been matched by the factory. If matchedRule is nil, then -// no further notifications are sent. If no callback function is set, then all -// rules are applied by default. In addition, callers can invoke the -// DisableOptimizations convenience method to disable all rules. +// no further notifications are sent, and all rules are applied by default. In +// addition, callers can invoke the DisableOptimizations convenience method to +// disable all rules. func (f *Factory) NotifyOnMatchedRule(matchedRule MatchedRuleFunc) { f.matchedRule = matchedRule } diff --git a/pkg/sql/opt/norm/testdata/combo b/pkg/sql/opt/norm/testdata/combo index ea0c904b2224..c568772b029a 100644 --- a/pkg/sql/opt/norm/testdata/combo +++ b/pkg/sql/opt/norm/testdata/combo @@ -1,5 +1,12 @@ exec-ddl -CREATE TABLE a (x INT PRIMARY KEY, i INT, f FLOAT, s STRING, j JSON) +CREATE TABLE a ( + x INT PRIMARY KEY, + i INT, + f FLOAT, + s STRING, + j JSON, + UNIQUE INDEX (s DESC, f) STORING (j) +) ---- TABLE a ├── x int not null @@ -7,8 +14,13 @@ TABLE a ├── f float ├── s string ├── j jsonb - └── INDEX primary - └── x int not null + ├── INDEX primary + │ └── x int not null + └── INDEX secondary + ├── s string desc + ├── f float + ├── x int not null (storing) + └── j jsonb (storing) exec-ddl CREATE TABLE t.b (x INT PRIMARY KEY, z INT) @@ -25,15 +37,17 @@ TABLE b optsteps SELECT s FROM a INNER JOIN b ON a.x=b.x AND i+1=10 ---- ----- -*** Initial expr: +================================================================================ +Initial expression + Cost: 2000.00 +================================================================================ project ├── columns: s:4(string) ├── inner-join │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) b.x:6(int!null) b.z:7(int) │ ├── scan a │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) - │ │ └── keys: (1) + │ │ └── keys: (1) weak(3,4) │ ├── scan b │ │ ├── columns: b.x:6(int!null) b.z:7(int) │ │ └── keys: (6) @@ -48,15 +62,17 @@ SELECT s FROM a INNER JOIN b ON a.x=b.x AND i+1=10 │ └── const: 10 [type=int] └── projections [outer=(4)] └── variable: a.s [type=string, outer=(4)] - -*** NormalizeCmpPlusConst applied; best expr changed: +================================================================================ +NormalizeCmpPlusConst + Cost: 2000.00 +================================================================================ project ├── columns: s:4(string) ├── inner-join │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) b.x:6(int!null) b.z:7(int) │ ├── scan a │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) - │ │ └── keys: (1) + │ │ └── keys: (1) weak(3,4) │ ├── scan b │ │ ├── columns: b.x:6(int!null) b.z:7(int) │ │ └── keys: (6) @@ -75,15 +91,17 @@ SELECT s FROM a INNER JOIN b ON a.x=b.x AND i+1=10 + │ └── const: 1 [type=int] └── projections [outer=(4)] └── variable: a.s [type=string, outer=(4)] - -*** EnsureJoinFiltersAnd applied; best expr changed: +================================================================================ +EnsureJoinFiltersAnd + Cost: 2000.00 +================================================================================ project ├── columns: s:4(string) ├── inner-join │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) b.x:6(int!null) b.z:7(int) │ ├── scan a │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) - │ │ └── keys: (1) + │ │ └── keys: (1) weak(3,4) │ ├── scan b │ │ ├── columns: b.x:6(int!null) b.z:7(int) │ │ └── keys: (6) @@ -99,8 +117,10 @@ SELECT s FROM a INNER JOIN b ON a.x=b.x AND i+1=10 │ └── const: 1 [type=int] └── projections [outer=(4)] └── variable: a.s [type=string, outer=(4)] - -*** PushFilterIntoJoinLeft applied; best expr changed: +================================================================================ +PushFilterIntoJoinLeft + Cost: 2100.00 +================================================================================ project ├── columns: s:4(string) ├── inner-join @@ -108,11 +128,11 @@ SELECT s FROM a INNER JOIN b ON a.x=b.x AND i+1=10 - │ ├── scan a + │ ├── select │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) - - │ │ └── keys: (1) - + │ │ ├── keys: (1) + - │ │ └── keys: (1) weak(3,4) + + │ │ ├── keys: (1) weak(3,4) + │ │ ├── scan a + │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) - + │ │ │ └── keys: (1) + + │ │ │ └── keys: (1) weak(3,4) + │ │ └── filters [type=bool, outer=(2)] + │ │ └── eq [type=bool, outer=(2)] + │ │ ├── variable: a.i [type=int, outer=(2)] @@ -137,32 +157,35 @@ SELECT s FROM a INNER JOIN b ON a.x=b.x AND i+1=10 + │ └── variable: b.x [type=int, outer=(6)] └── projections [outer=(4)] └── variable: a.s [type=string, outer=(4)] - -*** FilterUnusedJoinLeftCols applied; best expr changed: +================================================================================ +FilterUnusedJoinLeftCols + Cost: 2100.00 +================================================================================ project ├── columns: s:4(string) ├── inner-join - │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) b.x:6(int!null) b.z:7(int) - │ ├── select - │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) + - │ │ ├── keys: (1) weak(3,4) + - │ │ ├── scan a + │ ├── columns: a.x:1(int!null) a.s:4(string) b.x:6(int!null) b.z:7(int) + │ ├── project + │ │ ├── columns: a.x:1(int!null) a.s:4(string) - │ │ ├── keys: (1) - - │ │ ├── scan a + + │ │ ├── keys: (1) + │ │ ├── select │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) - - │ │ │ └── keys: (1) + - │ │ │ └── keys: (1) weak(3,4) - │ │ └── filters [type=bool, outer=(2)] - │ │ └── eq [type=bool, outer=(2)] - │ │ ├── variable: a.i [type=int, outer=(2)] - │ │ └── minus [type=int] - │ │ ├── const: 10 [type=int] - │ │ └── const: 1 [type=int] - + │ │ │ ├── keys: (1) + + │ │ │ ├── keys: (1) weak(3,4) + │ │ │ ├── scan a + │ │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) - + │ │ │ │ └── keys: (1) + + │ │ │ │ └── keys: (1) weak(3,4) + │ │ │ └── filters [type=bool, outer=(2)] + │ │ │ └── eq [type=bool, outer=(2)] + │ │ │ ├── variable: a.i [type=int, outer=(2)] @@ -181,8 +204,10 @@ SELECT s FROM a INNER JOIN b ON a.x=b.x AND i+1=10 │ └── variable: b.x [type=int, outer=(6)] └── projections [outer=(4)] └── variable: a.s [type=string, outer=(4)] - -*** FilterUnusedSelectCols applied; best expr changed: +================================================================================ +FilterUnusedSelectCols + Cost: 2100.00 +================================================================================ project ├── columns: s:4(string) ├── inner-join @@ -192,12 +217,14 @@ SELECT s FROM a INNER JOIN b ON a.x=b.x AND i+1=10 │ │ ├── keys: (1) │ │ ├── select - │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) + - │ │ │ ├── keys: (1) weak(3,4) + │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.s:4(string) - │ │ │ ├── keys: (1) + + │ │ │ ├── keys: (1) │ │ │ ├── scan a - │ │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) + - │ │ │ │ └── keys: (1) weak(3,4) + │ │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.s:4(string) - │ │ │ │ └── keys: (1) + + │ │ │ │ └── keys: (1) │ │ │ └── filters [type=bool, outer=(2)] │ │ │ └── eq [type=bool, outer=(2)] │ │ │ ├── variable: a.i [type=int, outer=(2)] @@ -216,8 +243,10 @@ SELECT s FROM a INNER JOIN b ON a.x=b.x AND i+1=10 │ └── variable: b.x [type=int, outer=(6)] └── projections [outer=(4)] └── variable: a.s [type=string, outer=(4)] - -*** FilterUnusedJoinRightCols applied; best expr changed: +================================================================================ +FilterUnusedJoinRightCols + Cost: 2100.00 +================================================================================ project ├── columns: s:4(string) ├── inner-join @@ -251,14 +280,19 @@ SELECT s FROM a INNER JOIN b ON a.x=b.x AND i+1=10 │ └── variable: b.x [type=int, outer=(6)] └── projections [outer=(4)] └── variable: a.s [type=string, outer=(4)] - -*** GenerateIndexScans applied; best expr unchanged. - -*** ConstrainScan applied; best expr unchanged. - -*** GenerateIndexScans applied; best expr unchanged. - -*** Final best expr: +-------------------------------------------------------------------------------- +GenerateIndexScans (no changes) +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- +ConstrainScan (no changes) +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- +GenerateIndexScans (no changes) +-------------------------------------------------------------------------------- +================================================================================ +Final best expression + Cost: 2100.00 +================================================================================ project ├── columns: s:4(string) ├── inner-join @@ -290,15 +324,16 @@ SELECT s FROM a INNER JOIN b ON a.x=b.x AND i+1=10 │ └── variable: b.x [type=int, outer=(6)] └── projections [outer=(4)] └── variable: a.s [type=string, outer=(4)] ----- ----- + # Select/Project/Limit/Offset rules have cyclical dependencies. optsteps SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET 1 ---- ----- -*** Initial expr: +================================================================================ +Initial expression + Cost: 14500.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -316,7 +351,7 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ │ │ ├── columns: a.x:1(int) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) b.x:6(int) b.z:7(int) │ │ │ │ ├── scan a │ │ │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) - │ │ │ │ │ └── keys: (1) + │ │ │ │ │ └── keys: (1) weak(3,4) │ │ │ │ ├── scan b │ │ │ │ │ ├── columns: b.x:6(int!null) b.z:7(int) │ │ │ │ │ └── keys: (6) @@ -333,8 +368,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ └── const: 1 [type=int] │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** EnsureJoinFilters applied; best expr changed: +================================================================================ +EnsureJoinFilters + Cost: 14500.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -352,7 +389,7 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ │ │ ├── columns: a.x:1(int) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) b.x:6(int) b.z:7(int) │ │ │ │ ├── scan a │ │ │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) - │ │ │ │ │ └── keys: (1) + │ │ │ │ │ └── keys: (1) weak(3,4) │ │ │ │ ├── scan b │ │ │ │ │ ├── columns: b.x:6(int!null) b.z:7(int) │ │ │ │ │ └── keys: (6) @@ -373,8 +410,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ └── const: 1 [type=int] │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** EnsureSelectFilters applied; best expr changed: +================================================================================ +EnsureSelectFilters + Cost: 14500.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -392,7 +431,7 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ │ │ ├── columns: a.x:1(int) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) b.x:6(int) b.z:7(int) │ │ │ │ ├── scan a │ │ │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) - │ │ │ │ │ └── keys: (1) + │ │ │ │ │ └── keys: (1) weak(3,4) │ │ │ │ ├── scan b │ │ │ │ │ ├── columns: b.x:6(int!null) b.z:7(int) │ │ │ │ │ └── keys: (6) @@ -414,8 +453,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ └── const: 1 [type=int] │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** FilterUnusedSelectCols applied; best expr changed: +================================================================================ +FilterUnusedSelectCols + Cost: 14500.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -433,7 +474,7 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET - │ │ │ │ ├── columns: a.x:1(int) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) b.x:6(int) b.z:7(int) - │ │ │ │ ├── scan a - │ │ │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) - - │ │ │ │ │ └── keys: (1) + - │ │ │ │ │ └── keys: (1) weak(3,4) - │ │ │ │ ├── scan b - │ │ │ │ │ ├── columns: b.x:6(int!null) b.z:7(int) - │ │ │ │ │ └── keys: (6) @@ -448,7 +489,7 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET + │ │ │ │ │ ├── columns: a.x:1(int) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) b.x:6(int) b.z:7(int) + │ │ │ │ │ ├── scan a + │ │ │ │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) - + │ │ │ │ │ │ └── keys: (1) + + │ │ │ │ │ │ └── keys: (1) weak(3,4) + │ │ │ │ │ ├── scan b + │ │ │ │ │ │ ├── columns: b.x:6(int!null) b.z:7(int) + │ │ │ │ │ │ └── keys: (6) @@ -469,8 +510,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ └── const: 1 [type=int] │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** FilterUnusedJoinLeftCols applied; best expr changed: +================================================================================ +FilterUnusedJoinLeftCols + Cost: 14500.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -491,8 +534,9 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET + │ │ │ │ │ ├── columns: a.x:1(int) a.i:2(int) b.x:6(int) b.z:7(int) │ │ │ │ │ ├── scan a - │ │ │ │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) + - │ │ │ │ │ │ └── keys: (1) weak(3,4) + │ │ │ │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) - │ │ │ │ │ │ └── keys: (1) + + │ │ │ │ │ │ └── keys: (1) │ │ │ │ │ ├── scan b │ │ │ │ │ │ ├── columns: b.x:6(int!null) b.z:7(int) │ │ │ │ │ │ └── keys: (6) @@ -513,8 +557,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ └── const: 1 [type=int] │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** FilterUnusedJoinRightCols applied; best expr changed: +================================================================================ +FilterUnusedJoinRightCols + Cost: 14500.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -557,8 +603,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ └── const: 1 [type=int] │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** PushSelectIntoProject applied; best expr changed: +================================================================================ +PushSelectIntoProject + Cost: 13250.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -617,8 +665,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ └── const: 1 [type=int] │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** EliminateEmptyAnd applied; best expr changed: +================================================================================ +EliminateEmptyAnd + Cost: 13250.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -663,8 +713,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ └── const: 1 [type=int] │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** EliminateSelect applied; best expr changed: +================================================================================ +EliminateSelect + Cost: 14500.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -728,8 +780,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ └── const: 1 [type=int] │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** EliminateProjectProject applied; best expr changed: +================================================================================ +EliminateProjectProject + Cost: 14500.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -787,8 +841,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ └── const: 1 [type=int] │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** FilterUnusedSelectCols applied; best expr changed: +================================================================================ +FilterUnusedSelectCols + Cost: 14500.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -846,8 +902,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ └── const: 1 [type=int] │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** EliminateProjectProject applied; best expr changed: +================================================================================ +EliminateProjectProject + Cost: 14500.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -905,8 +963,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ │ └── const: 1 [type=int] │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** PushOffsetIntoProject applied; best expr changed: +================================================================================ +PushOffsetIntoProject + Cost: 14500.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -973,8 +1033,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET + │ ├── variable: a.i [type=int, outer=(2)] + │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** FilterUnusedOffsetCols applied; best expr changed: +================================================================================ +FilterUnusedOffsetCols + Cost: 14500.00 +================================================================================ limit ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -1035,8 +1097,10 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET │ ├── variable: a.i [type=int, outer=(2)] │ └── const: 1 [type=int] └── const: 5 [type=int] - -*** PushLimitIntoProject applied; best expr changed: +================================================================================ +PushLimitIntoProject + Cost: 14500.00 +================================================================================ -limit +project ├── columns: i:2(int) column8:8(int) @@ -1087,12 +1151,16 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET + └── plus [type=int, outer=(2)] + ├── variable: a.i [type=int, outer=(2)] + └── const: 1 [type=int] - -*** GenerateIndexScans applied; best expr unchanged. - -*** GenerateIndexScans applied; best expr unchanged. - -*** Final best expr: +-------------------------------------------------------------------------------- +GenerateIndexScans (no changes) +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- +GenerateIndexScans (no changes) +-------------------------------------------------------------------------------- +================================================================================ +Final best expression + Cost: 14500.00 +================================================================================ project ├── columns: i:2(int) column8:8(int) ├── ordering: +2 @@ -1134,16 +1202,17 @@ SELECT i, i+1 FROM a FULL JOIN b ON a.x=b.x WHERE i=10 ORDER BY i LIMIT 5 OFFSET └── plus [type=int, outer=(2)] ├── variable: a.i [type=int, outer=(2)] └── const: 1 [type=int] ----- ----- + # Cyclical rules that trigger assert in AddAltFingerprint without extra code to # check whether nested rule has already called AddAltFingerprint. optsteps SELECT SUM(x) FROM a GROUP BY s, x HAVING SUM(x)=1 ---- ----- -*** Initial expr: +================================================================================ +Initial expression + Cost: 1010.00 +================================================================================ project ├── columns: column6:6(decimal) ├── select @@ -1158,7 +1227,7 @@ SELECT SUM(x) FROM a GROUP BY s, x HAVING SUM(x)=1 │ │ │ ├── keys: (1) │ │ │ ├── scan a │ │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) - │ │ │ │ └── keys: (1) + │ │ │ │ └── keys: (1) weak(3,4) │ │ │ └── projections [outer=(1,4)] │ │ │ ├── variable: a.s [type=string, outer=(4)] │ │ │ └── variable: a.x [type=int, outer=(1)] @@ -1170,8 +1239,10 @@ SELECT SUM(x) FROM a GROUP BY s, x HAVING SUM(x)=1 │ └── const: 1 [type=decimal] └── projections [outer=(6)] └── variable: column6 [type=decimal, outer=(6)] - -*** FilterUnusedScanCols applied; best expr changed: +================================================================================ +FilterUnusedScanCols + Cost: 1010.00 +================================================================================ project ├── columns: column6:6(decimal) ├── select @@ -1186,8 +1257,9 @@ SELECT SUM(x) FROM a GROUP BY s, x HAVING SUM(x)=1 │ │ │ ├── keys: (1) │ │ │ ├── scan a - │ │ │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) + - │ │ │ │ └── keys: (1) weak(3,4) + │ │ │ │ ├── columns: a.x:1(int!null) a.s:4(string) - │ │ │ │ └── keys: (1) + + │ │ │ │ └── keys: (1) │ │ │ └── projections [outer=(1,4)] │ │ │ ├── variable: a.s [type=string, outer=(4)] │ │ │ └── variable: a.x [type=int, outer=(1)] @@ -1199,8 +1271,10 @@ SELECT SUM(x) FROM a GROUP BY s, x HAVING SUM(x)=1 │ └── const: 1 [type=decimal] └── projections [outer=(6)] └── variable: column6 [type=decimal, outer=(6)] - -*** EliminateProject applied; best expr changed: +================================================================================ +EliminateProject + Cost: 1010.00 +================================================================================ project ├── columns: column6:6(decimal) ├── select @@ -1230,8 +1304,10 @@ SELECT SUM(x) FROM a GROUP BY s, x HAVING SUM(x)=1 │ └── const: 1 [type=decimal] └── projections [outer=(6)] └── variable: column6 [type=decimal, outer=(6)] - -*** EnsureSelectFilters applied; best expr changed: +================================================================================ +EnsureSelectFilters + Cost: 1010.00 +================================================================================ project ├── columns: column6:6(decimal) ├── select @@ -1256,8 +1332,10 @@ SELECT SUM(x) FROM a GROUP BY s, x HAVING SUM(x)=1 + │ └── const: 1 [type=decimal] └── projections [outer=(6)] └── variable: column6 [type=decimal, outer=(6)] - -*** FilterUnusedSelectCols applied; best expr changed: +================================================================================ +FilterUnusedSelectCols + Cost: 1010.00 +================================================================================ project ├── columns: column6:6(decimal) ├── select @@ -1294,8 +1372,10 @@ SELECT SUM(x) FROM a GROUP BY s, x HAVING SUM(x)=1 │ └── const: 1 [type=decimal] └── projections [outer=(6)] └── variable: column6 [type=decimal, outer=(6)] - -*** PushSelectIntoProject applied; best expr changed: +================================================================================ +PushSelectIntoProject + Cost: 1011.00 +================================================================================ project ├── columns: column6:6(decimal) ├── select @@ -1336,8 +1416,10 @@ SELECT SUM(x) FROM a GROUP BY s, x HAVING SUM(x)=1 + │ └── filters [type=bool] └── projections [outer=(6)] └── variable: column6 [type=decimal, outer=(6)] - -*** EliminateEmptyAnd applied; best expr changed: +================================================================================ +EliminateEmptyAnd + Cost: 1011.00 +================================================================================ project ├── columns: column6:6(decimal) ├── select @@ -1367,8 +1449,10 @@ SELECT SUM(x) FROM a GROUP BY s, x HAVING SUM(x)=1 + │ └── true [type=bool] └── projections [outer=(6)] └── variable: column6 [type=decimal, outer=(6)] - -*** EliminateSelect applied; best expr changed: +================================================================================ +EliminateSelect + Cost: 1010.00 +================================================================================ project ├── columns: column6:6(decimal) - ├── select @@ -1415,8 +1499,10 @@ SELECT SUM(x) FROM a GROUP BY s, x HAVING SUM(x)=1 + │ └── variable: column6 [type=decimal, outer=(6)] └── projections [outer=(6)] └── variable: column6 [type=decimal, outer=(6)] - -*** EliminateProject applied; best expr changed: +================================================================================ +EliminateProject + Cost: 1010.00 +================================================================================ project ├── columns: column6:6(decimal) - ├── project @@ -1457,10 +1543,35 @@ SELECT SUM(x) FROM a GROUP BY s, x HAVING SUM(x)=1 + │ └── const: 1 [type=decimal] └── projections [outer=(6)] └── variable: column6 [type=decimal, outer=(6)] - -*** GenerateIndexScans applied; best expr unchanged. - -*** Final best expr: +-------------------------------------------------------------------------------- +GenerateIndexScans (higher cost) +-------------------------------------------------------------------------------- + project + ├── columns: column6:6(decimal) + ├── select + │ ├── columns: a.x:1(int!null) a.s:4(string) column6:6(decimal) + │ ├── keys: (1) + │ ├── group-by + │ │ ├── columns: a.x:1(int!null) a.s:4(string) column6:6(decimal) + │ │ ├── grouping columns: a.x:1(int!null) a.s:4(string) + │ │ ├── keys: (1) + - │ │ ├── scan a + + │ │ ├── scan a@secondary + │ │ │ ├── columns: a.x:1(int!null) a.s:4(string) + │ │ │ └── keys: (1) + │ │ └── aggregations [outer=(1)] + │ │ └── function: sum [type=decimal, outer=(1)] + │ │ └── variable: a.x [type=int, outer=(1)] + │ └── filters [type=bool, outer=(6), constraints=(/6: [/1 - /1]; tight)] + │ └── eq [type=bool, outer=(6), constraints=(/6: [/1 - /1]; tight)] + │ ├── variable: column6 [type=decimal, outer=(6)] + │ └── const: 1 [type=decimal] + └── projections [outer=(6)] + └── variable: column6 [type=decimal, outer=(6)] +================================================================================ +Final best expression + Cost: 1010.00 +================================================================================ project ├── columns: column6:6(decimal) ├── select @@ -1482,5 +1593,190 @@ SELECT SUM(x) FROM a GROUP BY s, x HAVING SUM(x)=1 │ └── const: 1 [type=decimal] └── projections [outer=(6)] └── variable: column6 [type=decimal, outer=(6)] + +optsteps +SELECT s, x FROM a WHERE s='foo' AND f>100 ---- ----- +================================================================================ +Initial expression + Cost: 1100.00 +================================================================================ + project + ├── columns: s:4(string) x:1(int!null) + ├── keys: (1) + ├── select + │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) + │ ├── keys: (1) weak(3,4) + │ ├── scan a + │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) + │ │ └── keys: (1) weak(3,4) + │ └── and [type=bool, outer=(3,4), constraints=(/3: [/100.00000000000001 - ]; /4: [/'foo' - /'foo']; tight)] + │ ├── eq [type=bool, outer=(4), constraints=(/4: [/'foo' - /'foo']; tight)] + │ │ ├── variable: a.s [type=string, outer=(4)] + │ │ └── const: 'foo' [type=string] + │ └── gt [type=bool, outer=(3), constraints=(/3: [/100.00000000000001 - ]; tight)] + │ ├── variable: a.f [type=float, outer=(3)] + │ └── const: 100.0 [type=float] + └── projections [outer=(1,4)] + ├── variable: a.s [type=string, outer=(4)] + └── variable: a.x [type=int, outer=(1)] +================================================================================ +EnsureSelectFiltersAnd + Cost: 1100.00 +================================================================================ + project + ├── columns: s:4(string) x:1(int!null) + ├── keys: (1) + ├── select + │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) + │ ├── keys: (1) weak(3,4) + │ ├── scan a + │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) + │ │ └── keys: (1) weak(3,4) + - │ └── and [type=bool, outer=(3,4), constraints=(/3: [/100.00000000000001 - ]; /4: [/'foo' - /'foo']; tight)] + + │ └── filters [type=bool, outer=(3,4), constraints=(/3: [/100.00000000000001 - ]; /4: [/'foo' - /'foo']; tight)] + │ ├── eq [type=bool, outer=(4), constraints=(/4: [/'foo' - /'foo']; tight)] + │ │ ├── variable: a.s [type=string, outer=(4)] + │ │ └── const: 'foo' [type=string] + │ └── gt [type=bool, outer=(3), constraints=(/3: [/100.00000000000001 - ]; tight)] + │ ├── variable: a.f [type=float, outer=(3)] + │ └── const: 100.0 [type=float] + └── projections [outer=(1,4)] + ├── variable: a.s [type=string, outer=(4)] + └── variable: a.x [type=int, outer=(1)] +================================================================================ +FilterUnusedSelectCols + Cost: 1100.00 +================================================================================ + project + ├── columns: s:4(string) x:1(int!null) + ├── keys: (1) + ├── select + - │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) + + │ ├── columns: a.x:1(int!null) a.f:3(float) a.s:4(string) + │ ├── keys: (1) weak(3,4) + │ ├── scan a + - │ │ ├── columns: a.x:1(int!null) a.i:2(int) a.f:3(float) a.s:4(string) a.j:5(jsonb) + + │ │ ├── columns: a.x:1(int!null) a.f:3(float) a.s:4(string) + │ │ └── keys: (1) weak(3,4) + │ └── filters [type=bool, outer=(3,4), constraints=(/3: [/100.00000000000001 - ]; /4: [/'foo' - /'foo']; tight)] + │ ├── eq [type=bool, outer=(4), constraints=(/4: [/'foo' - /'foo']; tight)] + │ │ ├── variable: a.s [type=string, outer=(4)] + │ │ └── const: 'foo' [type=string] + │ └── gt [type=bool, outer=(3), constraints=(/3: [/100.00000000000001 - ]; tight)] + │ ├── variable: a.f [type=float, outer=(3)] + │ └── const: 100.0 [type=float] + └── projections [outer=(1,4)] + ├── variable: a.s [type=string, outer=(4)] + └── variable: a.x [type=int, outer=(1)] +-------------------------------------------------------------------------------- +GenerateIndexScans (higher cost) +-------------------------------------------------------------------------------- + project + ├── columns: s:4(string) x:1(int!null) + ├── keys: (1) + ├── select + │ ├── columns: a.x:1(int!null) a.f:3(float) a.s:4(string) + │ ├── keys: (1) weak(3,4) + - │ ├── scan a + + │ ├── scan a@secondary + │ │ ├── columns: a.x:1(int!null) a.f:3(float) a.s:4(string) + │ │ └── keys: (1) weak(3,4) + │ └── filters [type=bool, outer=(3,4), constraints=(/3: [/100.00000000000001 - ]; /4: [/'foo' - /'foo']; tight)] + │ ├── eq [type=bool, outer=(4), constraints=(/4: [/'foo' - /'foo']; tight)] + │ │ ├── variable: a.s [type=string, outer=(4)] + │ │ └── const: 'foo' [type=string] + │ └── gt [type=bool, outer=(3), constraints=(/3: [/100.00000000000001 - ]; tight)] + │ ├── variable: a.f [type=float, outer=(3)] + │ └── const: 100.0 [type=float] + └── projections [outer=(1,4)] + ├── variable: a.s [type=string, outer=(4)] + └── variable: a.x [type=int, outer=(1)] +-------------------------------------------------------------------------------- +ConstrainScan (higher cost) +-------------------------------------------------------------------------------- + project + ├── columns: s:4(string) x:1(int!null) + ├── keys: (1) + ├── select + │ ├── columns: a.x:1(int!null) a.f:3(float) a.s:4(string) + │ ├── keys: (1) weak(3,4) + - │ ├── scan a@secondary + + │ ├── scan a + │ │ ├── columns: a.x:1(int!null) a.f:3(float) a.s:4(string) + │ │ └── keys: (1) weak(3,4) + │ └── filters [type=bool, outer=(3,4), constraints=(/3: [/100.00000000000001 - ]; /4: [/'foo' - /'foo']; tight)] + │ ├── eq [type=bool, outer=(4), constraints=(/4: [/'foo' - /'foo']; tight)] + │ │ ├── variable: a.s [type=string, outer=(4)] + │ │ └── const: 'foo' [type=string] + │ └── gt [type=bool, outer=(3), constraints=(/3: [/100.00000000000001 - ]; tight)] + │ ├── variable: a.f [type=float, outer=(3)] + │ └── const: 100.0 [type=float] + └── projections [outer=(1,4)] + ├── variable: a.s [type=string, outer=(4)] + └── variable: a.x [type=int, outer=(1)] +================================================================================ +ConstrainScan + Cost: 110.00 +================================================================================ + project + ├── columns: s:4(string) x:1(int!null) + ├── keys: (1) + ├── select + │ ├── columns: a.x:1(int!null) a.f:3(float) a.s:4(string) + │ ├── keys: (1) weak(3,4) + - │ ├── scan a + + │ ├── scan a@secondary + │ │ ├── columns: a.x:1(int!null) a.f:3(float) a.s:4(string) + + │ │ ├── constraint: /-4/3: [/'foo'/100.00000000000001 - /'foo'] + │ │ └── keys: (1) weak(3,4) + - │ └── filters [type=bool, outer=(3,4), constraints=(/3: [/100.00000000000001 - ]; /4: [/'foo' - /'foo']; tight)] + - │ ├── eq [type=bool, outer=(4), constraints=(/4: [/'foo' - /'foo']; tight)] + - │ │ ├── variable: a.s [type=string, outer=(4)] + - │ │ └── const: 'foo' [type=string] + - │ └── gt [type=bool, outer=(3), constraints=(/3: [/100.00000000000001 - ]; tight)] + - │ ├── variable: a.f [type=float, outer=(3)] + - │ └── const: 100.0 [type=float] + + │ └── filters [type=bool] + + │ ├── true [type=bool] + + │ └── true [type=bool] + └── projections [outer=(1,4)] + ├── variable: a.s [type=string, outer=(4)] + └── variable: a.x [type=int, outer=(1)] +================================================================================ +SimplifyFilters + Cost: 100.00 +================================================================================ + project + ├── columns: s:4(string) x:1(int!null) + ├── keys: (1) + - ├── select + + ├── scan a@secondary + │ ├── columns: a.x:1(int!null) a.f:3(float) a.s:4(string) + - │ ├── keys: (1) weak(3,4) + - │ ├── scan a@secondary + - │ │ ├── columns: a.x:1(int!null) a.f:3(float) a.s:4(string) + - │ │ ├── constraint: /-4/3: [/'foo'/100.00000000000001 - /'foo'] + - │ │ └── keys: (1) weak(3,4) + - │ └── filters [type=bool] + - │ ├── true [type=bool] + - │ └── true [type=bool] + + │ ├── constraint: /-4/3: [/'foo'/100.00000000000001 - /'foo'] + + │ └── keys: (1) weak(3,4) + └── projections [outer=(1,4)] + ├── variable: a.s [type=string, outer=(4)] + └── variable: a.x [type=int, outer=(1)] +================================================================================ +Final best expression + Cost: 100.00 +================================================================================ + project + ├── columns: s:4(string) x:1(int!null) + ├── keys: (1) + ├── scan a@secondary + │ ├── columns: a.x:1(int!null) a.f:3(float) a.s:4(string) + │ ├── constraint: /-4/3: [/'foo'/100.00000000000001 - /'foo'] + │ └── keys: (1) weak(3,4) + └── projections [outer=(1,4)] + ├── variable: a.s [type=string, outer=(4)] + └── variable: a.x [type=int, outer=(1)] diff --git a/pkg/sql/opt/norm/testdata/comp b/pkg/sql/opt/norm/testdata/comp index a15fceb2ae2e..05321dac9b27 100644 --- a/pkg/sql/opt/norm/testdata/comp +++ b/pkg/sql/opt/norm/testdata/comp @@ -451,4 +451,3 @@ project └── projections ├── true [type=bool] └── false [type=bool] - diff --git a/pkg/sql/opt/optgen/cmd/optgen/rule_gen.go b/pkg/sql/opt/optgen/cmd/optgen/rule_gen.go index 9f4f7adb2004..70a6b0a2bbaa 100644 --- a/pkg/sql/opt/optgen/cmd/optgen/rule_gen.go +++ b/pkg/sql/opt/optgen/cmd/optgen/rule_gen.go @@ -642,12 +642,12 @@ func (g *ruleGen) genExploreReplace(define *lang.DefineExpr, rule *lang.RuleExpr case *lang.CustomFuncExpr: // Top-level custom function returns a memo.Expr slice, so iterate // through that and memoize each expression. - g.w.writeIndent("exprs := ") + g.w.writeIndent("_exprs := ") g.genNestedExpr(rule.Replace) g.w.newline() g.w.writeIndent("_before := _e.mem.ExprCount(_root.Group)\n") - g.w.nestIndent("for i := range exprs {\n") - g.w.writeIndent("_e.mem.MemoizeDenormExpr(_root.Group, exprs[i])\n") + g.w.nestIndent("for i := range _exprs {\n") + g.w.writeIndent("_e.mem.MemoizeDenormExpr(_root.Group, _exprs[i])\n") g.w.unnest("}\n") default: diff --git a/pkg/sql/opt/optgen/cmd/optgen/testdata/explorer b/pkg/sql/opt/optgen/cmd/optgen/testdata/explorer index 4729b3e90857..c7f175e1ae23 100644 --- a/pkg/sql/opt/optgen/cmd/optgen/testdata/explorer +++ b/pkg/sql/opt/optgen/cmd/optgen/testdata/explorer @@ -282,10 +282,10 @@ func (_e *explorer) exploreScan(_rootState *exploreState, _root memo.ExprID) (_f def := _rootExpr.Def() if _e.isPrimaryScan(def) { if _e.o.matchedRule == nil || _e.o.matchedRule(opt.GenerateIndexScans) { - exprs := _e.generateIndexScans(def) + _exprs := _e.generateIndexScans(def) _before := _e.mem.ExprCount(_root.Group) - for i := range exprs { - _e.mem.MemoizeDenormExpr(_root.Group, exprs[i]) + for i := range _exprs { + _e.mem.MemoizeDenormExpr(_root.Group, _exprs[i]) } if _e.o.appliedRule != nil { _after := _e.mem.ExprCount(_root.Group) diff --git a/pkg/sql/opt/testutils/opt_steps.go b/pkg/sql/opt/testutils/opt_steps.go new file mode 100644 index 000000000000..35b73299c4e8 --- /dev/null +++ b/pkg/sql/opt/testutils/opt_steps.go @@ -0,0 +1,283 @@ +// Copyright 2018 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package testutils + +import ( + "github.com/cockroachdb/cockroach/pkg/sql/opt" + "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" + "github.com/cockroachdb/cockroach/pkg/sql/opt/xform" +) + +// optSteps implements the stepping algorithm used by the OptTester's OptSteps +// command. See the OptTester.OptSteps comment for more details on the command. +// +// The algorithm works as follows: +// 1. The first time optSteps.next() is called, optSteps returns the starting +// expression tree, with no transformations applied to it. +// +// 2. Each optSteps.next() call after that will perform N+1 transformations, +// where N is the the number of steps performed during the previous call +// (starting at 0 with the first call). +// +// 3. Each optSteps.next() call will build the expression tree from scratch +// and re-run all transformations that were run in the previous call, plus +// one additional transformation (N+1). Therefore, the output expression +// tree from each call will differ from the previous call only by the last +// transformation's changes. +// +// 4. optSteps hooks the optimizer's MatchedRule event in order to limit the +// number of transformations that can be applied, as well as to record the +// name of the last rule that was applied, for later output. +// +// 5. While this works well for normalization rules, exploration rules are +// more difficult. This is because exploration rules are not guaranteed to +// produce a lower cost tree. Unless extra measures are taken, the returned +// ExprView would not include the changed portion of the Memo, since +// ExprView only shows the lowest cost path through the Memo. +// +// 6. To address this issue, optSteps hooks the optimizer's AppliedRule event +// and records the expression(s) that the last transformation has affected. +// It then re-runs the optimizer, but this time using a special Coster +// implementation that fools the optimizer into thinking that the new +// expression(s) have the lowest cost. The coster does this by assigning an +// infinite cost to all other expressions in the same group as the new +// expression(s), as well as in all ancestor groups. +// +type optSteps struct { + tester *OptTester + + // steps is the maximum number of rules that can be applied by the optimizer + // during the current iteration. + steps int + + // remaining is the number of "unused" steps remaining in this iteration. + remaining int + + // lastMatched records the name of the rule that was most recently matched + // by the optimizer. + lastMatched opt.RuleName + + // lastApplied records the id of the expression that marks the portion of the + // tree affected by the most recent rule application. All expressions in the + // same memo group that are < lastApplied.Expr will assigned an infinite cost + // by the forcingCoster. Therefore, only expressions >= lastApplied.Expr can + // be in the output expression tree. + lastApplied memo.ExprID + + // ev is the expression tree produced by the most recent optSteps iteration. + ev memo.ExprView + + // better is true if ev is lower cost than the expression tree produced by + // the previous iteration of optSteps. + better bool + + // best is the textual representation of the most recent expression tree that + // was an improvement over the previous best tree. + best string +} + +func newOptSteps(tester *OptTester) *optSteps { + return &optSteps{tester: tester} +} + +// exprView returns the expression tree produced by the most recent optSteps +// iteration. +func (os *optSteps) exprView() memo.ExprView { + return os.ev +} + +// lastRuleName returns the name of the rule that was most recently matched by +// the optimizer. +func (os *optSteps) lastRuleName() opt.RuleName { + return os.lastMatched +} + +// isBetter returns true if exprView is lower cost than the expression tree +// produced by the previous iteration of optSteps. +func (os *optSteps) isBetter() bool { + return os.better +} + +// done returns true if there are no more rules to apply. Further calls to the +// next method will result in a panic. +func (os *optSteps) done() bool { + // remaining starts out equal to steps, and is decremented each time a rule + // is applied. If it never reaches zero, then all possible rules were + // already applied, and optimization is complete. + return os.remaining != 0 +} + +// next triggers the next iteration of optSteps. If there is no error, then +// results of the iteration can be accessed via the exprView, lastRuleName, and +// isBetter methods. +func (os *optSteps) next() error { + if os.done() { + panic("iteration already complete") + } + + // Create optimizer that will run for a fixed number of steps. + o := os.createOptimizer(os.steps) + root, required, err := os.tester.buildExpr(o.Factory()) + if err != nil { + return err + } + + // Hook the AppliedRule notification in order to track the portion of the + // expression tree affected by each transformation rule. + os.lastApplied = memo.InvalidExprID + o.NotifyOnAppliedRule(func(ruleName opt.RuleName, group memo.GroupID, added int) { + if added > 0 { + // This was an exploration rule that added one or more expressions to + // an existing group. Record the id of the first of those expressions. + // Previous expressions will be suppressed. + ord := memo.ExprOrdinal(o.Memo().ExprCount(group) - added) + os.lastApplied = memo.ExprID{Group: group, Expr: ord} + } else { + // This was a normalization that created a new memo group, or it was + // an exploration rule that didn't add any expressions to the group. + // Either way, none of the expressions in the group need to be + // suppressed. + os.lastApplied = memo.MakeNormExprID(group) + } + }) + + os.ev = o.Optimize(root, required) + text := os.ev.String() + + // If the expression text changes, then it must have gotten better. + os.better = text != os.best + if os.better { + os.best = text + } else if !os.done() { + // The expression is not better, so suppress the lowest cost expressions + // so that the changed portions of the tree will be part of the output. + o2 := os.createOptimizer(os.steps) + root, required, err := os.tester.buildExpr(o2.Factory()) + if err != nil { + return err + } + + // Set up the coster that will assign infinite costs to the expressions + // that need to be suppressed. + coster := newForcingCoster(o2.Coster()) + os.suppressExprs(coster, o.Memo(), root) + + o2.SetCoster(coster) + os.ev = o2.Optimize(root, required) + } + + os.steps++ + return nil +} + +func (os *optSteps) createOptimizer(steps int) *xform.Optimizer { + o := xform.NewOptimizer(&os.tester.evalCtx) + + // Override NotifyOnMatchedRule to stop optimizing after "steps" rule matches. + os.remaining = steps + o.NotifyOnMatchedRule(func(ruleName opt.RuleName) bool { + if os.remaining == 0 { + return false + } + os.remaining-- + os.lastMatched = ruleName + return true + }) + + return o +} + +// suppressExprs walks the tree and adds expressions which need to be suppressed +// to the forcingCoster. The expressions which need to suppressed are: +// 1. Expressions in the same group as the lastApplied expression, but with +// a lower ordinal position in the group. +// 2. Expressions in ancestor groups of the lastApplied expression that are +// not themselves ancestors of the lastApplied group. +// +// suppressExprs does this by recursively traversing the memo, starting at the +// root group. If a group expression is not an ancestor of the last applied +// group, then it is suppressed. If it is an ancestor, then suppressExprs +// recurses on any child group that is an ancestor. +func (os *optSteps) suppressExprs(coster *forcingCoster, mem *memo.Memo, group memo.GroupID) { + for e := 0; e < mem.ExprCount(group); e++ { + eid := memo.ExprID{Group: group, Expr: memo.ExprOrdinal(e)} + if eid.Group == os.lastApplied.Group { + if eid.Expr < os.lastApplied.Expr { + coster.SuppressExpr(eid) + } + continue + } + + found := false + expr := mem.Expr(eid) + for g := 0; g < expr.ChildCount(); g++ { + child := expr.ChildGroup(mem, g) + if os.findLastApplied(mem, child) { + os.suppressExprs(coster, mem, child) + found = true + break + } + } + + if !found { + coster.SuppressExpr(eid) + } + } +} + +// findLastApplied returns true if the given group is the last applied group or +// one of its ancestor groups. +func (os *optSteps) findLastApplied(mem *memo.Memo, group memo.GroupID) bool { + if group == os.lastApplied.Group { + return true + } + + for e := 0; e < mem.ExprCount(group); e++ { + eid := memo.ExprID{Group: group, Expr: memo.ExprOrdinal(e)} + expr := mem.Expr(eid) + for g := 0; g < expr.ChildCount(); g++ { + if os.findLastApplied(mem, expr.ChildGroup(mem, g)) { + return true + } + } + } + + return false +} + +// forcingCoster implements the xform.Coster interface so that it can suppress +// expressions in the memo that can't be part of the output tree. +type forcingCoster struct { + inner xform.Coster + suppressed map[memo.ExprID]bool +} + +func newForcingCoster(inner xform.Coster) *forcingCoster { + return &forcingCoster{inner: inner, suppressed: make(map[memo.ExprID]bool)} +} + +func (fc *forcingCoster) SuppressExpr(eid memo.ExprID) { + fc.suppressed[eid] = true +} + +// ComputeCost is part of the xform.Coster interface. +func (fc *forcingCoster) ComputeCost(candidate *memo.BestExpr, props *memo.LogicalProps) memo.Cost { + if fc.suppressed[candidate.Expr()] { + // Suppressed expressions get assigned MaxCost so that they never have + // the lowest cost. + return memo.MaxCost + } + return fc.inner.ComputeCost(candidate, props) +} diff --git a/pkg/sql/opt/testutils/opt_tester.go b/pkg/sql/opt/testutils/opt_tester.go index 611dcd5cdbf9..94dcde4412d6 100644 --- a/pkg/sql/opt/testutils/opt_tester.go +++ b/pkg/sql/opt/testutils/opt_tester.go @@ -72,6 +72,10 @@ type OptTesterFlags struct { // an UnsupportedExpr node. This is temporary; it is used for interfacing with // the old planning code. AllowUnsupportedExpr bool + + // Verbose indicates whether verbose test debugging information will be + // output to stdout when commands run. Only certain commands support this. + Verbose bool } // NewOptTester constructs a new instance of the OptTester for the given SQL @@ -123,45 +127,47 @@ func NewOptTester(catalog opt.Catalog, sql string) *OptTester { // // - allow-unsupported: wrap unsupported expressions in UnsupportedOp. // -func (e *OptTester) RunCommand(tb testing.TB, d *datadriven.TestData) string { +func (ot *OptTester) RunCommand(tb testing.TB, d *datadriven.TestData) string { // Allow testcases to override the flags. for _, a := range d.CmdArgs { - if err := e.Flags.Set(a); err != nil { + if err := ot.Flags.Set(a); err != nil { d.Fatalf(tb, "%s", err) } } + ot.Flags.Verbose = testing.Verbose() + switch d.Cmd { case "exec-ddl": - testCatalog, ok := e.catalog.(*TestCatalog) + testCatalog, ok := ot.catalog.(*TestCatalog) if !ok { tb.Fatal("exec-ddl can only be used with TestCatalog") } return ExecuteTestDDL(tb, d.Input, testCatalog) case "build": - ev, err := e.OptBuild() + ev, err := ot.OptBuild() if err != nil { return fmt.Sprintf("error: %s\n", strings.TrimSpace(err.Error())) } - return ev.FormatString(e.Flags.ExprFormat) + return ev.FormatString(ot.Flags.ExprFormat) case "opt": - ev, err := e.Optimize() + ev, err := ot.Optimize() if err != nil { d.Fatalf(tb, "%v", err) } - return ev.FormatString(e.Flags.ExprFormat) + return ev.FormatString(ot.Flags.ExprFormat) case "optsteps": - result, err := e.OptSteps(testing.Verbose()) + result, err := ot.OptSteps() if err != nil { d.Fatalf(tb, "%v", err) } return result case "memo": - result, err := e.Memo() + result, err := ot.Memo() if err != nil { d.Fatalf(tb, "%v", err) } @@ -212,41 +218,63 @@ func (f *OptTesterFlags) Set(arg datadriven.CmdArg) error { // OptBuild constructs an opt expression tree for the SQL query, with no // transformations applied to it. The untouched output of the optbuilder is the // final expression tree. -func (e *OptTester) OptBuild() (memo.ExprView, error) { - return e.optimizeExpr(false /* allowOptimizations */) +func (ot *OptTester) OptBuild() (memo.ExprView, error) { + return ot.optimizeExpr(false /* allowOptimizations */) } // Optimize constructs an opt expression tree for the SQL query, with all // transformations applied to it. The result is the memo expression tree with // the lowest estimated cost. -func (e *OptTester) Optimize() (memo.ExprView, error) { - return e.optimizeExpr(true /* allowOptimizations */) +func (ot *OptTester) Optimize() (memo.ExprView, error) { + return ot.optimizeExpr(true /* allowOptimizations */) } // Memo returns a string that shows the memo data structure that is constructed // by the optimizer. -func (e *OptTester) Memo() (string, error) { - o := xform.NewOptimizer(&e.evalCtx) - root, required, err := e.buildExpr(o.Factory()) +func (ot *OptTester) Memo() (string, error) { + o := xform.NewOptimizer(&ot.evalCtx) + root, required, err := ot.buildExpr(o.Factory()) if err != nil { return "", err } o.Optimize(root, required) - return o.Memo().FormatString(e.Flags.MemoFormat), nil + return o.Memo().FormatString(ot.Flags.MemoFormat), nil } -// OptSteps returns a string that shows each optimization step using the -// standard unified diff format. It is used for debugging the optimizer. -// If verbose is true, each step is also printed on stdout. -func (e *OptTester) OptSteps(verbose bool) (string, error) { +// OptSteps steps through the transformations performed by the optimizer on the +// memo, one-by-one. The output of each step is the lowest cost expression tree +// that also contains the expressions that were changed or added by the +// transformation. The output of each step is diff'd against the output of a +// previous step, using the standard unified diff format. +// +// CREATE TABLE a (x INT PRIMARY KEY, y INT, UNIQUE INDEX (y)) +// +// SELECT x FROM a WHERE x=1 +// +// At the time of this writing, this query triggers 6 rule applications: +// EnsureSelectFilters Wrap Select predicate with Filters operator +// FilterUnusedSelectCols Do not return unused "y" column from Scan +// EliminateProject Remove unneeded Project operator +// GenerateIndexScans Explore scanning "y" index to get "x" values +// ConstrainScan Explore pushing "x=1" into "x" index Scan +// ConstrainScan Explore pushing "x=1" into "y" index Scan +// +// Some steps produce better plans that have a lower execution cost. Other steps +// don't. However, it's useful to see both kinds of steps. The optsteps output +// distinguishes these two cases by using stronger "====" header delimiters when +// a better plan has been found, and weaker "----" header delimiters when not. +// In both cases, the output shows the expressions that were changed or added by +// the rule, even if the total expression tree cost worsened. +// +func (ot *OptTester) OptSteps() (string, error) { var buf bytes.Buffer - var prev, next string - if verbose { + var prevBest, prev, next string + if ot.Flags.Verbose { fmt.Print("------ optsteps verbose output starts ------\n") } output := func(format string, args ...interface{}) { fmt.Fprintf(&buf, format, args...) - if verbose { + if ot.Flags.Verbose { fmt.Printf(format, args...) } } @@ -257,66 +285,90 @@ func (e *OptTester) OptSteps(verbose bool) (string, error) { output(" %s\n", line) } } - for i := 0; ; i++ { - o := xform.NewOptimizer(&e.evalCtx) - - // Override SetOnRuleMatch to stop optimizing after the ith rule matches. - steps := i - lastRuleName := opt.InvalidRuleName - o.NotifyOnMatchedRule(func(ruleName opt.RuleName) bool { - if steps == 0 { - return false - } - steps-- - lastRuleName = ruleName - return true - }) - root, required, err := e.buildExpr(o.Factory()) + // bestHeader is used when the expression is an improvement over the previous + // expression. + bestHeader := func(ev memo.ExprView, format string, args ...interface{}) { + output("%s\n", strings.Repeat("=", 80)) + output(format, args...) + output(" Cost: %.2f\n", ev.Cost()) + output("%s\n", strings.Repeat("=", 80)) + } + + // altHeader is used when the expression doesn't improve over the previous + // expression, but it's still desirable to see what changed. + altHeader := func(format string, args ...interface{}) { + output("%s\n", strings.Repeat("-", 80)) + output(format, args...) + output("%s\n", strings.Repeat("-", 80)) + } + + os := newOptSteps(ot) + for { + err := os.next() if err != nil { return "", err } - next = o.Optimize(root, required).FormatString(e.Flags.ExprFormat) - if steps != 0 { - // All steps were not used, so must be done. + next = os.exprView().FormatString(ot.Flags.ExprFormat) + + // This call comes after setting "next", because we want to output the + // final expression, even though there were no diffs from the previous + // iteration. + if os.done() { break } - if i == 0 { - output("*** Initial expr:\n") + if prev == "" { // Output starting tree. + bestHeader(os.exprView(), "Initial expression\n") indent(next) + prevBest = next + } else if next == prev { + altHeader("%s (no changes)\n", os.lastRuleName()) } else { - output("\n*** %s applied; ", lastRuleName.String()) + var diff difflib.UnifiedDiff + if os.isBetter() { + // New expression is better than the previous expression. Diff + // it against the previous *best* expression (might not be the + // previous expression). + bestHeader(os.exprView(), "%s\n", os.lastRuleName()) + + diff = difflib.UnifiedDiff{ + A: difflib.SplitLines(prevBest), + B: difflib.SplitLines(next), + Context: 100, + } - if prev == next { - // The expression can be unchanged if a part of the memo changed that - // does not affect the final best expression. - output("best expr unchanged.\n") + prevBest = next } else { - output("best expr changed:\n") - diff := difflib.UnifiedDiff{ + // New expression is not better than the previous expression, but + // still show the change. Diff it against the previous expression, + // regardless if it was a "best" expression or not. + altHeader("%s (higher cost)\n", os.lastRuleName()) + + next = os.exprView().FormatString(ot.Flags.ExprFormat) + diff = difflib.UnifiedDiff{ A: difflib.SplitLines(prev), B: difflib.SplitLines(next), Context: 100, } - - text, _ := difflib.GetUnifiedDiffString(diff) - // Skip the "@@ ... @@" header (first line). - text = strings.SplitN(text, "\n", 2)[1] - indent(text) } + + text, _ := difflib.GetUnifiedDiffString(diff) + // Skip the "@@ ... @@" header (first line). + text = strings.SplitN(text, "\n", 2)[1] + indent(text) } prev = next } // Output ending tree. - output("\n*** Final best expr:\n") + bestHeader(os.exprView(), "Final best expression\n") indent(next) - if verbose { + if ot.Flags.Verbose { fmt.Print("------ optsteps verbose output ends ------\n") } @@ -325,8 +377,8 @@ func (e *OptTester) OptSteps(verbose bool) (string, error) { // ExecBuild builds the exec node tree for the SQL query. This can be executed // by the exec engine. -func (e *OptTester) ExecBuild(eng exec.TestEngine) (exec.Node, error) { - ev, err := e.Optimize() +func (ot *OptTester) ExecBuild(eng exec.TestEngine) (exec.Node, error) { + ev, err := ot.Optimize() if err != nil { return nil, err } @@ -335,8 +387,8 @@ func (e *OptTester) ExecBuild(eng exec.TestEngine) (exec.Node, error) { // Explain builds the exec node tree for the SQL query and then runs the // explain command that describes the physical execution plan. -func (e *OptTester) Explain(eng exec.TestEngine) ([]tree.Datums, error) { - node, err := e.ExecBuild(eng) +func (ot *OptTester) Explain(eng exec.TestEngine) ([]tree.Datums, error) { + node, err := ot.ExecBuild(eng) if err != nil { return nil, err } @@ -344,8 +396,8 @@ func (e *OptTester) Explain(eng exec.TestEngine) ([]tree.Datums, error) { } // Exec builds the exec node tree for the SQL query and then executes it. -func (e *OptTester) Exec(eng exec.TestEngine) (sqlbase.ResultColumns, []tree.Datums, error) { - node, err := e.ExecBuild(eng) +func (ot *OptTester) Exec(eng exec.TestEngine) (sqlbase.ResultColumns, []tree.Datums, error) { + node, err := ot.ExecBuild(eng) if err != nil { return nil, nil, err } @@ -360,25 +412,25 @@ func (e *OptTester) Exec(eng exec.TestEngine) (sqlbase.ResultColumns, []tree.Dat return columns, datums, err } -func (e *OptTester) buildExpr( +func (ot *OptTester) buildExpr( factory *norm.Factory, ) (root memo.GroupID, required *memo.PhysicalProps, _ error) { - stmt, err := parser.ParseOne(e.sql) + stmt, err := parser.ParseOne(ot.sql) if err != nil { return 0, nil, err } - b := optbuilder.New(e.ctx, &e.semaCtx, &e.evalCtx, e.catalog, factory, stmt) - b.AllowUnsupportedExpr = e.Flags.AllowUnsupportedExpr + b := optbuilder.New(ot.ctx, &ot.semaCtx, &ot.evalCtx, ot.catalog, factory, stmt) + b.AllowUnsupportedExpr = ot.Flags.AllowUnsupportedExpr return b.Build() } -func (e *OptTester) optimizeExpr(allowOptimizations bool) (memo.ExprView, error) { - o := xform.NewOptimizer(&e.evalCtx) +func (ot *OptTester) optimizeExpr(allowOptimizations bool) (memo.ExprView, error) { + o := xform.NewOptimizer(&ot.evalCtx) if !allowOptimizations { o.DisableOptimizations() } - root, required, err := e.buildExpr(o.Factory()) + root, required, err := ot.buildExpr(o.Factory()) if err != nil { return memo.ExprView{}, err } diff --git a/pkg/sql/opt/xform/coster.go b/pkg/sql/opt/xform/coster.go index a1ca14050d7f..a6f37701743b 100644 --- a/pkg/sql/opt/xform/coster.go +++ b/pkg/sql/opt/xform/coster.go @@ -19,17 +19,35 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" ) -// coster encapsulates the cost model for the optimizer. The coster assigns an -// estimated cost to each expression in the memo so that the optimizer can -// choose the lowest cost expression tree. The estimated cost is a best-effort -// approximation of the actual cost of execution, based on table and index -// statistics that are propagated throughout the logical expression tree. +// Coster is used by the optimizer to assign a cost to a candidate expression +// that can provide a set of required physical properties. If a candidate +// expression has a lower cost than any other expression in the memo group, then +// it becomes the new best expression for the group. +// +// Coster is an interface so that different costing algorithms can be used by +// the optimizer. For example, the OptSteps command uses a custom coster that +// assigns infinite costs to some expressions in order to prevent them from +// being part of the lowest cost tree (for debugging purposes). +type Coster interface { + // ComputeCost returns the estimated cost of executing the candidate + // expression. The optimizer does not expect the cost to correspond to any + // real-world metric, but does expect costs to be comparable to one another, + // as well as summable. + ComputeCost(candidate *memo.BestExpr, props *memo.LogicalProps) memo.Cost +} + +// coster encapsulates the default cost model for the optimizer. The coster +// assigns an estimated cost to each expression in the memo so that the +// optimizer can choose the lowest cost expression tree. The estimated cost is +// a best-effort approximation of the actual cost of execution, based on table +// and index statistics that are propagated throughout the logical expression +// tree. type coster struct { mem *memo.Memo } -func (c *coster) init(mem *memo.Memo) { - c.mem = mem +func newCoster(mem *memo.Memo) *coster { + return &coster{mem: mem} } // computeCost calculates the estimated cost of the candidate best expression, @@ -38,27 +56,24 @@ func (c *coster) init(mem *memo.Memo) { // branch-and-bound pruning will work properly. // // TODO: This is just a skeleton, and needs to compute real costs. -func (c *coster) computeCost(candidate *memo.BestExpr, props *memo.LogicalProps) { - var cost memo.Cost +func (c *coster) ComputeCost(candidate *memo.BestExpr, props *memo.LogicalProps) memo.Cost { switch candidate.Operator() { case opt.SortOp: - cost = c.computeSortCost(candidate, props) + return c.computeSortCost(candidate, props) case opt.ScanOp: - cost = c.computeScanCost(candidate, props) + return c.computeScanCost(candidate, props) case opt.SelectOp: - cost = c.computeSelectCost(candidate, props) + return c.computeSelectCost(candidate, props) case opt.ValuesOp: - cost = c.computeValuesCost(candidate, props) + return c.computeValuesCost(candidate, props) default: // By default, cost of parent is sum of child costs. - cost = c.computeChildrenCost(candidate) + return c.computeChildrenCost(candidate) } - - candidate.SetCost(cost) } func (c *coster) computeSortCost(candidate *memo.BestExpr, props *memo.LogicalProps) memo.Cost { diff --git a/pkg/sql/opt/xform/explorer.og.go b/pkg/sql/opt/xform/explorer.og.go index 1fe29e1b8f5e..165af648426d 100644 --- a/pkg/sql/opt/xform/explorer.og.go +++ b/pkg/sql/opt/xform/explorer.og.go @@ -32,10 +32,10 @@ func (_e *explorer) exploreScan(_rootState *exploreState, _root memo.ExprID) (_f def := _rootExpr.Def() if _e.canGenerateIndexScans(def) { if _e.o.matchedRule == nil || _e.o.matchedRule(opt.GenerateIndexScans) { - exprs := _e.generateIndexScans(def) + _exprs := _e.generateIndexScans(def) _before := _e.mem.ExprCount(_root.Group) - for i := range exprs { - _e.mem.MemoizeDenormExpr(_root.Group, exprs[i]) + for i := range _exprs { + _e.mem.MemoizeDenormExpr(_root.Group, _exprs[i]) } if _e.o.appliedRule != nil { _after := _e.mem.ExprCount(_root.Group) @@ -72,10 +72,10 @@ func (_e *explorer) exploreSelect(_rootState *exploreState, _root memo.ExprID) ( if _e.canConstrainScan(def) { filter := _rootExpr.Filter() if _e.o.matchedRule == nil || _e.o.matchedRule(opt.ConstrainScan) { - exprs := _e.constrainScan(filter, def) + _exprs := _e.constrainScan(filter, def) _before := _e.mem.ExprCount(_root.Group) - for i := range exprs { - _e.mem.MemoizeDenormExpr(_root.Group, exprs[i]) + for i := range _exprs { + _e.mem.MemoizeDenormExpr(_root.Group, _exprs[i]) } if _e.o.appliedRule != nil { _after := _e.mem.ExprCount(_root.Group) diff --git a/pkg/sql/opt/xform/optimizer.go b/pkg/sql/opt/xform/optimizer.go index adb792834f15..c21ce29628d9 100644 --- a/pkg/sql/opt/xform/optimizer.go +++ b/pkg/sql/opt/xform/optimizer.go @@ -25,19 +25,14 @@ import ( ) // MatchedRuleFunc defines the callback function for the NotifyOnMatchedRule -// event supported by the Optimizer and Factory. It is invoked each time an -// optimization rule (Normalize or Explore) has been matched by the optimizer. -// The name of the matched rule is passed as a parameter. If the function -// returns false, then the rule is not applied (i.e. skipped). -type MatchedRuleFunc func(ruleName opt.RuleName) bool +// event supported by the optimizer. See the comment in factory.go for more +// details. +type MatchedRuleFunc = norm.MatchedRuleFunc -// AppliedRuleFunc defines the callback function for the AppliedRuleFunc event -// supported by the Optimizer and Factory. It is invoked each time an -// optimization rule (Normalize or Explore) has been applied by the optimizer. -// The function is called with the name of the rule and the memo group it -// affected. If the rule was an exploration rule, then the added parameter -// gives the number of expressions added to the group by the rule. -type AppliedRuleFunc func(ruleName opt.RuleName, group memo.GroupID, added int) +// AppliedRuleFunc defines the callback function for the NotifyOnAppliedRule +// event supported by the optimizer. See the comment in factory.go for more +// details. +type AppliedRuleFunc = norm.AppliedRuleFunc // Optimizer transforms an input expression tree into the logically equivalent // output expression tree with the lowest possible execution cost. @@ -53,7 +48,7 @@ type Optimizer struct { evalCtx *tree.EvalContext f *norm.Factory mem *memo.Memo - coster coster + coster Coster explorer explorer // stateMap allocates temporary storage that's used to speed up optimization. @@ -79,9 +74,9 @@ func NewOptimizer(evalCtx *tree.EvalContext) *Optimizer { evalCtx: evalCtx, f: f, mem: f.Memo(), + coster: newCoster(f.Memo()), stateMap: make(map[optStateKey]*optState), } - o.coster.init(o.mem) o.explorer.init(o) return o } @@ -93,6 +88,20 @@ func (o *Optimizer) Factory() *norm.Factory { return o.f } +// Coster returns the coster instance that the optimizer is currently using to +// estimate the cost of executing portions of the expression tree. When a new +// optimizer is constructed, it creates a default coster that will be used +// unless it is overridden with a call to SetCoster. +func (o *Optimizer) Coster() Coster { + return o.coster +} + +// SetCoster overrides the default coster. The optimizer will now use the given +// coster to estimate the cost of expression execution. +func (o *Optimizer) SetCoster(coster Coster) { + o.coster = coster +} + // DisableOptimizations disables all transformation rules, including normalize // and explore rules. The unaltered input expression tree becomes the output // expression tree (because no transforms are applied). @@ -102,15 +111,15 @@ func (o *Optimizer) DisableOptimizations() { // NotifyOnMatchedRule sets a callback function which is invoked each time an // optimization rule (Normalize or Explore) has been matched by the optimizer. -// If matchedRule is nil, then no notifications are sent. If no callback -// function is set, then all rules are applied by default. In addition, callers -// can invoke the DisableOptimizations convenience method to disable all rules. +// If matchedRule is nil, then no notifications are sent, and all rules are +// applied by default. In addition, callers can invoke the DisableOptimizations +// convenience method to disable all rules. func (o *Optimizer) NotifyOnMatchedRule(matchedRule MatchedRuleFunc) { o.matchedRule = matchedRule // Also pass through the call to the factory so that normalization rules // make same callback. - o.f.NotifyOnMatchedRule(norm.MatchedRuleFunc(matchedRule)) + o.f.NotifyOnMatchedRule(matchedRule) } // NotifyOnAppliedRule sets a callback function which is invoked each time an @@ -121,7 +130,7 @@ func (o *Optimizer) NotifyOnAppliedRule(appliedRule AppliedRuleFunc) { // Also pass through the call to the factory so that normalization rules // make same callback. - o.f.NotifyOnAppliedRule(norm.AppliedRuleFunc(appliedRule)) + o.f.NotifyOnAppliedRule(appliedRule) } // Memo returns the memo structure that the optimizer is using to optimize. @@ -454,7 +463,8 @@ func (o *Optimizer) enforceProps( // group. If so, then the candidate becomes the new lowest cost expression. func (o *Optimizer) ratchetCost(candidate *memo.BestExpr) { group := candidate.Group() - o.coster.computeCost(candidate, o.mem.GroupProperties(group)) + cost := o.coster.ComputeCost(candidate, o.mem.GroupProperties(group)) + candidate.SetCost(cost) state := o.lookupOptState(group, candidate.Required()) if state.best == memo.UnknownBestExprID { // Lazily allocate the best expression only when it's needed.