Skip to content

Commit

Permalink
opt: inline single-use WithExprs
Browse files Browse the repository at this point in the history 
This commit adds a rule to inline With expressions which are referenced
at most once. This avoid buffering in the common case and also allows
optimizations to cross this boundary.

Release note (sql change): CTEs which are used a single time will no
longer create an optimization fence.
  • Loading branch information
Justin Jaffray committed Sep 11, 2019
1 parent 78599a1 commit a37fe87
Show file tree
Hide file tree
Showing 15 changed files with 733 additions and 137 deletions.
4 changes: 3 additions & 1 deletion pkg/sql/logictest/testdata/logic_test/apply_join
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,9 @@ WHERE
SELECT
COALESCE(
tab_9962._string,
tab_9963._string
tab_9963._string,
(SELECT * FROM with_2063),
(SELECT * FROM with_2063)
)
)
)
2 changes: 1 addition & 1 deletion pkg/sql/logictest/testdata/logic_test/with
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ CREATE TABLE ab (a INT PRIMARY KEY, b INT)
statement ok
INSERT INTO ab VALUES (1, 2), (3, 4), (5, 6)

query I
query I rowsort
WITH a AS (SELECT a FROM ab ORDER BY b) SELECT * FROM a
----
1
Expand Down
49 changes: 13 additions & 36 deletions pkg/sql/opt/exec/execbuilder/testdata/with
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,11 @@ query TTTTT
EXPLAIN (VERBOSE)
WITH t AS (SELECT a FROM y) SELECT * FROM t
----
root · · (a) ·
├── scan buffer node · · (a) ·
│ label buffer 1 (t) · ·
└── subquery · · (a) ·
│ id @S1 · ·
│ original sql SELECT a FROM y · ·
│ exec mode all rows · ·
└── buffer node · · (a) ·
│ label buffer 1 (t) · ·
└── scan · · (a) ·
· table y@primary · ·
· spans ALL · ·
render · · (a) ·
│ render 0 a · ·
└── scan · · (a) ·
· table y@primary · ·
· spans ALL · ·

query TTTTT
EXPLAIN (VERBOSE)
Expand Down Expand Up @@ -83,27 +76,11 @@ EXPLAIN (VERBOSE)
FROM
w, table39010
----
root · · (col) ·
├── render · · (col) ·
│ │ render 0 col · ·
│ └── hash-join · · ("?column?", "?column?", col) ·
│ │ type cross · ·
│ ├── render · · ("?column?", "?column?") ·
│ │ │ render 0 "?column?" · ·
│ │ │ render 1 "?column?" · ·
│ │ └── scan buffer node · · ("?column?") ·
│ │ label buffer 1 (w) · ·
│ └── scan · · (col) ·
│ table table39010@primary · ·
│ spans ALL · ·
└── subquery · · (col) ·
│ id @S1 · ·
│ original sql SELECT NULL, NULL FROM table39010 · ·
│ exec mode all rows · ·
└── buffer node · · ("?column?") ·
│ label buffer 1 (w) · ·
└── render · · ("?column?") ·
│ render 0 NULL · ·
└── scan · · () ·
· table table39010@primary · ·
· spans ALL · ·
hash-join · · (col) ·
│ type cross · ·
├── scan · · () ·
│ table table39010@primary · ·
│ spans ALL · ·
└── scan · · (col) ·
· table table39010@primary · ·
· spans ALL · ·
4 changes: 4 additions & 0 deletions pkg/sql/opt/memo/expr_format.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,10 @@ func (f *ExprFmtCtx) formatRelational(e RelExpr, tp treeprinter.Node) {
if r.JoinSize > 1 {
tp.Childf("join-size: %d", r.JoinSize)
}
if len(relational.Shared.Rule.WithUses) > 0 {
// Go map
tp.Childf("cte-uses: %v", relational.Shared.Rule.WithUses)
}
}

switch t := e.(type) {
Expand Down
18 changes: 14 additions & 4 deletions pkg/sql/opt/memo/testdata/logprops/with
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ with &1 (foo)
├── columns: x:3(int!null) y:4(int)
├── key: (3)
├── fd: (3)-->(4)
├── cte-uses: map[1:1]
├── scan xy
│ ├── columns: xy.x:1(int!null) xy.y:2(int)
│ ├── key: (1)
Expand All @@ -21,7 +22,8 @@ with &1 (foo)
│ ├── xy.x:1(int) => x:3(int)
│ └── xy.y:2(int) => y:4(int)
├── key: (3)
└── fd: (3)-->(4)
├── fd: (3)-->(4)
└── cte-uses: map[1:1]

# Side effects should be propagated up to the top-level from the Binding side
# of a WITH.
Expand Down Expand Up @@ -68,6 +70,7 @@ with &1 (foo)
├── side-effects
├── key: ()
├── fd: ()-->(3)
├── cte-uses: map[1:1]
├── project
│ ├── columns: "?column?":1(int!null)
│ ├── cardinality: [1 - 1]
Expand All @@ -87,13 +90,15 @@ with &1 (foo)
├── key: ()
├── fd: ()-->(3)
├── prune: (3)
├── cte-uses: map[1:1]
├── with-scan &1 (foo)
│ ├── columns: "?column?":2(int!null)
│ ├── mapping:
│ │ └── "?column?":1(int) => "?column?":2(int)
│ ├── cardinality: [1 - 1]
│ ├── key: ()
│ └── fd: ()-->(2)
│ ├── fd: ()-->(2)
│ └── cte-uses: map[1:1]
└── projections
└── div [type=decimal, side-effects]
├── const: 1 [type=int]
Expand All @@ -108,6 +113,7 @@ with &1 (foo)
├── has-placeholder
├── key: ()
├── fd: ()-->(3)
├── cte-uses: map[1:1]
├── project
│ ├── columns: int8:1(int)
│ ├── cardinality: [1 - 1]
Expand All @@ -128,13 +134,15 @@ with &1 (foo)
├── key: ()
├── fd: ()-->(3)
├── prune: (3)
├── cte-uses: map[1:1]
├── with-scan &1 (foo)
│ ├── columns: int8:2(int)
│ ├── mapping:
│ │ └── int8:1(int) => int8:2(int)
│ ├── cardinality: [1 - 1]
│ ├── key: ()
│ └── fd: ()-->(2)
│ ├── fd: ()-->(2)
│ └── cte-uses: map[1:1]
└── projections
└── const: 1 [type=int]

Expand Down Expand Up @@ -163,6 +171,7 @@ inner-join-apply
│ ├── cardinality: [1 - 1]
│ ├── key: ()
│ ├── fd: ()-->(3)
│ ├── cte-uses: map[1:1]
│ ├── project
│ │ ├── columns: "?column?":2(int)
│ │ ├── outer: (1)
Expand All @@ -184,5 +193,6 @@ inner-join-apply
│ │ └── "?column?":2(int) => "?column?":3(int)
│ ├── cardinality: [1 - 1]
│ ├── key: ()
│ └── fd: ()-->(3)
│ ├── fd: ()-->(3)
│ └── cte-uses: map[1:1]
└── filters (true)
90 changes: 43 additions & 47 deletions pkg/sql/opt/memo/testdata/stats/groupby
View file
Original file line number Diff line number Diff line change
Expand Up @@ -465,54 +465,50 @@ WITH q (a, b) AS (SELECT * FROM (VALUES (true, NULL), (false, NULL), (true, 5)))
GROUP BY q.b
HAVING bool_or(q.a)
----
with &1 (q)
project
├── columns: "?column?":6(int!null)
├── cardinality: [0 - 3]
├── stats: [rows=0.27]
├── fd: ()-->(6)
├── values
│ ├── columns: column1:1(bool!null) column2:2(int)
│ ├── cardinality: [3 - 3]
│ ├── stats: [rows=3, distinct(1)=0.3, null(1)=0, distinct(2)=0.3, null(2)=0.03]
│ ├── (true, NULL) [type=tuple{bool, int}]
│ ├── (false, NULL) [type=tuple{bool, int}]
│ └── (true, 5) [type=tuple{bool, int}]
└── project
├── columns: "?column?":6(int!null)
├── cardinality: [0 - 3]
├── stats: [rows=0.27]
├── fd: ()-->(6)
├── select
│ ├── columns: b:4(int) bool_or:5(bool!null)
│ ├── cardinality: [0 - 3]
│ ├── stats: [rows=0.27, distinct(5)=0.27, null(5)=0]
│ ├── key: (4)
│ ├── fd: ()-->(5)
│ ├── group-by
│ │ ├── columns: b:4(int) bool_or:5(bool)
│ │ ├── grouping columns: b:4(int)
│ │ ├── cardinality: [0 - 3]
│ │ ├── stats: [rows=0.3, distinct(4)=0.3, null(4)=0.03, distinct(5)=0.3, null(5)=0.03]
│ │ ├── key: (4)
│ │ ├── fd: (4)-->(5)
│ │ ├── select
│ │ │ ├── columns: a:3(bool!null) b:4(int)
│ │ │ ├── cardinality: [0 - 3]
│ │ │ ├── stats: [rows=3, distinct(3)=0.3, null(3)=0, distinct(4)=0.3, null(4)=0.03]
│ │ │ ├── fd: ()-->(3)
│ │ │ ├── with-scan &1 (q)
│ │ │ │ ├── columns: a:3(bool!null) b:4(int)
│ │ │ │ ├── mapping:
│ │ │ │ │ ├── column1:1(bool) => a:3(bool)
│ │ │ │ │ └── column2:2(int) => b:4(int)
│ │ │ │ ├── cardinality: [3 - 3]
│ │ │ │ └── stats: [rows=3, distinct(3)=0.3, null(3)=0, distinct(4)=0.3, null(4)=0.03]
│ │ │ └── filters
│ │ │ └── variable: a [type=bool, outer=(3), constraints=(/3: [/true - /true]; tight), fd=()-->(3)]
│ │ └── aggregations
│ │ └── bool-or [type=bool, outer=(3)]
│ │ └── variable: a [type=bool]
│ └── filters
│ └── variable: bool_or [type=bool, outer=(5), constraints=(/5: [/true - /true]; tight), fd=()-->(5)]
└── projections
└── const: 1 [type=int]
├── select
│ ├── columns: b:4(int) bool_or:5(bool!null)
│ ├── cardinality: [0 - 3]
│ ├── stats: [rows=0.27, distinct(5)=0.27, null(5)=0]
│ ├── key: (4)
│ ├── fd: ()-->(5)
│ ├── group-by
│ │ ├── columns: b:4(int) bool_or:5(bool)
│ │ ├── grouping columns: b:4(int)
│ │ ├── cardinality: [0 - 3]
│ │ ├── stats: [rows=0.3, distinct(4)=0.3, null(4)=0.03, distinct(5)=0.3, null(5)=0.03]
│ │ ├── key: (4)
│ │ ├── fd: (4)-->(5)
│ │ ├── project
│ │ │ ├── columns: a:3(bool) b:4(int)
│ │ │ ├── cardinality: [0 - 3]
│ │ │ ├── stats: [rows=3, distinct(4)=0.3, null(4)=0.03]
│ │ │ ├── fd: ()-->(3)
│ │ │ ├── select
│ │ │ │ ├── columns: column1:1(bool!null) column2:2(int)
│ │ │ │ ├── cardinality: [0 - 3]
│ │ │ │ ├── stats: [rows=3, distinct(1)=0.3, null(1)=0, distinct(2)=0.3, null(2)=0.03]
│ │ │ │ ├── fd: ()-->(1)
│ │ │ │ ├── values
│ │ │ │ │ ├── columns: column1:1(bool!null) column2:2(int)
│ │ │ │ │ ├── cardinality: [3 - 3]
│ │ │ │ │ ├── stats: [rows=3, distinct(1)=0.3, null(1)=0, distinct(2)=0.3, null(2)=0.03]
│ │ │ │ │ ├── (true, NULL) [type=tuple{bool, int}]
│ │ │ │ │ ├── (false, NULL) [type=tuple{bool, int}]
│ │ │ │ │ └── (true, 5) [type=tuple{bool, int}]
│ │ │ │ └── filters
│ │ │ │ └── variable: column1 [type=bool, outer=(1), constraints=(/1: [/true - /true]; tight), fd=()-->(1)]
│ │ │ └── projections
│ │ │ ├── variable: column1 [type=bool, outer=(1)]
│ │ │ └── variable: column2 [type=int, outer=(2)]
│ │ └── aggregations
│ │ └── bool-or [type=bool, outer=(3)]
│ │ └── variable: a [type=bool]
│ └── filters
│ └── variable: bool_or [type=bool, outer=(5), constraints=(/5: [/true - /true]; tight), fd=()-->(5)]
└── projections
└── const: 1 [type=int]
62 changes: 26 additions & 36 deletions pkg/sql/opt/memo/testdata/stats/scan
View file
Original file line number Diff line number Diff line change
Expand Up @@ -540,44 +540,34 @@ FROM
WHERE
subq.col1;
----
with &1 (subq)
project
├── columns: "?column?":26(int!null)
├── stats: [rows=165000]
├── stats: [rows=0.95099005]
├── fd: ()-->(26)
├── project
│ ├── columns: col1:23(bool) tab1.g:18(int4!null)
│ ├── stats: [rows=333333.333, distinct(18)=33333.3333, null(18)=0, distinct(23)=2, null(23)=3333.33333]
│ ├── inner-join (hash)
│ │ ├── columns: tab0.e:5(varchar) tab0.f:6("char") tab0.h:8(varchar) tab0.j:10(float!null) tab1.e:16(varchar) tab1.f:17("char") tab1.g:18(int4!null) tab1.j:21(float!null)
│ │ ├── stats: [rows=333333.333, distinct(10)=100, null(10)=0, distinct(18)=100, null(18)=0, distinct(21)=100, null(21)=0]
│ │ ├── scan tab0
│ │ │ ├── columns: tab0.e:5(varchar) tab0.f:6("char") tab0.h:8(varchar) tab0.j:10(float!null)
│ │ │ └── stats: [rows=1000, distinct(10)=100, null(10)=0]
│ │ ├── scan tab1
│ │ │ ├── columns: tab1.e:16(varchar) tab1.f:17("char") tab1.g:18(int4!null) tab1.j:21(float!null)
│ │ │ └── stats: [rows=1000, distinct(18)=100, null(18)=0, distinct(21)=100, null(21)=0]
│ │ └── filters
│ │ └── tab0.j IN (tab1.j,) [type=bool, outer=(10,21)]
│ └── projections
│ └── CASE WHEN ilike_escape(regexp_replace(tab0.h, tab1.e, tab0.f, tab0.e::STRING), tab1.f, '') THEN true ELSE false END [type=bool, outer=(5,6,8,16,17)]
└── project
├── columns: "?column?":26(int!null)
├── stats: [rows=165000]
├── fd: ()-->(26)
├── select
│ ├── columns: col0:24(int4!null) col1:25(bool!null)
│ ├── stats: [rows=165000, distinct(24)=33300.7812, null(24)=0, distinct(25)=1, null(25)=0]
│ ├── fd: ()-->(25)
│ ├── with-scan &1 (subq)
│ │ ├── columns: col0:24(int4!null) col1:25(bool)
│ │ ├── mapping:
│ │ │ ├── tab1.g:18(int4) => col0:24(int4)
│ │ │ └── col1:23(bool) => col1:25(bool)
│ │ └── stats: [rows=333333.333, distinct(24)=33333.3333, null(24)=0, distinct(25)=2, null(25)=3333.33333]
│ └── filters
│ └── variable: col1 [type=bool, outer=(25), constraints=(/25: [/true - /true]; tight), fd=()-->(25)]
└── projections
└── const: 1 [type=int]
├── select
│ ├── columns: col1:25(bool!null)
│ ├── stats: [rows=0.95099005, distinct(25)=0.95099005, null(25)=0]
│ ├── fd: ()-->(25)
│ ├── project
│ │ ├── columns: col1:25(bool)
│ │ ├── stats: [rows=333333.333, distinct(25)=333333.333, null(25)=16336.65]
│ │ ├── inner-join (hash)
│ │ │ ├── columns: tab0.e:5(varchar) tab0.f:6("char") tab0.h:8(varchar) tab0.j:10(float!null) tab1.e:16(varchar) tab1.f:17("char") tab1.j:21(float!null)
│ │ │ ├── stats: [rows=333333.333, distinct(10)=100, null(10)=0, distinct(21)=100, null(21)=0, distinct(5,6,8,16,17)=333333.333, null(5,6,8,16,17)=16336.65]
│ │ │ ├── scan tab0
│ │ │ │ ├── columns: tab0.e:5(varchar) tab0.f:6("char") tab0.h:8(varchar) tab0.j:10(float!null)
│ │ │ │ └── stats: [rows=1000, distinct(10)=100, null(10)=0, distinct(5,6,8)=1000, null(5,6,8)=29.701]
│ │ │ ├── scan tab1
│ │ │ │ ├── columns: tab1.e:16(varchar) tab1.f:17("char") tab1.j:21(float!null)
│ │ │ │ └── stats: [rows=1000, distinct(21)=100, null(21)=0, distinct(16,17)=1000, null(16,17)=19.9]
│ │ │ └── filters
│ │ │ └── tab0.j IN (tab1.j,) [type=bool, outer=(10,21)]
│ │ └── projections
│ │ └── CASE WHEN ilike_escape(regexp_replace(tab0.h, tab1.e, tab0.f, tab0.e::STRING), tab1.f, '') THEN true ELSE false END [type=bool, outer=(5,6,8,16,17)]
│ └── filters
│ └── variable: col1 [type=bool, outer=(25), constraints=(/25: [/true - /true]; tight), fd=()-->(25)]
└── projections
└── const: 1 [type=int]

# ---------------------
# Tests with Histograms
Expand Down
2 changes: 1 addition & 1 deletion pkg/sql/opt/memo/testdata/stats/select
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1367,7 +1367,7 @@ ALTER TABLE t38344 INJECT STATISTICS '[
]'
----

norm
norm disable=InlineWith
WITH t(x) AS (
SELECT (t1.x::int << 5533)::bool OR t2.x AS x
FROM t38344 AS t1 LEFT JOIN t38344 AS t2 ON true
Expand Down
4 changes: 2 additions & 2 deletions pkg/sql/opt/memo/testdata/stats/with
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ ALTER TABLE a INJECT STATISTICS '[
]'
----

build colstat=4 colstat=5 colstat=6
build colstat=4 colstat=5 colstat=6 disable=InlineWith
WITH foo AS (SELECT * FROM a) SELECT * FROM foo
----
with &1 (foo)
Expand All @@ -55,7 +55,7 @@ with &1 (foo)
└── fd: (4)-->(5,6)

# Regression test for #40296.
opt
opt disable=InlineWith
WITH
t0 AS ((VALUES (0, 0:::OID, NULL, '')) UNION (VALUES (NULL, 0:::OID,'1970-09-08'::DATE, NULL)))
SELECT
Expand Down
Loading

0 comments on commit a37fe87

Please sign in to comment.