Skip to content

Commit

Permalink
opt: match ScalarGroupBy in EliminateIndexJoinOrProjectInsideGroupBy
Browse files Browse the repository at this point in the history 
This commit extends EliminateIndexJoinOrProjectInsideGroupBy so that
ScalarGroupBy expressions are also matched. This allows the rule to
eliminate unnecessary index joins in more cases.

The primary motivation for this change was to make partial index
validation queries more efficient. These queries always have
ScalarGroupBy expressions because they are in the form:

    SELECT count(1) FROM table@partial_index WHERE predicate

Prior to this change, an index join was planned for these queries which
would operate on every row in the partial index. This could be extremely
expensive for large partial indexes.

Fixes #70116

Release note (performance improvement): A limitation has been fixed that
made creating partial indexes inefficient.
  • Loading branch information
@mgartner
mgartner committed Sep 13, 2021
1 parent d69481b commit 2751851
Show file tree
Hide file tree
Showing 6 changed files with 128 additions and 121 deletions.
25 changes: 25 additions & 0 deletions pkg/sql/opt/exec/execbuilder/testdata/partial_index
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,3 +146,28 @@ PREPARE q66465_b AS SELECT k FROM t66465@idx WHERE k + i > 0 AND t > now()

statement ok
EXECUTE q66465_b

# Regression test for #70116. Partial index validation queries performed during
# a backfill should be efficient.

statement ok
CREATE TABLE t70116 (
k INT PRIMARY KEY,
a INT,
b INT,
INDEX a_idx (a) WHERE b > 0
)

# This query mimics a partial index validation query.
query T
EXPLAIN SELECT count(1) FROM t70116@a_idx WHERE b > 0
----
distribution: local
vectorized: true
·
• group (scalar)
│
└── • scan
missing stats
table: t70116@a_idx (partial index)
spans: FULL SCAN
2 changes: 1 addition & 1 deletion pkg/sql/opt/xform/rules/groupby.opt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@
# projections instead of using an IndexJoin. The Project can be eliminated for
# the same reasons as the IndexJoin.
[EliminateIndexJoinOrProjectInsideGroupBy, Explore]
(GroupBy | DistinctOn | EnsureUpsertDistinctOn
(GroupBy | ScalarGroupBy | DistinctOn | EnsureUpsertDistinctOn
(IndexJoin | Project $input:*)
$aggs:*
$private:* &
Expand Down
64 changes: 24 additions & 40 deletions pkg/sql/opt/xform/testdata/external/tpce
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2747,26 +2747,18 @@ project
│ ├── cardinality: [1 - 1]
│ ├── key: ()
│ ├── fd: ()-->(10)
│ ├── project
│ │ ├── columns: tx_id:1!null tx_rate:3!null
│ │ ├── key: (1)
│ │ ├── fd: (1)-->(3)
│ │ └── project
│ │ ├── columns: tx_id:1!null tx_rate:3!null cx_tx_id:6!null
│ │ ├── key: (6)
│ │ ├── fd: (1)-->(3), (1)==(6), (6)==(1)
│ │ └── inner-join (lookup taxrate)
│ │ ├── columns: tx_id:1!null tx_rate:3!null cx_tx_id:6!null cx_c_id:7!null
│ │ ├── key columns: [6] = [1]
│ │ ├── lookup columns are key
│ │ ├── key: (6)
│ │ ├── fd: ()-->(7), (1)-->(3), (1)==(6), (6)==(1)
│ │ ├── scan customer_taxrate@secondary
│ │ │ ├── columns: cx_tx_id:6!null cx_c_id:7!null
│ │ │ ├── constraint: /7/6: [/0 - /0]
│ │ │ ├── key: (6)
│ │ │ └── fd: ()-->(7)
│ │ └── filters (true)
│ ├── inner-join (lookup taxrate)
│ │ ├── columns: tx_id:1!null tx_rate:3!null cx_tx_id:6!null cx_c_id:7!null
│ │ ├── key columns: [6] = [1]
│ │ ├── lookup columns are key
│ │ ├── key: (6)
│ │ ├── fd: ()-->(7), (1)-->(3), (1)==(6), (6)==(1)
│ │ ├── scan customer_taxrate@secondary
│ │ │ ├── columns: cx_tx_id:6!null cx_c_id:7!null
│ │ │ ├── constraint: /7/6: [/0 - /0]
│ │ │ ├── key: (6)
│ │ │ └── fd: ()-->(7)
│ │ └── filters (true)
│ └── aggregations
│ └── sum [as=sum:10, outer=(3)]
│ └── tx_rate:3
Expand Down Expand Up @@ -4003,26 +3995,18 @@ project
│ │ │ │ ├── cardinality: [1 - 1]
│ │ │ │ ├── key: ()
│ │ │ │ ├── fd: ()-->(44)
│ │ │ │ ├── project
│ │ │ │ │ ├── columns: tx_id:35!null tx_rate:37!null
│ │ │ │ │ ├── key: (35)
│ │ │ │ │ ├── fd: (35)-->(37)
│ │ │ │ │ └── project
│ │ │ │ │ ├── columns: tx_id:35!null tx_rate:37!null cx_tx_id:40!null
│ │ │ │ │ ├── key: (40)
│ │ │ │ │ ├── fd: (35)-->(37), (35)==(40), (40)==(35)
│ │ │ │ │ └── inner-join (lookup taxrate)
│ │ │ │ │ ├── columns: tx_id:35!null tx_rate:37!null cx_tx_id:40!null cx_c_id:41!null
│ │ │ │ │ ├── key columns: [40] = [35]
│ │ │ │ │ ├── lookup columns are key
│ │ │ │ │ ├── key: (40)
│ │ │ │ │ ├── fd: ()-->(41), (35)-->(37), (35)==(40), (40)==(35)
│ │ │ │ │ ├── scan customer_taxrate@secondary
│ │ │ │ │ │ ├── columns: cx_tx_id:40!null cx_c_id:41!null
│ │ │ │ │ │ ├── constraint: /41/40: [/0 - /0]
│ │ │ │ │ │ ├── key: (40)
│ │ │ │ │ │ └── fd: ()-->(41)
│ │ │ │ │ └── filters (true)
│ │ │ │ ├── inner-join (lookup taxrate)
│ │ │ │ │ ├── columns: tx_id:35!null tx_rate:37!null cx_tx_id:40!null cx_c_id:41!null
│ │ │ │ │ ├── key columns: [40] = [35]
│ │ │ │ │ ├── lookup columns are key
│ │ │ │ │ ├── key: (40)
│ │ │ │ │ ├── fd: ()-->(41), (35)-->(37), (35)==(40), (40)==(35)
│ │ │ │ │ ├── scan customer_taxrate@secondary
│ │ │ │ │ │ ├── columns: cx_tx_id:40!null cx_c_id:41!null
│ │ │ │ │ │ ├── constraint: /41/40: [/0 - /0]
│ │ │ │ │ │ ├── key: (40)
│ │ │ │ │ │ └── fd: ()-->(41)
│ │ │ │ │ └── filters (true)
│ │ │ │ └── aggregations
│ │ │ │ └── sum [as=sum:44, outer=(37)]
│ │ │ │ └── tx_rate:37
Expand Down
64 changes: 24 additions & 40 deletions pkg/sql/opt/xform/testdata/external/tpce-no-stats
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2781,26 +2781,18 @@ project
│ ├── cardinality: [1 - 1]
│ ├── key: ()
│ ├── fd: ()-->(10)
│ ├── project
│ │ ├── columns: tx_id:1!null tx_rate:3!null
│ │ ├── key: (1)
│ │ ├── fd: (1)-->(3)
│ │ └── project
│ │ ├── columns: tx_id:1!null tx_rate:3!null cx_tx_id:6!null
│ │ ├── key: (6)
│ │ ├── fd: (1)-->(3), (1)==(6), (6)==(1)
│ │ └── inner-join (lookup taxrate)
│ │ ├── columns: tx_id:1!null tx_rate:3!null cx_tx_id:6!null cx_c_id:7!null
│ │ ├── key columns: [6] = [1]
│ │ ├── lookup columns are key
│ │ ├── key: (6)
│ │ ├── fd: ()-->(7), (1)-->(3), (1)==(6), (6)==(1)
│ │ ├── scan customer_taxrate@secondary
│ │ │ ├── columns: cx_tx_id:6!null cx_c_id:7!null
│ │ │ ├── constraint: /7/6: [/0 - /0]
│ │ │ ├── key: (6)
│ │ │ └── fd: ()-->(7)
│ │ └── filters (true)
│ ├── inner-join (lookup taxrate)
│ │ ├── columns: tx_id:1!null tx_rate:3!null cx_tx_id:6!null cx_c_id:7!null
│ │ ├── key columns: [6] = [1]
│ │ ├── lookup columns are key
│ │ ├── key: (6)
│ │ ├── fd: ()-->(7), (1)-->(3), (1)==(6), (6)==(1)
│ │ ├── scan customer_taxrate@secondary
│ │ │ ├── columns: cx_tx_id:6!null cx_c_id:7!null
│ │ │ ├── constraint: /7/6: [/0 - /0]
│ │ │ ├── key: (6)
│ │ │ └── fd: ()-->(7)
│ │ └── filters (true)
│ └── aggregations
│ └── sum [as=sum:10, outer=(3)]
│ └── tx_rate:3
Expand Down Expand Up @@ -4036,26 +4028,18 @@ project
│ │ │ │ ├── cardinality: [1 - 1]
│ │ │ │ ├── key: ()
│ │ │ │ ├── fd: ()-->(44)
│ │ │ │ ├── project
│ │ │ │ │ ├── columns: tx_id:35!null tx_rate:37!null
│ │ │ │ │ ├── key: (35)
│ │ │ │ │ ├── fd: (35)-->(37)
│ │ │ │ │ └── project
│ │ │ │ │ ├── columns: tx_id:35!null tx_rate:37!null cx_tx_id:40!null
│ │ │ │ │ ├── key: (40)
│ │ │ │ │ ├── fd: (35)-->(37), (35)==(40), (40)==(35)
│ │ │ │ │ └── inner-join (lookup taxrate)
│ │ │ │ │ ├── columns: tx_id:35!null tx_rate:37!null cx_tx_id:40!null cx_c_id:41!null
│ │ │ │ │ ├── key columns: [40] = [35]
│ │ │ │ │ ├── lookup columns are key
│ │ │ │ │ ├── key: (40)
│ │ │ │ │ ├── fd: ()-->(41), (35)-->(37), (35)==(40), (40)==(35)
│ │ │ │ │ ├── scan customer_taxrate@secondary
│ │ │ │ │ │ ├── columns: cx_tx_id:40!null cx_c_id:41!null
│ │ │ │ │ │ ├── constraint: /41/40: [/0 - /0]
│ │ │ │ │ │ ├── key: (40)
│ │ │ │ │ │ └── fd: ()-->(41)
│ │ │ │ │ └── filters (true)
│ │ │ │ ├── inner-join (lookup taxrate)
│ │ │ │ │ ├── columns: tx_id:35!null tx_rate:37!null cx_tx_id:40!null cx_c_id:41!null
│ │ │ │ │ ├── key columns: [40] = [35]
│ │ │ │ │ ├── lookup columns are key
│ │ │ │ │ ├── key: (40)
│ │ │ │ │ ├── fd: ()-->(41), (35)-->(37), (35)==(40), (40)==(35)
│ │ │ │ │ ├── scan customer_taxrate@secondary
│ │ │ │ │ │ ├── columns: cx_tx_id:40!null cx_c_id:41!null
│ │ │ │ │ │ ├── constraint: /41/40: [/0 - /0]
│ │ │ │ │ │ ├── key: (40)
│ │ │ │ │ │ └── fd: ()-->(41)
│ │ │ │ │ └── filters (true)
│ │ │ │ └── aggregations
│ │ │ │ └── sum [as=sum:44, outer=(37)]
│ │ │ │ └── tx_rate:37
Expand Down
16 changes: 16 additions & 0 deletions pkg/sql/opt/xform/testdata/rules/groupby
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2552,6 +2552,22 @@ group-by
└── aggregations
└── count-rows [as=count_rows:8]

# Rule applies for scalar group-by.
opt expect=EliminateIndexJoinOrProjectInsideGroupBy
SELECT count(1) FROM abcd@partial_ab WHERE c > 0
----
scalar-group-by
├── columns: count:9!null
├── cardinality: [1 - 1]
├── key: ()
├── fd: ()-->(9)
├── scan abcd@partial_ab,partial
│ ├── columns: rowid:5!null
│ ├── flags: force-index=partial_ab
│ └── key: (5)
└── aggregations
└── count-rows [as=count:9]

# Rule applies for distinct-on.
opt expect=EliminateIndexJoinOrProjectInsideGroupBy
SELECT DISTINCT a, b FROM abcd WHERE c > 0
Expand Down
78 changes: 38 additions & 40 deletions pkg/sql/opt/xform/testdata/rules/select
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8057,46 +8057,44 @@ scalar-group-by
├── cardinality: [1 - 1]
├── key: ()
├── fd: ()-->(7)
├── project
│ ├── columns: u:2 v:3
│ └── distinct-on
│ ├── columns: k:1!null u:2 v:3
│ ├── grouping columns: k:1!null
│ ├── internal-ordering: +1
│ ├── key: (1)
│ ├── fd: (1)-->(2,3)
│ ├── union-all
│ │ ├── columns: k:1!null u:2 v:3
│ │ ├── left columns: k:8 u:9 v:10
│ │ ├── right columns: k:14 u:15 v:16
│ │ ├── ordering: +1
│ │ ├── index-join d
│ │ │ ├── columns: k:8!null u:9!null v:10
│ │ │ ├── key: (8)
│ │ │ ├── fd: ()-->(9), (8)-->(10)
│ │ │ ├── ordering: +8 opt(9) [actual: +8]
│ │ │ └── scan d@u
│ │ │ ├── columns: k:8!null u:9!null
│ │ │ ├── constraint: /9/8: [/1 - /1]
│ │ │ ├── key: (8)
│ │ │ ├── fd: ()-->(9)
│ │ │ └── ordering: +8 opt(9) [actual: +8]
│ │ └── index-join d
│ │ ├── columns: k:14!null u:15 v:16!null
│ │ ├── key: (14)
│ │ ├── fd: ()-->(16), (14)-->(15)
│ │ ├── ordering: +14 opt(16) [actual: +14]
│ │ └── scan d@v
│ │ ├── columns: k:14!null v:16!null
│ │ ├── constraint: /16/14: [/1 - /1]
│ │ ├── key: (14)
│ │ ├── fd: ()-->(16)
│ │ └── ordering: +14 opt(16) [actual: +14]
│ └── aggregations
│ ├── const-agg [as=u:2, outer=(2)]
│ │ └── u:2
│ └── const-agg [as=v:3, outer=(3)]
│ └── v:3
├── distinct-on
│ ├── columns: k:1!null u:2 v:3
│ ├── grouping columns: k:1!null
│ ├── internal-ordering: +1
│ ├── key: (1)
│ ├── fd: (1)-->(2,3)
│ ├── union-all
│ │ ├── columns: k:1!null u:2 v:3
│ │ ├── left columns: k:8 u:9 v:10
│ │ ├── right columns: k:14 u:15 v:16
│ │ ├── ordering: +1
│ │ ├── index-join d
│ │ │ ├── columns: k:8!null u:9!null v:10
│ │ │ ├── key: (8)
│ │ │ ├── fd: ()-->(9), (8)-->(10)
│ │ │ ├── ordering: +8 opt(9) [actual: +8]
│ │ │ └── scan d@u
│ │ │ ├── columns: k:8!null u:9!null
│ │ │ ├── constraint: /9/8: [/1 - /1]
│ │ │ ├── key: (8)
│ │ │ ├── fd: ()-->(9)
│ │ │ └── ordering: +8 opt(9) [actual: +8]
│ │ └── index-join d
│ │ ├── columns: k:14!null u:15 v:16!null
│ │ ├── key: (14)
│ │ ├── fd: ()-->(16), (14)-->(15)
│ │ ├── ordering: +14 opt(16) [actual: +14]
│ │ └── scan d@v
│ │ ├── columns: k:14!null v:16!null
│ │ ├── constraint: /16/14: [/1 - /1]
│ │ ├── key: (14)
│ │ ├── fd: ()-->(16)
│ │ └── ordering: +14 opt(16) [actual: +14]
│ └── aggregations
│ ├── const-agg [as=u:2, outer=(2)]
│ │ └── u:2
│ └── const-agg [as=v:3, outer=(3)]
│ └── v:3
└── aggregations
└── count-rows [as=count_rows:7]

Expand Down

0 comments on commit 2751851

Please sign in to comment.