diff --git a/pkg/sql/opt/memo/testdata/stats/partial-index-scan b/pkg/sql/opt/memo/testdata/stats/partial-index-scan index 88645c2758d1..ec1eced8ad76 100644 --- a/pkg/sql/opt/memo/testdata/stats/partial-index-scan +++ b/pkg/sql/opt/memo/testdata/stats/partial-index-scan @@ -430,17 +430,20 @@ CREATE TABLE t ( opt SELECT * FROM t WHERE pk2 = 1 AND b1 = false AND b2 = false ---- -index-join t +project ├── columns: pk1:1(int!null) pk2:2(int!null) b1:3(bool!null) b2:4(bool!null) ├── stats: [rows=1.245025, distinct(2)=1, null(2)=0, distinct(3)=1, null(3)=0, distinct(4)=1, null(4)=0, distinct(2-4)=1, null(2-4)=0] ├── key: (1) ├── fd: ()-->(2-4) - └── scan t@secondary,partial - ├── columns: pk1:1(int!null) pk2:2(int!null) - ├── constraint: /2/1: [/1 - /1] - ├── stats: [rows=1.245025, distinct(2)=1, null(2)=0, distinct(3)=1, null(3)=0, distinct(4)=1, null(4)=0, distinct(2-4)=1, null(2-4)=0] - ├── key: (1) - └── fd: ()-->(2) + ├── scan t@secondary,partial + │ ├── columns: pk1:1(int!null) pk2:2(int!null) + │ ├── constraint: /2/1: [/1 - /1] + │ ├── stats: [rows=1.245025, distinct(2)=1, null(2)=0, distinct(3)=1, null(3)=0, distinct(4)=1, null(4)=0, distinct(2-4)=1, null(2-4)=0] + │ ├── key: (1) + │ └── fd: ()-->(2) + └── projections + ├── false [as=b1:3, type=bool] + └── false [as=b2:4, type=bool] # --------------------- @@ -701,7 +704,7 @@ CREATE INDEX idx ON hist (i) WHERE i > 100 AND i <= 200 AND s = 'banana' opt SELECT * FROM hist WHERE i > 125 AND i < 150 AND s = 'banana' ---- -index-join hist +project ├── columns: k:1(int!null) i:2(int!null) s:3(string!null) ├── stats: [rows=6.91433927, distinct(2)=3.09090909, null(2)=0, distinct(3)=1, null(3)=0, distinct(2,3)=3.09090909, null(2,3)=0] │ histogram(2)= 0 0 6.6262 0.2881 @@ -710,16 +713,18 @@ index-join hist │ <--- 'banana' ├── key: (1) ├── fd: ()-->(3), (1)-->(2) - └── scan hist@idx,partial - ├── columns: k:1(int!null) i:2(int!null) - ├── constraint: /2/1: [/126 - /149] - ├── stats: [rows=6.91433927, distinct(2)=3.09090909, null(2)=0, distinct(3)=1, null(3)=0, distinct(2,3)=3.09090909, null(2,3)=0] - │ histogram(2)= 0 0 6.6262 0.2881 - │ <--- 125 -------- 149 - - │ histogram(3)= 0 6.9143 - │ <--- 'banana' - ├── key: (1) - └── fd: (1)-->(2) + ├── scan hist@idx,partial + │ ├── columns: k:1(int!null) i:2(int!null) + │ ├── constraint: /2/1: [/126 - /149] + │ ├── stats: [rows=6.91433927, distinct(2)=3.09090909, null(2)=0, distinct(3)=1, null(3)=0, distinct(2,3)=3.09090909, null(2,3)=0] + │ │ histogram(2)= 0 0 6.6262 0.2881 + │ │ <--- 125 -------- 149 - + │ │ histogram(3)= 0 6.9143 + │ │ <--- 'banana' + │ ├── key: (1) + │ └── fd: (1)-->(2) + └── projections + └── 'banana' [as=s:3, type=string] exec-ddl DROP INDEX idx diff --git a/pkg/sql/opt/xform/index_scan_builder.go b/pkg/sql/opt/xform/index_scan_builder.go index 6e818d653542..d23a1633bff8 100644 --- a/pkg/sql/opt/xform/index_scan_builder.go +++ b/pkg/sql/opt/xform/index_scan_builder.go @@ -13,7 +13,6 @@ package xform import ( "github.com/cockroachdb/cockroach/pkg/sql/inverted" "github.com/cockroachdb/cockroach/pkg/sql/opt" - "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/opt/invertedexpr" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" "github.com/cockroachdb/cockroach/pkg/sql/opt/norm" @@ -32,26 +31,27 @@ import ( // make the following calls: // // var sb indexScanBuilder -// sb.init(c, tabID) -// sb.setScan(scanPrivate) -// sb.addSelect(filters) -// sb.addIndexJoin(cols) -// expr := sb.build() +// sb.Init(c, tabID) +// sb.SetScan(scanPrivate) +// sb.AddSelect(filters) +// sb.AddIndexJoin(cols) +// expr := sb.Build() // type indexScanBuilder struct { c *CustomFuncs f *norm.Factory mem *memo.Memo tabID opt.TableID - pkCols opt.ColSet scanPrivate memo.ScanPrivate + constProjections memo.ProjectionsExpr innerFilters memo.FiltersExpr outerFilters memo.FiltersExpr invertedFilterPrivate memo.InvertedFilterPrivate indexJoinPrivate memo.IndexJoinPrivate } -func (b *indexScanBuilder) init(c *CustomFuncs, tabID opt.TableID) { +// Init initializes an indexScanBuilder. +func (b *indexScanBuilder) Init(c *CustomFuncs, tabID opt.TableID) { // This initialization pattern ensures that fields are not unwittingly // reused. Field reuse must be explicit. *b = indexScanBuilder{ @@ -62,42 +62,46 @@ func (b *indexScanBuilder) init(c *CustomFuncs, tabID opt.TableID) { } } -// primaryKeyCols returns the columns from the scanned table's primary index. -func (b *indexScanBuilder) primaryKeyCols() opt.ColSet { - // Ensure that pkCols set is initialized with the primary index columns. - if b.pkCols.Empty() { - primaryIndex := b.c.e.mem.Metadata().Table(b.tabID).Index(cat.PrimaryIndex) - for i, cnt := 0, primaryIndex.KeyColumnCount(); i < cnt; i++ { - b.pkCols.Add(b.tabID.IndexColumnID(primaryIndex, i)) - } +// SetScan constructs a standalone Scan expression. As a side effect, it clears +// any expressions added during previous invocations of the builder. SetScan +// makes a copy of scanPrivate so that it doesn't escape. +func (b *indexScanBuilder) SetScan(scanPrivate *memo.ScanPrivate) { + *b = indexScanBuilder{ + c: b.c, + f: b.f, + mem: b.mem, + tabID: b.tabID, + scanPrivate: *scanPrivate, } - return b.pkCols } -// setScan constructs a standalone Scan expression. As a side effect, it clears -// any expressions added during previous invocations of the builder. setScan -// makes a copy of scanPrivate so that it doesn't escape. -func (b *indexScanBuilder) setScan(scanPrivate *memo.ScanPrivate) { - b.scanPrivate = *scanPrivate - b.innerFilters = nil - b.outerFilters = nil - b.invertedFilterPrivate = memo.InvertedFilterPrivate{} - b.indexJoinPrivate = memo.IndexJoinPrivate{} +// AddConstProjections wraps the input expression with a Project expression with +// the given constant projection expressions. +func (b *indexScanBuilder) AddConstProjections(proj memo.ProjectionsExpr) { + if len(proj) != 0 { + if b.hasConstProjections() { + panic(errors.AssertionFailedf("cannot call AddConstProjections twice")) + } + if b.hasInnerFilters() || b.hasOuterFilters() { + panic(errors.AssertionFailedf("cannot call AddConstProjections after filters are added")) + } + b.constProjections = proj + } } -// addInvertedFilter wraps the input expression with an InvertedFilter +// AddInvertedFilter wraps the input expression with an InvertedFilter // expression having the given span expression. -func (b *indexScanBuilder) addInvertedFilter( +func (b *indexScanBuilder) AddInvertedFilter( spanExpr *inverted.SpanExpression, pfState *invertedexpr.PreFiltererStateForInvertedFilterer, invertedCol opt.ColumnID, ) { if spanExpr != nil { - if b.invertedFilterPrivate.InvertedColumn != 0 { - panic(errors.AssertionFailedf("cannot call addInvertedFilter twice")) + if b.hasInvertedFilter() { + panic(errors.AssertionFailedf("cannot call AddInvertedFilter twice")) } - if b.indexJoinPrivate.Table != 0 { - panic(errors.AssertionFailedf("cannot add inverted filter after index join is added")) + if b.hasIndexJoin() { + panic(errors.AssertionFailedf("cannot call AddInvertedFilter after index join is added")) } b.invertedFilterPrivate = memo.InvertedFilterPrivate{ InvertedExpression: spanExpr, @@ -107,30 +111,30 @@ func (b *indexScanBuilder) addInvertedFilter( } } -// addSelect wraps the input expression with a Select expression having the +// AddSelect wraps the input expression with a Select expression having the // given filter. -func (b *indexScanBuilder) addSelect(filters memo.FiltersExpr) { +func (b *indexScanBuilder) AddSelect(filters memo.FiltersExpr) { if len(filters) != 0 { - if b.indexJoinPrivate.Table == 0 { - if b.innerFilters != nil { - panic(errors.AssertionFailedf("cannot call addSelect methods twice before index join is added")) + if !b.hasIndexJoin() { + if b.hasInnerFilters() { + panic(errors.AssertionFailedf("cannot call AddSelect methods twice before index join is added")) } b.innerFilters = filters } else { - if b.outerFilters != nil { - panic(errors.AssertionFailedf("cannot call addSelect methods twice after index join is added")) + if b.hasOuterFilters() { + panic(errors.AssertionFailedf("cannot call AddSelect methods twice after index join is added")) } b.outerFilters = filters } } } -// addSelectAfterSplit first splits the given filter into two parts: a filter +// AddSelectAfterSplit first splits the given filter into two parts: a filter // that only involves columns in the given set, and a remaining filter that // includes everything else. The filter that is bound by the columns becomes a // Select expression that wraps the input expression, and the remaining filter // is returned (or 0 if there is no remaining filter). -func (b *indexScanBuilder) addSelectAfterSplit( +func (b *indexScanBuilder) AddSelectAfterSplit( filters memo.FiltersExpr, cols opt.ColSet, ) (remainingFilters memo.FiltersExpr) { if len(filters) == 0 { @@ -139,7 +143,7 @@ func (b *indexScanBuilder) addSelectAfterSplit( if b.c.FiltersBoundBy(filters, cols) { // Filter is fully bound by the cols, so add entire filter. - b.addSelect(filters) + b.AddSelect(filters) return nil } @@ -152,18 +156,18 @@ func (b *indexScanBuilder) addSelectAfterSplit( } // Add conditions which are fully bound by the cols and return the rest. - b.addSelect(boundConditions) + b.AddSelect(boundConditions) return b.c.ExtractUnboundConditions(filters, cols) } -// addIndexJoin wraps the input expression with an IndexJoin expression that +// AddIndexJoin wraps the input expression with an IndexJoin expression that // produces the given set of columns by lookup in the primary index. -func (b *indexScanBuilder) addIndexJoin(cols opt.ColSet) { - if b.indexJoinPrivate.Table != 0 { - panic(errors.AssertionFailedf("cannot call addIndexJoin twice")) +func (b *indexScanBuilder) AddIndexJoin(cols opt.ColSet) { + if b.hasIndexJoin() { + panic(errors.AssertionFailedf("cannot call AddIndexJoin twice")) } - if b.outerFilters != nil { - panic(errors.AssertionFailedf("cannot add index join after an outer filter has been added")) + if b.hasOuterFilters() { + panic(errors.AssertionFailedf("cannot call AddIndexJoin after an outer filter has been added")) } b.indexJoinPrivate = memo.IndexJoinPrivate{ Table: b.tabID, @@ -171,19 +175,34 @@ func (b *indexScanBuilder) addIndexJoin(cols opt.ColSet) { } } -// build constructs the final memo expression by composing together the various +// Build constructs the final memo expression by composing together the various // expressions that were specified by previous calls to various add methods. -func (b *indexScanBuilder) build(grp memo.RelExpr) { +func (b *indexScanBuilder) Build(grp memo.RelExpr) { // 1. Only scan. - if len(b.innerFilters) == 0 && b.indexJoinPrivate.Table == 0 { + if !b.hasConstProjections() && !b.hasInnerFilters() && !b.hasInvertedFilter() && !b.hasIndexJoin() { b.mem.AddScanToGroup(&memo.ScanExpr{ScanPrivate: b.scanPrivate}, grp) return } - // 2. Wrap scan in inner filter if it was added. input := b.f.ConstructScan(&b.scanPrivate) - if len(b.innerFilters) != 0 { - if b.indexJoinPrivate.Table == 0 && b.invertedFilterPrivate.InvertedColumn == 0 { + + // 2. Wrap input in a Project if constant projections were added. + if b.hasConstProjections() { + if !b.hasInnerFilters() && !b.hasInvertedFilter() && !b.hasIndexJoin() { + b.mem.AddProjectToGroup(&memo.ProjectExpr{ + Input: input, + Projections: b.constProjections, + Passthrough: b.scanPrivate.Cols, + }, grp) + return + } + + input = b.f.ConstructProject(input, b.constProjections, b.scanPrivate.Cols) + } + + // 3. Wrap input in inner filter if it was added. + if b.hasInnerFilters() { + if !b.hasInvertedFilter() && !b.hasIndexJoin() { b.mem.AddSelectToGroup(&memo.SelectExpr{Input: input, Filters: b.innerFilters}, grp) return } @@ -191,9 +210,9 @@ func (b *indexScanBuilder) build(grp memo.RelExpr) { input = b.f.ConstructSelect(input, b.innerFilters) } - // 3. Wrap input in inverted filter if it was added. - if b.invertedFilterPrivate.InvertedColumn != 0 { - if b.indexJoinPrivate.Table == 0 { + // 4. Wrap input in inverted filter if it was added. + if b.hasInvertedFilter() { + if !b.hasIndexJoin() { invertedFilter := &memo.InvertedFilterExpr{ Input: input, InvertedFilterPrivate: b.invertedFilterPrivate, } @@ -204,9 +223,9 @@ func (b *indexScanBuilder) build(grp memo.RelExpr) { input = b.f.ConstructInvertedFilter(input, &b.invertedFilterPrivate) } - // 4. Wrap input in index join if it was added. - if b.indexJoinPrivate.Table != 0 { - if len(b.outerFilters) == 0 { + // 5. Wrap input in index join if it was added. + if b.hasIndexJoin() { + if !b.hasOuterFilters() { indexJoin := &memo.IndexJoinExpr{Input: input, IndexJoinPrivate: b.indexJoinPrivate} b.mem.AddIndexJoinToGroup(indexJoin, grp) return @@ -215,11 +234,38 @@ func (b *indexScanBuilder) build(grp memo.RelExpr) { input = b.f.ConstructIndexJoin(input, &b.indexJoinPrivate) } - // 5. Wrap input in outer filter (which must exist at this point). - if len(b.outerFilters) == 0 { - // indexJoinDef == 0: outerFilters == 0 handled by #1 and #2 above. - // indexJoinDef != 0: outerFilters == 0 handled by #3 above. + // 6. Wrap input in outer filter (which must exist at this point). + if !b.hasOuterFilters() { + // indexJoinDef == 0: outerFilters == 0 handled by #1-4 above. + // indexJoinDef != 0: outerFilters == 0 handled by #5 above. panic(errors.AssertionFailedf("outer filter cannot be 0 at this point")) } b.mem.AddSelectToGroup(&memo.SelectExpr{Input: input, Filters: b.outerFilters}, grp) } + +// hasConstProjections returns true if constant projections have been added to +// the builder. +func (b *indexScanBuilder) hasConstProjections() bool { + return len(b.constProjections) != 0 +} + +// hasInnerFilters returns true if inner filters have been added to the builder. +func (b *indexScanBuilder) hasInnerFilters() bool { + return len(b.innerFilters) != 0 +} + +// hasOuterFilters returns true if outer filters have been added to the builder. +func (b *indexScanBuilder) hasOuterFilters() bool { + return len(b.outerFilters) != 0 +} + +// hasInvertedFilter returns true if inverted filters have been added to the +// builder. +func (b *indexScanBuilder) hasInvertedFilter() bool { + return b.invertedFilterPrivate.InvertedColumn != 0 +} + +// hasIndexJoin returns true if an index join has been added to the builder. +func (b *indexScanBuilder) hasIndexJoin() bool { + return b.indexJoinPrivate.Table != 0 +} diff --git a/pkg/sql/opt/xform/join_funcs.go b/pkg/sql/opt/xform/join_funcs.go index 39b7a05d23fb..7fefd5628d5b 100644 --- a/pkg/sql/opt/xform/join_funcs.go +++ b/pkg/sql/opt/xform/join_funcs.go @@ -229,8 +229,8 @@ func (c *CustomFuncs) GenerateLookupJoins( var pkCols opt.ColList var iter scanIndexIter - iter.Init(c.e.mem, &c.im, scanPrivate, on, rejectInvertedIndexes) - iter.ForEach(func(index cat.Index, onFilters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool) { + iter.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, scanPrivate, on, rejectInvertedIndexes) + iter.ForEach(func(index cat.Index, onFilters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool, constProj memo.ProjectionsExpr) { // Find the longest prefix of index key columns that are constrained by // an equality with another column or a constant. numIndexKeyCols := index.LaxKeyColumnCount() @@ -364,7 +364,11 @@ func (c *CustomFuncs) GenerateLookupJoins( lookupJoin.Cols = lookupJoin.LookupExpr.OuterCols() lookupJoin.Cols.UnionWith(inputProps.OutputCols) - if isCovering { + // TODO(mgartner): The right side of the join can "produce" columns held + // constant by a partial index predicate, but the lookup joiner does not + // currently support this. For now, if constProj is non-empty we + // consider the index non-covering. + if isCovering && len(constProj) == 0 { // Case 1 (see function comment). lookupJoin.Cols.UnionWith(scanPrivate.Cols) @@ -639,8 +643,8 @@ func (c *CustomFuncs) GenerateInvertedJoins( var optionalFilters memo.FiltersExpr var iter scanIndexIter - iter.Init(c.e.mem, &c.im, scanPrivate, on, rejectNonInvertedIndexes) - iter.ForEach(func(index cat.Index, onFilters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool) { + iter.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, scanPrivate, on, rejectNonInvertedIndexes) + iter.ForEach(func(index cat.Index, onFilters memo.FiltersExpr, indexCols opt.ColSet, _ bool, _ memo.ProjectionsExpr) { invertedJoin := memo.InvertedJoinExpr{Input: input} numPrefixCols := index.NonInvertedPrefixColumnCount() diff --git a/pkg/sql/opt/xform/limit_funcs.go b/pkg/sql/opt/xform/limit_funcs.go index 23960d64ce0c..6dbf038f7b12 100644 --- a/pkg/sql/opt/xform/limit_funcs.go +++ b/pkg/sql/opt/xform/limit_funcs.go @@ -19,6 +19,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/opt/props" "github.com/cockroachdb/cockroach/pkg/sql/opt/props/physical" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/errors" ) // LimitScanPrivate constructs a new ScanPrivate value that is based on the @@ -93,14 +94,25 @@ func (c *CustomFuncs) GenerateLimitedScans( ) { limitVal := int64(*limit.(*tree.DInt)) + var pkCols opt.ColSet var sb indexScanBuilder - sb.init(c, scanPrivate.Table) + sb.Init(c, scanPrivate.Table) // Iterate over all non-inverted, non-partial indexes, looking for those // that can be limited. var iter scanIndexIter - iter.Init(c.e.mem, &c.im, scanPrivate, nil /* filters */, rejectInvertedIndexes|rejectPartialIndexes) - iter.ForEach(func(index cat.Index, filters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool) { + iter.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, scanPrivate, nil /* filters */, rejectInvertedIndexes|rejectPartialIndexes) + iter.ForEach(func(index cat.Index, filters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool, constProj memo.ProjectionsExpr) { + // The iterator rejects partial indexes because there are no filters to + // imply a partial index predicate. constProj is a projection of + // constant values based on a partial index predicate. It should always + // be empty because we iterate only on non-partial indexes. If it is + // not, we panic to avoid performing a logically incorrect + // transformation. + if len(constProj) != 0 { + panic(errors.AssertionFailedf("expected constProj to be empty")) + } + newScanPrivate := *scanPrivate newScanPrivate.Index = index.Ordinal() @@ -118,8 +130,8 @@ func (c *CustomFuncs) GenerateLimitedScans( // If the alternate index includes the set of needed columns, then construct // a new Scan operator using that index. if isCovering { - sb.setScan(&newScanPrivate) - sb.build(grp) + sb.SetScan(&newScanPrivate) + sb.Build(grp) return } @@ -129,18 +141,23 @@ func (c *CustomFuncs) GenerateLimitedScans( return } + // Calculate the PK columns once. + if pkCols.Empty() { + pkCols = c.PrimaryKeyCols(scanPrivate.Table) + } + // Scan whatever columns we need which are available from the index, plus // the PK columns. newScanPrivate.Cols = indexCols.Intersection(scanPrivate.Cols) - newScanPrivate.Cols.UnionWith(sb.primaryKeyCols()) - sb.setScan(&newScanPrivate) + newScanPrivate.Cols.UnionWith(pkCols) + sb.SetScan(&newScanPrivate) // The Scan operator will go into its own group (because it projects a // different set of columns), and the IndexJoin operator will be added to // the same group as the original Limit operator. - sb.addIndexJoin(scanPrivate.Cols) + sb.AddIndexJoin(scanPrivate.Cols) - sb.build(grp) + sb.Build(grp) }) } diff --git a/pkg/sql/opt/xform/scan_funcs.go b/pkg/sql/opt/xform/scan_funcs.go index 9b38e13fe8da..175a4a97ec31 100644 --- a/pkg/sql/opt/xform/scan_funcs.go +++ b/pkg/sql/opt/xform/scan_funcs.go @@ -19,6 +19,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/util" + "github.com/cockroachdb/errors" ) // GenerateIndexScans enumerates all non-inverted secondary indexes on the given @@ -39,9 +40,20 @@ import ( // index joins are introduced into the memo. func (c *CustomFuncs) GenerateIndexScans(grp memo.RelExpr, scanPrivate *memo.ScanPrivate) { // Iterate over all non-inverted and non-partial secondary indexes. + var pkCols opt.ColSet var iter scanIndexIter - iter.Init(c.e.mem, &c.im, scanPrivate, nil /* filters */, rejectPrimaryIndex|rejectInvertedIndexes) - iter.ForEach(func(index cat.Index, filters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool) { + iter.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, scanPrivate, nil /* filters */, rejectPrimaryIndex|rejectInvertedIndexes) + iter.ForEach(func(index cat.Index, filters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool, constProj memo.ProjectionsExpr) { + // The iterator only produces pseudo-partial indexes (the predicate is + // true) because no filters are passed to iter.Init to imply a partial + // index predicate. constProj is a projection of constant values based + // on a partial index predicate. It should always be empty because a + // pseudo-partial index cannot hold a column constant. If it is not, we + // panic to avoid performing a logically incorrect transformation. + if len(constProj) != 0 { + panic(errors.AssertionFailedf("expected constProj to be empty")) + } + // If the secondary index includes the set of needed columns, then construct // a new Scan operator using that index. if isCovering { @@ -60,18 +72,23 @@ func (c *CustomFuncs) GenerateIndexScans(grp memo.RelExpr, scanPrivate *memo.Sca } var sb indexScanBuilder - sb.init(c, scanPrivate.Table) + sb.Init(c, scanPrivate.Table) + + // Calculate the PK columns once. + if pkCols.Empty() { + pkCols = c.PrimaryKeyCols(scanPrivate.Table) + } // Scan whatever columns we need which are available from the index, plus // the PK columns. newScanPrivate := *scanPrivate newScanPrivate.Index = index.Ordinal() newScanPrivate.Cols = indexCols.Intersection(scanPrivate.Cols) - newScanPrivate.Cols.UnionWith(sb.primaryKeyCols()) - sb.setScan(&newScanPrivate) + newScanPrivate.Cols.UnionWith(pkCols) + sb.SetScan(&newScanPrivate) - sb.addIndexJoin(scanPrivate.Cols) - sb.build(grp) + sb.AddIndexJoin(scanPrivate.Cols) + sb.Build(grp) }) } diff --git a/pkg/sql/opt/xform/scan_index_iter.go b/pkg/sql/opt/xform/scan_index_iter.go index 67992352bd2e..35ba8d4a2914 100644 --- a/pkg/sql/opt/xform/scan_index_iter.go +++ b/pkg/sql/opt/xform/scan_index_iter.go @@ -11,10 +11,13 @@ package xform import ( + "github.com/cockroachdb/cockroach/pkg/sql/catalog/colinfo" "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" + "github.com/cockroachdb/cockroach/pkg/sql/opt/norm" "github.com/cockroachdb/cockroach/pkg/sql/opt/partialidx" + "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/errors" ) @@ -48,7 +51,8 @@ const ( // scanIndexIter is a helper struct that facilitates iteration over the indexes // of a Scan operator table. type scanIndexIter struct { - mem *memo.Memo + evalCtx *tree.EvalContext + f *norm.Factory im *partialidx.Implicator tabMeta *opt.TableMeta @@ -82,6 +86,8 @@ type scanIndexIter struct { // Init initializes a new scanIndexIter. func (it *scanIndexIter) Init( + evalCtx *tree.EvalContext, + f *norm.Factory, mem *memo.Memo, im *partialidx.Implicator, scanPrivate *memo.ScanPrivate, @@ -91,7 +97,8 @@ func (it *scanIndexIter) Init( // This initialization pattern ensures that fields are not unwittingly // reused. Field reuse must be explicit. *it = scanIndexIter{ - mem: mem, + evalCtx: evalCtx, + f: f, im: im, tabMeta: mem.Metadata().TableMeta(scanPrivate.Table), scanPrivate: scanPrivate, @@ -159,17 +166,28 @@ func (it *scanIndexIter) SetOriginalFilters(filters memo.FiltersExpr) { // ForEachStartingAfter functions. It is invoked for each index enumerated. // // The function is called with the enumerated index, the filters that must be -// applied after a scan over the index, the index columns, and a boolean that is -// true if the index covers the scanPrivate's columns. If the index is a partial -// index, the filters are the remaining filters after proving partial index -// implication (see partialidx.Implicator). Otherwise, the filters are the same -// filters that were passed to Init. +// applied after a scan over the index, and the index columns. The isCovering +// boolean is true if the index covers the scanPrivate's columns, indicating +// that an index join with the primary index is not necessary. A partial index +// may cover columns not actually stored in the index because the predicate +// holds the column constant. In this case, the provided constProj must be +// projected after the scan to produce all required columns. +// +// If the index is a partial index, the filters are the remaining filters after +// proving partial index implication (see partialidx.Implicator). Otherwise, the +// filters are the same filters that were passed to Init. // // Note that the filters argument CANNOT be mutated in the callback function // because these filters are used internally by scanIndexIter for partial index // implication while iterating over indexes. In tests the filtersMutateChecker // will detect a callback that mutates filters and panic. -type enumerateIndexFunc func(idx cat.Index, filters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool) +type enumerateIndexFunc func( + idx cat.Index, + filters memo.FiltersExpr, + indexCols opt.ColSet, + isCovering bool, + constProj memo.ProjectionsExpr, +) // ForEach calls the given callback function for every index of the Scan // operator's table in the order they appear in the catalog. @@ -234,8 +252,9 @@ func (it *scanIndexIter) ForEachStartingAfter(ord int, f enumerateIndexFunc) { // If the index is a partial index, check whether the filters imply the // predicate. + var predFilters memo.FiltersExpr if isPartialIndex { - predFilters := *pred.(*memo.FiltersExpr) + predFilters = *pred.(*memo.FiltersExpr) // If there are no filters, then skip over any partial indexes that // are not pseudo-partial indexes. @@ -260,7 +279,23 @@ func (it *scanIndexIter) ForEachStartingAfter(ord int, f enumerateIndexFunc) { indexCols := it.tabMeta.IndexColumns(ord) isCovering := it.scanPrivate.Cols.SubsetOf(indexCols) - f(index, filters, indexCols, isCovering) + // If the index does not contain all required columns, attempt to use + // columns held constant in the partial index predicate to cover the + // columns. + var constProj memo.ProjectionsExpr + if !isCovering && len(predFilters) > 0 { + constCols := it.extractConstNonCompositeColumns(predFilters) + if !constCols.Empty() && it.scanPrivate.Cols.SubsetOf(indexCols.Union(constCols)) { + isCovering = true + + // Build a projection only for constant columns not in the + // index. + constCols = constCols.Difference(indexCols) + constProj = it.buildConstProjectionsFromPredicate(predFilters, constCols) + } + } + + f(index, filters, indexCols, isCovering, constProj) // Verify that f did not mutate filters or originalFilters (in test // builds only). @@ -300,6 +335,55 @@ func (it *scanIndexIter) filtersImplyPredicate( return nil, false } +// extractConstNonCompositeColumns returns the set of columns held constant by +// the given filters and of types that do not have composite encodings. +func (it *scanIndexIter) extractConstNonCompositeColumns(f memo.FiltersExpr) opt.ColSet { + constCols := memo.ExtractConstColumns(f, it.evalCtx) + var constNonCompositeCols opt.ColSet + for col, ok := constCols.Next(0); ok; col, ok = constCols.Next(col + 1) { + ord := it.tabMeta.MetaID.ColumnOrdinal(col) + typ := it.tabMeta.Table.Column(ord).DatumType() + if !colinfo.HasCompositeKeyEncoding(typ) { + constNonCompositeCols.Add(col) + } + } + return constNonCompositeCols +} + +// buildConstProjectionsFromPredicate builds a ProjectionsExpr that projects +// constant values for the given constCols. The constant values are extracted +// from the given partial index predicate expression. Panics if a constant value +// cannot be extracted from pred for any of the constCols. +func (it *scanIndexIter) buildConstProjectionsFromPredicate( + pred memo.FiltersExpr, constCols opt.ColSet, +) memo.ProjectionsExpr { + proj := make(memo.ProjectionsExpr, 0, constCols.Len()) + for col, ok := constCols.Next(0); ok; col, ok = constCols.Next(col + 1) { + ord := it.tabMeta.MetaID.ColumnOrdinal(col) + typ := it.tabMeta.Table.Column(ord).DatumType() + + val := memo.ExtractValueForConstColumn(pred, it.evalCtx, col) + if val == nil { + panic(errors.AssertionFailedf("could not extract constant value for column %d", col)) + } + + var scalar opt.ScalarExpr + if val == tree.DNull { + // NULL values should always be a memo.NullExpr, not a + // memo.ConstExpr. + scalar = memo.NullSingleton + } else { + scalar = it.f.ConstructConst(val, typ) + } + + proj = append(proj, it.f.ConstructProjectionsItem( + scalar, + col, + )) + } + return proj +} + // hasRejectFlags tests whether the given flags are all set. func (it *scanIndexIter) hasRejectFlags(subset indexRejectFlags) bool { return it.rejectFlags&subset == subset diff --git a/pkg/sql/opt/xform/select_funcs.go b/pkg/sql/opt/xform/select_funcs.go index 3619932aec84..e0e9b5624455 100644 --- a/pkg/sql/opt/xform/select_funcs.go +++ b/pkg/sql/opt/xform/select_funcs.go @@ -97,42 +97,49 @@ func (c *CustomFuncs) GeneratePartialIndexScans( grp memo.RelExpr, scanPrivate *memo.ScanPrivate, filters memo.FiltersExpr, ) { // Iterate over all partial indexes. + var pkCols opt.ColSet var iter scanIndexIter - iter.Init(c.e.mem, &c.im, scanPrivate, filters, rejectNonPartialIndexes|rejectInvertedIndexes) - iter.ForEach(func(index cat.Index, remainingFilters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool) { + iter.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, scanPrivate, filters, rejectNonPartialIndexes|rejectInvertedIndexes) + iter.ForEach(func(index cat.Index, remainingFilters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool, constProj memo.ProjectionsExpr) { var sb indexScanBuilder - sb.init(c, scanPrivate.Table) + sb.Init(c, scanPrivate.Table) newScanPrivate := *scanPrivate newScanPrivate.Index = index.Ordinal() + newScanPrivate.Cols = indexCols.Intersection(scanPrivate.Cols) // If index is covering, just add a Select with the remaining filters, // if there are any. if isCovering { - sb.setScan(&newScanPrivate) - sb.addSelect(remainingFilters) - sb.build(grp) + sb.SetScan(&newScanPrivate) + sb.AddConstProjections(constProj) + sb.AddSelect(remainingFilters) + sb.Build(grp) return } + // Calculate the PK columns once. + if pkCols.Empty() { + pkCols = c.PrimaryKeyCols(scanPrivate.Table) + } + // If the index is not covering, scan the needed index columns plus // primary key columns. - newScanPrivate.Cols = indexCols.Intersection(scanPrivate.Cols) - newScanPrivate.Cols.UnionWith(sb.primaryKeyCols()) - sb.setScan(&newScanPrivate) + newScanPrivate.Cols.UnionWith(pkCols) + sb.SetScan(&newScanPrivate) // Add a Select with any remaining filters that can be filtered before // the IndexJoin. If there are no remaining filters this is a no-op. If // all or parts of the remaining filters cannot be applied until after // the IndexJoin, the new value of remainingFilters will contain those // filters. - remainingFilters = sb.addSelectAfterSplit(remainingFilters, newScanPrivate.Cols) + remainingFilters = sb.AddSelectAfterSplit(remainingFilters, newScanPrivate.Cols) // Add an IndexJoin to retrieve the columns not provided by the Scan. - sb.addIndexJoin(scanPrivate.Cols) + sb.AddIndexJoin(scanPrivate.Cols) // Add a Select with any remaining filters. - sb.addSelect(remainingFilters) - sb.build(grp) + sb.AddSelect(remainingFilters) + sb.Build(grp) }) } @@ -202,8 +209,9 @@ func (c *CustomFuncs) GeneratePartialIndexScans( func (c *CustomFuncs) GenerateConstrainedScans( grp memo.RelExpr, scanPrivate *memo.ScanPrivate, explicitFilters memo.FiltersExpr, ) { + var pkCols opt.ColSet var sb indexScanBuilder - sb.init(c, scanPrivate.Table) + sb.Init(c, scanPrivate.Table) // Generate implicit filters from constraints and computed columns as // optional filters to help constrain an index scan. @@ -218,8 +226,8 @@ func (c *CustomFuncs) GenerateConstrainedScans( md := c.e.mem.Metadata() tabMeta := md.TableMeta(scanPrivate.Table) var iter scanIndexIter - iter.Init(c.e.mem, &c.im, scanPrivate, explicitFilters, rejectInvertedIndexes) - iter.ForEach(func(index cat.Index, filters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool) { + iter.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, scanPrivate, explicitFilters, rejectInvertedIndexes) + iter.ForEach(func(index cat.Index, filters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool, constProj memo.ProjectionsExpr) { // We only consider the partition values when a particular index can otherwise // not be constrained. For indexes that are constrained, the partitioned values // add no benefit as they don't really constrain anything. @@ -329,21 +337,26 @@ func (c *CustomFuncs) GenerateConstrainedScans( // Construct new constrained ScanPrivate. newScanPrivate := *scanPrivate newScanPrivate.Index = index.Ordinal() + newScanPrivate.Cols = indexCols.Intersection(scanPrivate.Cols) newScanPrivate.Constraint = constraint // Record whether we were able to use partitions to constrain the scan. newScanPrivate.PartitionConstrainedScan = (len(partitionFilters) > 0) - // If the alternate index includes the set of needed columns, then construct - // a new Scan operator using that index. + // If the alternate index includes the set of needed columns, then + // construct a new Scan operator using that index. if isCovering { - sb.setScan(&newScanPrivate) + sb.SetScan(&newScanPrivate) + + // Project constants from partial index predicate filters, if there + // are any. + sb.AddConstProjections(constProj) // If there are remaining filters, then the constrained Scan operator // will be created in a new group, and a Select operator will be added // to the same group as the original operator. - sb.addSelect(remainingFilters) + sb.AddSelect(remainingFilters) - sb.build(grp) + sb.Build(grp) return } @@ -353,19 +366,23 @@ func (c *CustomFuncs) GenerateConstrainedScans( return } - // Scan whatever columns we need which are available from the index, plus - // the PK columns. - newScanPrivate.Cols = indexCols.Intersection(scanPrivate.Cols) - newScanPrivate.Cols.UnionWith(sb.primaryKeyCols()) - sb.setScan(&newScanPrivate) + // Calculate the PK columns once. + if pkCols.Empty() { + pkCols = c.PrimaryKeyCols(scanPrivate.Table) + } + + // If the index is not covering, scan the needed index columns plus + // primary key columns. + newScanPrivate.Cols.UnionWith(pkCols) + sb.SetScan(&newScanPrivate) // If remaining filter exists, split it into one part that can be pushed // below the IndexJoin, and one part that needs to stay above. - remainingFilters = sb.addSelectAfterSplit(remainingFilters, newScanPrivate.Cols) - sb.addIndexJoin(scanPrivate.Cols) - sb.addSelect(remainingFilters) + remainingFilters = sb.AddSelectAfterSplit(remainingFilters, newScanPrivate.Cols) + sb.AddIndexJoin(scanPrivate.Cols) + sb.AddSelect(remainingFilters) - sb.build(grp) + sb.Build(grp) }) } @@ -664,8 +681,9 @@ func (c *CustomFuncs) partitionValuesFilters( func (c *CustomFuncs) GenerateInvertedIndexScans( grp memo.RelExpr, scanPrivate *memo.ScanPrivate, filters memo.FiltersExpr, ) { + var pkCols opt.ColSet var sb indexScanBuilder - sb.init(c, scanPrivate.Table) + sb.Init(c, scanPrivate.Table) tabMeta := c.e.mem.Metadata().TableMeta(scanPrivate.Table) // Generate implicit filters from constraints and computed columns as @@ -676,8 +694,8 @@ func (c *CustomFuncs) GenerateInvertedIndexScans( // Iterate over all inverted indexes. var iter scanIndexIter - iter.Init(c.e.mem, &c.im, scanPrivate, filters, rejectNonInvertedIndexes) - iter.ForEach(func(index cat.Index, filters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool) { + iter.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, scanPrivate, filters, rejectNonInvertedIndexes) + iter.ForEach(func(index cat.Index, filters memo.FiltersExpr, indexCols opt.ColSet, _ bool, _ memo.ProjectionsExpr) { // Check whether the filter can constrain the index. spanExpr, constraint, remainingFilters, pfState, ok := invertedidx.TryFilterInvertedIndex( c.e.evalCtx, c.e.f, filters, optionalFilters, scanPrivate.Table, index, tabMeta.ComputedCols, @@ -714,12 +732,16 @@ func (c *CustomFuncs) GenerateInvertedIndexScans( newScanPrivate.Constraint = constraint newScanPrivate.InvertedConstraint = spansToRead + // Calculate the PK columns once. + if pkCols.Empty() { + pkCols = c.PrimaryKeyCols(scanPrivate.Table) + } + // We will need an inverted filter above the scan if the spanExpr might // produce duplicate primary keys or requires at least one UNION or // INTERSECTION. In this case, we must scan both the primary key columns // and the inverted key column. needInvertedFilter := !spanExpr.Unique || spanExpr.Operator != inverted.None - pkCols := sb.primaryKeyCols() newScanPrivate.Cols = pkCols.Copy() var invertedCol opt.ColumnID if needInvertedFilter { @@ -734,20 +756,20 @@ func (c *CustomFuncs) GenerateInvertedIndexScans( // index join will be removed by EliminateIndexJoinInsideProject, but // it'd be more efficient to not create the index join in the first // place. - sb.setScan(&newScanPrivate) + sb.SetScan(&newScanPrivate) // Add an inverted filter if needed. if needInvertedFilter { - sb.addInvertedFilter(spanExpr, pfState, invertedCol) + sb.AddInvertedFilter(spanExpr, pfState, invertedCol) } // If remaining filter exists, split it into one part that can be pushed // below the IndexJoin, and one part that needs to stay above. - filters = sb.addSelectAfterSplit(filters, pkCols) - sb.addIndexJoin(scanPrivate.Cols) - sb.addSelect(filters) + filters = sb.AddSelectAfterSplit(filters, pkCols) + sb.AddIndexJoin(scanPrivate.Cols) + sb.AddSelect(filters) - sb.build(grp) + sb.Build(grp) }) } @@ -893,8 +915,8 @@ func (c *CustomFuncs) GenerateZigzagJoins( // TODO(mgartner): We should consider primary indexes when it has multiple // columns and only the first is being constrained. var iter scanIndexIter - iter.Init(c.e.mem, &c.im, scanPrivate, filters, rejectPrimaryIndex|rejectInvertedIndexes) - iter.ForEach(func(leftIndex cat.Index, outerFilters memo.FiltersExpr, leftCols opt.ColSet, _ bool) { + iter.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, scanPrivate, filters, rejectPrimaryIndex|rejectInvertedIndexes) + iter.ForEach(func(leftIndex cat.Index, outerFilters memo.FiltersExpr, leftCols opt.ColSet, _ bool, _ memo.ProjectionsExpr) { leftFixed := c.indexConstrainedCols(leftIndex, scanPrivate.Table, fixedCols) // Short-circuit quickly if the first column in the index is not a fixed // column. @@ -903,9 +925,9 @@ func (c *CustomFuncs) GenerateZigzagJoins( } var iter2 scanIndexIter - iter2.Init(c.e.mem, &c.im, scanPrivate, outerFilters, rejectPrimaryIndex|rejectInvertedIndexes) + iter2.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, scanPrivate, outerFilters, rejectPrimaryIndex|rejectInvertedIndexes) iter2.SetOriginalFilters(filters) - iter2.ForEachStartingAfter(leftIndex.Ordinal(), func(rightIndex cat.Index, innerFilters memo.FiltersExpr, rightCols opt.ColSet, _ bool) { + iter2.ForEachStartingAfter(leftIndex.Ordinal(), func(rightIndex cat.Index, innerFilters memo.FiltersExpr, rightCols opt.ColSet, _ bool, _ memo.ProjectionsExpr) { rightFixed := c.indexConstrainedCols(rightIndex, scanPrivate.Table, fixedCols) // If neither side contributes a fixed column not contributed by the // other, then there's no reason to zigzag on this pair of indexes. @@ -1201,12 +1223,12 @@ func (c *CustomFuncs) GenerateInvertedIndexZigzagJoins( } var sb indexScanBuilder - sb.init(c, scanPrivate.Table) + sb.Init(c, scanPrivate.Table) // Iterate over all inverted indexes. var iter scanIndexIter - iter.Init(c.e.mem, &c.im, scanPrivate, filters, rejectNonInvertedIndexes) - iter.ForEach(func(index cat.Index, filters memo.FiltersExpr, indexCols opt.ColSet, _ bool) { + iter.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, scanPrivate, filters, rejectNonInvertedIndexes) + iter.ForEach(func(index cat.Index, filters memo.FiltersExpr, indexCols opt.ColSet, _ bool, _ memo.ProjectionsExpr) { if index.NonInvertedPrefixColumnCount() > 0 { // TODO(mgartner): We don't yet support using multi-column inverted // indexes with zigzag joins. diff --git a/pkg/sql/opt/xform/testdata/rules/join b/pkg/sql/opt/xform/testdata/rules/join index c6db624e9535..e0b4b7ec9722 100644 --- a/pkg/sql/opt/xform/testdata/rules/join +++ b/pkg/sql/opt/xform/testdata/rules/join @@ -4420,6 +4420,48 @@ project │ └── cost: 25.01 └── filters (true) +exec-ddl +DROP INDEX full_idx +---- + +exec-ddl +DROP INDEX partial_idx +---- + +exec-ddl +CREATE INDEX partial_idx ON partial_tab (i) WHERE s = 'foo' +---- + +# Generate a second inner join to lookup s even though the partial index +# predicate holds s constant. +# TODO(mgartner): The right side of the join can "produce" columns held constant +# by a partial index predicate, so the second inner-join to lookup s is not +# necessary. +opt expect=GenerateLookupJoinsWithFilter +SELECT m, s FROM small INNER LOOKUP JOIN partial_tab ON n = i WHERE s = 'foo' +---- +project + ├── columns: m:1 s:7!null + ├── fd: ()-->(7) + └── inner-join (lookup partial_tab) + ├── columns: m:1 n:2!null i:6!null s:7!null + ├── key columns: [5] = [5] + ├── lookup columns are key + ├── fd: ()-->(7), (2)==(6), (6)==(2) + ├── inner-join (lookup partial_tab@partial_idx,partial) + │ ├── columns: m:1 n:2!null k:5!null i:6!null + │ ├── flags: force lookup join (into right side) + │ ├── key columns: [2] = [6] + │ ├── fd: (5)-->(6), (2)==(6), (6)==(2) + │ ├── scan small + │ │ └── columns: m:1 n:2 + │ └── filters (true) + └── filters (true) + +exec-ddl +DROP INDEX partial_idx +---- + # ------------------------------------------------------- # GenerateInvertedJoins + GenerateInvertedJoinsFromSelect # ------------------------------------------------------- diff --git a/pkg/sql/opt/xform/testdata/rules/project b/pkg/sql/opt/xform/testdata/rules/project index c5bef4b4615c..0adba072dc56 100644 --- a/pkg/sql/opt/xform/testdata/rules/project +++ b/pkg/sql/opt/xform/testdata/rules/project @@ -5,7 +5,7 @@ CREATE TABLE a ( s STRING, b BOOL, j JSON, - INDEX i (i) WHERE s = 'foo', + INDEX i (i) WHERE s LIKE 'foo%', INVERTED INDEX j (j) ) ---- @@ -17,7 +17,7 @@ CREATE TABLE a ( # Eliminate the IndexJoin when the Project passthrough columns are a subset of # the IndexJoin's input columns. opt expect=EliminateIndexJoinInsideProject -SELECT k, i FROM a WHERE s = 'foo' +SELECT k, i FROM a WHERE s LIKE 'foo%' ---- project ├── columns: k:1!null i:2 @@ -31,7 +31,7 @@ project # Eliminate the IndexJoin when the Project projection outer columns are a subset of # the IndexJoin's input columns. opt expect=EliminateIndexJoinInsideProject -SELECT k + 1, i + 1 FROM a WHERE s = 'foo' +SELECT k + 1, i + 1 FROM a WHERE s LIKE 'foo%' ---- project ├── columns: "?column?":8!null "?column?":9 @@ -47,7 +47,7 @@ project # Do not eliminate the IndexJoin when the Project passthrough columns are not a # subset of the IndexJoin's input columns. opt expect-not=EliminateIndexJoinInsideProject -SELECT k, b FROM a WHERE s = 'foo' +SELECT k, b FROM a WHERE s LIKE 'foo%' ---- project ├── columns: k:1!null b:4 @@ -56,7 +56,7 @@ project └── index-join a ├── columns: k:1!null s:3!null b:4 ├── key: (1) - ├── fd: ()-->(3), (1)-->(4) + ├── fd: (1)-->(3,4) └── scan a@i,partial ├── columns: k:1!null └── key: (1) @@ -64,7 +64,7 @@ project # Do not eliminate the IndexJoin when the Project projection outer columns are # not a subset of the IndexJoin's input columns. opt expect-not=EliminateIndexJoinInsideProject -SELECT k, NOT b FROM a WHERE s = 'foo' +SELECT k, NOT b FROM a WHERE s LIKE 'foo%' ---- project ├── columns: k:1!null "?column?":8 @@ -73,7 +73,7 @@ project ├── index-join a │ ├── columns: k:1!null s:3!null b:4 │ ├── key: (1) - │ ├── fd: ()-->(3), (1)-->(4) + │ ├── fd: (1)-->(3,4) │ └── scan a@i,partial │ ├── columns: k:1!null │ └── key: (1) diff --git a/pkg/sql/opt/xform/testdata/rules/select b/pkg/sql/opt/xform/testdata/rules/select index aba5992455ce..c5d208877d9f 100644 --- a/pkg/sql/opt/xform/testdata/rules/select +++ b/pkg/sql/opt/xform/testdata/rules/select @@ -84,6 +84,7 @@ CREATE TABLE g exec-ddl CREATE TABLE p ( + k INT PRIMARY KEY, i INT, f FLOAT, s STRING, @@ -146,10 +147,10 @@ opt expect=GeneratePartialIndexScans SELECT i FROM p WHERE s = 'foo' ---- project - ├── columns: i:1 + ├── columns: i:2 └── scan p@idx,partial - ├── columns: i:1 s:3!null - └── fd: ()-->(3) + ├── columns: i:2 s:4!null + └── fd: ()-->(4) # Generate a partial index scan inside a select when the index is covering and # there are remaining filters. @@ -157,15 +158,15 @@ opt expect=GeneratePartialIndexScans SELECT i FROM p WHERE s = 'foo' AND (i = 1 OR f = 2.0) ---- project - ├── columns: i:1 + ├── columns: i:2 └── select - ├── columns: i:1 f:2 s:3!null - ├── fd: ()-->(3) + ├── columns: i:2 f:3 s:4!null + ├── fd: ()-->(4) ├── scan p@idx,partial - │ ├── columns: i:1 f:2 s:3!null - │ └── fd: ()-->(3) + │ ├── columns: i:2 f:3 s:4!null + │ └── fd: ()-->(4) └── filters - └── (i:1 = 1) OR (f:2 = 2.0) [outer=(1,2)] + └── (i:2 = 1) OR (f:3 = 2.0) [outer=(2,3)] # Generate a partial index scan inside an index-join when the index is not # covering and there no remaining filters. @@ -173,14 +174,14 @@ opt expect=GeneratePartialIndexScans SELECT b FROM p WHERE s = 'foo' ---- project - ├── columns: b:4 + ├── columns: b:5 └── index-join p - ├── columns: s:3!null b:4 - ├── fd: ()-->(3) + ├── columns: s:4!null b:5 + ├── fd: ()-->(4) └── scan p@idx,partial - ├── columns: s:3!null rowid:5!null - ├── key: (5) - └── fd: ()-->(3) + ├── columns: k:1!null s:4!null + ├── key: (1) + └── fd: ()-->(4) # Generate a partial index scan inside a select inside an index-join when the # index is not covering and there are remaining filters that are covered by the @@ -189,20 +190,20 @@ opt expect=GeneratePartialIndexScans SELECT b FROM p WHERE s = 'foo' AND (i = 1 OR f = 2.0) ---- project - ├── columns: b:4 + ├── columns: b:5 └── index-join p - ├── columns: i:1 f:2 s:3!null b:4 - ├── fd: ()-->(3) + ├── columns: i:2 f:3 s:4!null b:5 + ├── fd: ()-->(4) └── select - ├── columns: i:1 f:2 s:3!null rowid:5!null - ├── key: (5) - ├── fd: ()-->(3), (5)-->(1,2) + ├── columns: k:1!null i:2 f:3 s:4!null + ├── key: (1) + ├── fd: ()-->(4), (1)-->(2,3) ├── scan p@idx,partial - │ ├── columns: i:1 f:2 s:3!null rowid:5!null - │ ├── key: (5) - │ └── fd: ()-->(3), (5)-->(1,2) + │ ├── columns: k:1!null i:2 f:3 s:4!null + │ ├── key: (1) + │ └── fd: ()-->(4), (1)-->(2,3) └── filters - └── (i:1 = 1) OR (f:2 = 2.0) [outer=(1,2)] + └── (i:2 = 1) OR (f:3 = 2.0) [outer=(2,3)] # Generate a partial index scan inside an index-join inside a select when the # index is not covering and the remaining filters are not covered by the index. @@ -210,20 +211,20 @@ opt expect=GeneratePartialIndexScans SELECT b FROM p WHERE s = 'foo' AND b ---- project - ├── columns: b:4!null - ├── fd: ()-->(4) + ├── columns: b:5!null + ├── fd: ()-->(5) └── select - ├── columns: s:3!null b:4!null - ├── fd: ()-->(3,4) + ├── columns: s:4!null b:5!null + ├── fd: ()-->(4,5) ├── index-join p - │ ├── columns: s:3 b:4 - │ ├── fd: ()-->(3) + │ ├── columns: s:4 b:5 + │ ├── fd: ()-->(4) │ └── scan p@idx,partial - │ ├── columns: s:3!null rowid:5!null - │ ├── key: (5) - │ └── fd: ()-->(3) + │ ├── columns: k:1!null s:4!null + │ ├── key: (1) + │ └── fd: ()-->(4) └── filters - └── b:4 [outer=(4), constraints=(/4: [/true - /true]; tight), fd=()-->(4)] + └── b:5 [outer=(5), constraints=(/5: [/true - /true]; tight), fd=()-->(5)] # Generate a partial index scan inside a Select/IndexJoin/Select when the index # is not covering and the remaining filters are partially covered by the index. @@ -231,26 +232,26 @@ opt expect=GeneratePartialIndexScans SELECT b FROM p WHERE s = 'foo' AND f = 1 AND b ---- project - ├── columns: b:4!null - ├── fd: ()-->(4) + ├── columns: b:5!null + ├── fd: ()-->(5) └── select - ├── columns: f:2!null s:3!null b:4!null - ├── fd: ()-->(2-4) + ├── columns: f:3!null s:4!null b:5!null + ├── fd: ()-->(3-5) ├── index-join p - │ ├── columns: f:2 s:3 b:4 - │ ├── fd: ()-->(2,3) + │ ├── columns: f:3 s:4 b:5 + │ ├── fd: ()-->(3,4) │ └── select - │ ├── columns: f:2!null s:3!null rowid:5!null - │ ├── key: (5) - │ ├── fd: ()-->(2,3) + │ ├── columns: k:1!null f:3!null s:4!null + │ ├── key: (1) + │ ├── fd: ()-->(3,4) │ ├── scan p@idx,partial - │ │ ├── columns: f:2 s:3!null rowid:5!null - │ │ ├── key: (5) - │ │ └── fd: ()-->(3), (5)-->(2) + │ │ ├── columns: k:1!null f:3 s:4!null + │ │ ├── key: (1) + │ │ └── fd: ()-->(4), (1)-->(3) │ └── filters - │ └── f:2 = 1.0 [outer=(2), constraints=(/2: [/1.0 - /1.0]; tight), fd=()-->(2)] + │ └── f:3 = 1.0 [outer=(3), constraints=(/3: [/1.0 - /1.0]; tight), fd=()-->(3)] └── filters - └── b:4 [outer=(4), constraints=(/4: [/true - /true]; tight), fd=()-->(4)] + └── b:5 [outer=(5), constraints=(/5: [/true - /true]; tight), fd=()-->(5)] # Generate multiple partial index scans when there are multiple partial indexes # that have predicates implied by the filters. @@ -262,15 +263,15 @@ CREATE INDEX idx2 ON p (s) WHERE i > 0 memo expect=GeneratePartialIndexScans SELECT * FROM p WHERE i > 0 AND s = 'foo' ---- -memo (optimized, ~14KB, required=[presentation: i:1,f:2,s:3,b:4]) - ├── G1: (select G2 G3) (index-join G4 p,cols=(1-4)) (index-join G5 p,cols=(1-4)) (index-join G6 p,cols=(1-4)) (index-join G7 p,cols=(1-4)) - │ └── [presentation: i:1,f:2,s:3,b:4] - │ ├── best: (index-join G4 p,cols=(1-4)) +memo (optimized, ~14KB, required=[presentation: k:1,i:2,f:3,s:4,b:5]) + ├── G1: (select G2 G3) (index-join G4 p,cols=(1-5)) (index-join G5 p,cols=(1-5)) (index-join G6 p,cols=(1-5)) (index-join G7 p,cols=(1-5)) + │ └── [presentation: k:1,i:2,f:3,s:4,b:5] + │ ├── best: (index-join G4 p,cols=(1-5)) │ └── cost: 49.00 - ├── G2: (scan p,cols=(1-4)) + ├── G2: (scan p,cols=(1-5)) │ └── [] - │ ├── best: (scan p,cols=(1-4)) - │ └── cost: 1104.91 + │ ├── best: (scan p,cols=(1-5)) + │ └── cost: 1115.01 ├── G3: (filters G8 G9) ├── G4: (select G10 G11) │ └── [] @@ -280,24 +281,24 @@ memo (optimized, ~14KB, required=[presentation: i:1,f:2,s:3,b:4]) │ └── [] │ ├── best: (select G12 G13) │ └── cost: 354.03 - ├── G6: (scan p@idx,partial,cols=(1-3,5),constrained) + ├── G6: (scan p@idx,partial,cols=(1-4),constrained) │ └── [] - │ ├── best: (scan p@idx,partial,cols=(1-3,5),constrained) + │ ├── best: (scan p@idx,partial,cols=(1-4),constrained) │ └── cost: 14.09 - ├── G7: (scan p@idx2,partial,cols=(3,5),constrained) + ├── G7: (scan p@idx2,partial,cols=(1,4),constrained) │ └── [] - │ ├── best: (scan p@idx2,partial,cols=(3,5),constrained) + │ ├── best: (scan p@idx2,partial,cols=(1,4),constrained) │ └── cost: 13.72 ├── G8: (gt G14 G15) ├── G9: (eq G16 G17) - ├── G10: (scan p@idx,partial,cols=(1-3,5)) + ├── G10: (scan p@idx,partial,cols=(1-4)) │ └── [] - │ ├── best: (scan p@idx,partial,cols=(1-3,5)) + │ ├── best: (scan p@idx,partial,cols=(1-4)) │ └── cost: 14.81 ├── G11: (filters G8) - ├── G12: (scan p@idx2,partial,cols=(3,5)) + ├── G12: (scan p@idx2,partial,cols=(1,4)) │ └── [] - │ ├── best: (scan p@idx2,partial,cols=(3,5)) + │ ├── best: (scan p@idx2,partial,cols=(1,4)) │ └── cost: 350.68 ├── G13: (filters G9) ├── G14: (variable i) @@ -310,9 +311,9 @@ memo (optimized, ~14KB, required=[presentation: i:1,f:2,s:3,b:4]) memo expect-not=GeneratePartialIndexScans SELECT i FROM p WHERE s = 'bar' ---- -memo (optimized, ~8KB, required=[presentation: i:1]) +memo (optimized, ~8KB, required=[presentation: i:2]) ├── G1: (project G2 G3 i) - │ └── [presentation: i:1] + │ └── [presentation: i:2] │ ├── best: (project G2 G3 i) │ └── cost: 1094.84 ├── G2: (select G4 G5) @@ -320,9 +321,9 @@ memo (optimized, ~8KB, required=[presentation: i:1]) │ ├── best: (select G4 G5) │ └── cost: 1094.73 ├── G3: (projections) - ├── G4: (scan p,cols=(1,3)) + ├── G4: (scan p,cols=(2,4)) │ └── [] - │ ├── best: (scan p,cols=(1,3)) + │ ├── best: (scan p,cols=(2,4)) │ └── cost: 1084.71 ├── G5: (filters G6) ├── G6: (eq G7 G8) @@ -337,6 +338,122 @@ exec-ddl DROP INDEX idx2 ---- +exec-ddl +CREATE INDEX idx ON p (b) WHERE s IS NULL AND i = 0 +---- + +# Project constant columns in a partial index predicate rather than performing +# an index join. +opt expect=GeneratePartialIndexScans +SELECT k, b, s, i FROM p WHERE s IS NULL AND i = 0 +---- +project + ├── columns: k:1!null b:5 s:4 i:2!null + ├── key: (1) + ├── fd: ()-->(2,4), (1)-->(5) + ├── scan p@idx,partial + │ ├── columns: k:1!null b:5 + │ ├── key: (1) + │ └── fd: (1)-->(5) + └── projections + ├── 0 [as=i:2] + └── NULL [as=s:4] + +# Do not project constant columns in a partial index predicate when an index +# join must be performed to fetch other columns. +opt expect=GeneratePartialIndexScans +SELECT k, b, s, i, f FROM p WHERE s IS NULL AND i = 0 +---- +index-join p + ├── columns: k:1!null b:5 s:4 i:2!null f:3 + ├── key: (1) + ├── fd: ()-->(2,4), (1)-->(3,5) + └── scan p@idx,partial + ├── columns: k:1!null b:5 + ├── key: (1) + └── fd: (1)-->(5) + +exec-ddl +DROP INDEX idx +---- + +exec-ddl +CREATE INDEX idx ON p (b) STORING (i) WHERE s IS NULL +---- + +# Project constant columns in a partial index predicate and apply a filter. +opt expect=GeneratePartialIndexScans +SELECT k, b, s FROM p WHERE s IS NULL AND i = 0 +---- +project + ├── columns: k:1!null b:5 s:4 + ├── key: (1) + ├── fd: ()-->(4), (1)-->(5) + └── select + ├── columns: k:1!null i:2!null s:4 b:5 + ├── key: (1) + ├── fd: ()-->(2,4), (1)-->(5) + ├── project + │ ├── columns: s:4 k:1!null i:2 b:5 + │ ├── key: (1) + │ ├── fd: ()-->(4), (1)-->(2,5) + │ ├── scan p@idx,partial + │ │ ├── columns: k:1!null i:2 b:5 + │ │ ├── key: (1) + │ │ └── fd: (1)-->(2,5) + │ └── projections + │ └── NULL [as=s:4] + └── filters + └── i:2 = 0 [outer=(2), constraints=(/2: [/0 - /0]; tight), fd=()-->(2)] + +exec-ddl +DROP INDEX idx +---- + +exec-ddl +CREATE INDEX idx ON p (f, s) STORING (b) WHERE s IS NULL AND b AND i = 0 +---- + +# Project only constant columns in the partial index predicate that do not exist +# in the index. +opt expect=GeneratePartialIndexScans +SELECT * FROM p WHERE s IS NULL AND b AND i = 0 +---- +project + ├── columns: k:1!null i:2!null f:3 s:4 b:5!null + ├── key: (1) + ├── fd: ()-->(2,4,5), (1)-->(3) + ├── scan p@idx,partial + │ ├── columns: k:1!null f:3 s:4 b:5!null + │ ├── key: (1) + │ └── fd: ()-->(4,5), (1)-->(3) + └── projections + └── 0 [as=i:2] + +exec-ddl +DROP INDEX idx +---- + +exec-ddl +CREATE INDEX idx ON p (i) WHERE f = 1.0 +---- + +# Do not project constant columns with composite key encodings. +opt expect=GeneratePartialIndexScans +SELECT i, f FROM p WHERE f = 1.0 +---- +index-join p + ├── columns: i:2 f:3!null + ├── fd: ()-->(3) + └── scan p@idx,partial + ├── columns: k:1!null i:2 + ├── key: (1) + └── fd: (1)-->(2) + +exec-ddl +DROP INDEX idx +---- + opt SELECT k FROM virtual WHERE c IN (10, 20, 30) ---- @@ -1459,11 +1576,11 @@ opt SELECT i FROM p WHERE s = 'foo' AND i > 5 AND i < 10 ---- project - ├── columns: i:1!null + ├── columns: i:2!null └── scan p@idx,partial - ├── columns: i:1!null s:3!null - ├── constraint: /1/5: [/6 - /9] - └── fd: ()-->(3) + ├── columns: i:2!null s:4!null + ├── constraint: /2/1: [/6 - /9] + └── fd: ()-->(4) exec-ddl DROP INDEX idx @@ -1479,8 +1596,8 @@ opt SELECT i FROM p WHERE i > 10 AND i < 20 ---- scan p@idx,partial - ├── columns: i:1!null - └── constraint: /1/5: [/11 - /19] + ├── columns: i:2!null + └── constraint: /2/1: [/11 - /19] exec-ddl DROP INDEX idx @@ -1496,13 +1613,14 @@ opt SELECT * FROM p WHERE s = 'foo' AND i > 5 AND i < 10 ---- index-join p - ├── columns: i:1!null f:2 s:3!null b:4 - ├── fd: ()-->(3) + ├── columns: k:1!null i:2!null f:3 s:4!null b:5 + ├── key: (1) + ├── fd: ()-->(4), (1)-->(2,3,5) └── scan p@idx,partial - ├── columns: i:1!null rowid:5!null - ├── constraint: /1/5: [/6 - /9] - ├── key: (5) - └── fd: (5)-->(1) + ├── columns: k:1!null i:2!null + ├── constraint: /2/1: [/6 - /9] + ├── key: (1) + └── fd: (1)-->(2) exec-ddl DROP INDEX idx @@ -1518,19 +1636,19 @@ opt SELECT i FROM p WHERE i > 10 AND i < 20 AND b ---- project - ├── columns: i:1!null + ├── columns: i:2!null └── select - ├── columns: i:1!null b:4!null - ├── fd: ()-->(4) + ├── columns: i:2!null b:5!null + ├── fd: ()-->(5) ├── index-join p - │ ├── columns: i:1 b:4 + │ ├── columns: i:2 b:5 │ └── scan p@idx,partial - │ ├── columns: i:1!null rowid:5!null - │ ├── constraint: /1/5: [/11 - /19] - │ ├── key: (5) - │ └── fd: (5)-->(1) + │ ├── columns: k:1!null i:2!null + │ ├── constraint: /2/1: [/11 - /19] + │ ├── key: (1) + │ └── fd: (1)-->(2) └── filters - └── b:4 [outer=(4), constraints=(/4: [/true - /true]; tight), fd=()-->(4)] + └── b:5 [outer=(5), constraints=(/5: [/true - /true]; tight), fd=()-->(5)] exec-ddl DROP INDEX idx @@ -1547,24 +1665,26 @@ opt SELECT * FROM p WHERE i > 10 AND i < 20 AND s = 'bar' AND b ---- select - ├── columns: i:1!null f:2 s:3!null b:4!null - ├── fd: ()-->(3,4) + ├── columns: k:1!null i:2!null f:3 s:4!null b:5!null + ├── key: (1) + ├── fd: ()-->(4,5), (1)-->(2,3) ├── index-join p - │ ├── columns: i:1 f:2 s:3 b:4 - │ ├── fd: ()-->(3) + │ ├── columns: k:1!null i:2 f:3 s:4 b:5 + │ ├── key: (1) + │ ├── fd: ()-->(4), (1)-->(2,3,5) │ └── select - │ ├── columns: i:1!null s:3!null rowid:5!null - │ ├── key: (5) - │ ├── fd: ()-->(3), (5)-->(1) + │ ├── columns: k:1!null i:2!null s:4!null + │ ├── key: (1) + │ ├── fd: ()-->(4), (1)-->(2) │ ├── scan p@idx,partial - │ │ ├── columns: i:1!null s:3 rowid:5!null - │ │ ├── constraint: /1/5: [/11 - /19] - │ │ ├── key: (5) - │ │ └── fd: (5)-->(1,3) + │ │ ├── columns: k:1!null i:2!null s:4 + │ │ ├── constraint: /2/1: [/11 - /19] + │ │ ├── key: (1) + │ │ └── fd: (1)-->(2,4) │ └── filters - │ └── s:3 = 'bar' [outer=(3), constraints=(/3: [/'bar' - /'bar']; tight), fd=()-->(3)] + │ └── s:4 = 'bar' [outer=(4), constraints=(/4: [/'bar' - /'bar']; tight), fd=()-->(4)] └── filters - └── b:4 [outer=(4), constraints=(/4: [/true - /true]; tight), fd=()-->(4)] + └── b:5 [outer=(5), constraints=(/5: [/true - /true]; tight), fd=()-->(5)] exec-ddl DROP INDEX idx @@ -1584,61 +1704,62 @@ CREATE INDEX idx2 ON p (i) WHERE s = 'foo' memo SELECT i FROM p WHERE i = 3 AND s = 'foo' ---- -memo (optimized, ~18KB, required=[presentation: i:1]) - ├── G1: (project G2 G3 i) (project G4 G3 i) (project G5 G3 i) - │ └── [presentation: i:1] - │ ├── best: (project G5 G3 i) - │ └── cost: 4.98 - ├── G2: (select G6 G7) (select G8 G9) (index-join G4 p,cols=(1,3)) (select G10 G9) (index-join G5 p,cols=(1,3)) +memo (optimized, ~18KB, required=[presentation: i:2]) + ├── G1: (project G2 G3 i) + │ └── [presentation: i:2] + │ ├── best: (project G2 G3 i) + │ └── cost: 5.00 + ├── G2: (select G4 G5) (select G6 G7) (select G8 G7) (select G9 G7) (project G10 G11 i) │ └── [] - │ ├── best: (index-join G5 p,cols=(1,3)) - │ └── cost: 10.50 + │ ├── best: (project G10 G11 i) + │ └── cost: 4.98 ├── G3: (projections) - ├── G4: (select G11 G9) - │ └── [] - │ ├── best: (select G11 G9) - │ └── cost: 14.53 - ├── G5: (scan p@idx2,partial,cols=(1,5),constrained) - │ └── [] - │ ├── best: (scan p@idx2,partial,cols=(1,5),constrained) - │ └── cost: 4.96 - ├── G6: (scan p,cols=(1,3)) + ├── G4: (scan p,cols=(2,4)) │ └── [] - │ ├── best: (scan p,cols=(1,3)) + │ ├── best: (scan p,cols=(2,4)) │ └── cost: 1084.71 - ├── G7: (filters G12 G13) - ├── G8: (index-join G14 p,cols=(1,3)) + ├── G5: (filters G12 G13) + ├── G6: (index-join G14 p,cols=(2,4)) │ └── [] - │ ├── best: (index-join G14 p,cols=(1,3)) + │ ├── best: (index-join G14 p,cols=(2,4)) │ └── cost: 374.63 - ├── G9: (filters G12) - ├── G10: (index-join G15 p,cols=(1,3)) + ├── G7: (filters G12) + ├── G8: (project G15 G11 i) + │ └── [] + │ ├── best: (project G15 G11 i) + │ └── cost: 14.52 + ├── G9: (index-join G16 p,cols=(2,4)) │ └── [] - │ ├── best: (index-join G15 p,cols=(1,3)) + │ ├── best: (index-join G16 p,cols=(2,4)) │ └── cost: 70.38 - ├── G11: (scan p@idx2,partial,cols=(1,5)) + ├── G10: (scan p@idx2,partial,cols=(2),constrained) │ └── [] - │ ├── best: (scan p@idx2,partial,cols=(1,5)) - │ └── cost: 14.41 - ├── G12: (eq G16 G17) - ├── G13: (eq G18 G19) - ├── G14: (select G20 G21) + │ ├── best: (scan p@idx2,partial,cols=(2),constrained) + │ └── cost: 4.95 + ├── G11: (projections G17) + ├── G12: (eq G18 G19) + ├── G13: (eq G20 G17) + ├── G14: (select G21 G22) │ └── [] - │ ├── best: (select G20 G21) + │ ├── best: (select G21 G22) │ └── cost: 354.03 - ├── G15: (scan p@idx,partial,cols=(3,5),constrained) + ├── G15: (scan p@idx2,partial,cols=(2)) │ └── [] - │ ├── best: (scan p@idx,partial,cols=(3,5),constrained) + │ ├── best: (scan p@idx2,partial,cols=(2)) + │ └── cost: 14.31 + ├── G16: (scan p@idx,partial,cols=(1,4),constrained) + │ └── [] + │ ├── best: (scan p@idx,partial,cols=(1,4),constrained) │ └── cost: 13.72 - ├── G16: (variable i) - ├── G17: (const 3) - ├── G18: (variable s) - ├── G19: (const 'foo') - ├── G20: (scan p@idx,partial,cols=(3,5)) + ├── G17: (const 'foo') + ├── G18: (variable i) + ├── G19: (const 3) + ├── G20: (variable s) + ├── G21: (scan p@idx,partial,cols=(1,4)) │ └── [] - │ ├── best: (scan p@idx,partial,cols=(3,5)) + │ ├── best: (scan p@idx,partial,cols=(1,4)) │ └── cost: 350.68 - └── G21: (filters G13) + └── G22: (filters G13) exec-ddl DROP INDEX idx @@ -1657,9 +1778,9 @@ CREATE INDEX idx ON p (i) WHERE s = 'foo' memo expect-not=GenerateConstrainedScans SELECT i FROM p WHERE s = 'bar' AND i = 5 ---- -memo (optimized, ~7KB, required=[presentation: i:1]) +memo (optimized, ~7KB, required=[presentation: i:2]) ├── G1: (project G2 G3 i) - │ └── [presentation: i:1] + │ └── [presentation: i:2] │ ├── best: (project G2 G3 i) │ └── cost: 1094.76 ├── G2: (select G4 G5) @@ -1667,9 +1788,9 @@ memo (optimized, ~7KB, required=[presentation: i:1]) │ ├── best: (select G4 G5) │ └── cost: 1094.74 ├── G3: (projections) - ├── G4: (scan p,cols=(1,3)) + ├── G4: (scan p,cols=(2,4)) │ └── [] - │ ├── best: (scan p,cols=(1,3)) + │ ├── best: (scan p,cols=(2,4)) │ └── cost: 1084.71 ├── G5: (filters G6 G7) ├── G6: (eq G8 G9) @@ -1683,6 +1804,127 @@ exec-ddl DROP INDEX idx ---- +exec-ddl +CREATE INDEX idx ON p (b) WHERE s IS NULL AND i = 0 +---- + +# Project constant columns in a partial index predicate rather than performing +# an index join. +opt expect=GenerateConstrainedScans +SELECT k, b, s, i FROM p WHERE b AND s IS NULL AND i = 0 +---- +project + ├── columns: k:1!null b:5!null s:4 i:2!null + ├── key: (1) + ├── fd: ()-->(2,4,5) + ├── scan p@idx,partial + │ ├── columns: k:1!null b:5!null + │ ├── constraint: /5/1: [/true - /true] + │ ├── key: (1) + │ └── fd: ()-->(5) + └── projections + ├── 0 [as=i:2] + └── NULL [as=s:4] + +# Do not project constant columns in a partial index predicate when an index +# join must be performed to fetch other columns. +opt expect=GenerateConstrainedScans +SELECT k, b, s, i, f FROM p WHERE b AND s IS NULL AND i = 0 +---- +index-join p + ├── columns: k:1!null b:5!null s:4 i:2!null f:3 + ├── key: (1) + ├── fd: ()-->(2,4,5), (1)-->(3) + └── scan p@idx,partial + ├── columns: k:1!null b:5!null + ├── constraint: /5/1: [/true - /true] + ├── key: (1) + └── fd: ()-->(5) + +exec-ddl +DROP INDEX idx +---- + +exec-ddl +CREATE INDEX idx ON p (b) STORING (i) WHERE s IS NULL +---- + +# Project constant columns in a partial index predicate and apply a filter. +opt expect=GenerateConstrainedScans +SELECT k, b, s FROM p WHERE b AND s IS NULL AND i = 0 +---- +project + ├── columns: k:1!null b:5!null s:4 + ├── key: (1) + ├── fd: ()-->(4,5) + └── select + ├── columns: k:1!null i:2!null s:4 b:5!null + ├── key: (1) + ├── fd: ()-->(2,4,5) + ├── project + │ ├── columns: s:4 k:1!null i:2 b:5!null + │ ├── key: (1) + │ ├── fd: ()-->(4,5), (1)-->(2) + │ ├── scan p@idx,partial + │ │ ├── columns: k:1!null i:2 b:5!null + │ │ ├── constraint: /5/1: [/true - /true] + │ │ ├── key: (1) + │ │ └── fd: ()-->(5), (1)-->(2) + │ └── projections + │ └── NULL [as=s:4] + └── filters + └── i:2 = 0 [outer=(2), constraints=(/2: [/0 - /0]; tight), fd=()-->(2)] + +exec-ddl +DROP INDEX idx +---- + +exec-ddl +CREATE INDEX idx ON p (f, s) STORING (b) WHERE s IS NULL AND b AND i = 0 +---- + +# Project only constant columns in the partial index predicate that do not exist +# in the index. +opt expect=GenerateConstrainedScans +SELECT * FROM p WHERE f > 1.0 AND s IS NULL AND b AND i = 0 +---- +project + ├── columns: k:1!null i:2!null f:3!null s:4 b:5!null + ├── key: (1) + ├── fd: ()-->(2,4,5), (1)-->(3) + ├── scan p@idx,partial + │ ├── columns: k:1!null f:3!null s:4 b:5!null + │ ├── constraint: /3/4/1: [/1.0000000000000002 - ] + │ ├── key: (1) + │ └── fd: ()-->(4,5), (1)-->(3) + └── projections + └── 0 [as=i:2] + +exec-ddl +DROP INDEX idx +---- + +exec-ddl +CREATE INDEX idx ON p (i) WHERE f = 1.0 +---- + +# Do not project constant columns with composite key encodings. +opt expect=GenerateConstrainedScans +SELECT i, f FROM p WHERE i = 0 AND f = 1.0 +---- +index-join p + ├── columns: i:2!null f:3!null + ├── fd: ()-->(2,3) + └── scan p@idx,partial + ├── columns: k:1!null i:2!null + ├── constraint: /2/1: [/0 - /0] + ├── key: (1) + └── fd: ()-->(2) + +exec-ddl +DROP INDEX idx +---- + # Constrained partial index scan with a virtual computed column in the # predicate. opt @@ -5098,12 +5340,22 @@ CREATE INDEX j ON zz_partial (j) WHERE s = 'foo' # Don't generate a zigzag join when the expression that fixes the left columns # is removed during partial index implication of the right index. +# TODO(mgartner): Once again, we find ourselves with a suboptimal query plan +# with an unnecessary Project that projects s='foo' which is not needed by the +# parent expression. This is the result of GenerateConstrainedScans replacing +# the matched Select with a Project that is required to produce all the columns +# that the Select produced. One way to fix this would be to add a new +# exploration rule that can remove these unnecessary Projects, similar to +# EliminateIndexJoinInsideProject. opt expect-not=GenerateZigzagJoins format=hide-all SELECT k FROM zz_partial WHERE s = 'foo' AND j = 20 ---- project - └── scan zz_partial@j,partial - └── constraint: /3/1: [/20 - /20] + └── project + ├── scan zz_partial@j,partial + │ └── constraint: /3/1: [/20 - /20] + └── projections + └── 'foo' exec-ddl DROP INDEX zz_partial_s @@ -5127,8 +5379,11 @@ opt expect-not=GenerateZigzagJoins format=hide-all SELECT k FROM zz_partial WHERE i = 10 AND s = 'foo' ---- project - └── scan zz_partial@i,partial - └── constraint: /2/1: [/10 - /10] + └── project + ├── scan zz_partial@i,partial + │ └── constraint: /2/1: [/10 - /10] + └── projections + └── 'foo' exec-ddl DROP INDEX i @@ -8033,91 +8288,91 @@ CREATE INDEX idx_f ON p (f) WHERE s IN ('foo', 'bar', 'baz') # Apply when one side of the disjunction can be "constrained" by an # unconstrained partial index scan with no remaining filters. opt expect=SplitDisjunctionAddKey -SELECT * FROM p WHERE i = 10 OR s IN ('foo', 'bar', 'baz') +SELECT i, f, s, b FROM p WHERE i = 10 OR s IN ('foo', 'bar', 'baz') ---- project - ├── columns: i:1 f:2 s:3 b:4 + ├── columns: i:2 f:3 s:4 b:5 └── distinct-on - ├── columns: i:1 f:2 s:3 b:4 rowid:5!null - ├── grouping columns: rowid:5!null - ├── key: (5) - ├── fd: (5)-->(1-4) + ├── columns: k:1!null i:2 f:3 s:4 b:5 + ├── grouping columns: k:1!null + ├── key: (1) + ├── fd: (1)-->(2-5) ├── union-all - │ ├── columns: i:1 f:2 s:3 b:4 rowid:5!null - │ ├── left columns: i:7 f:8 s:9 b:10 rowid:11 - │ ├── right columns: i:13 f:14 s:15 b:16 rowid:17 + │ ├── columns: k:1!null i:2 f:3 s:4 b:5 + │ ├── left columns: k:7 i:8 f:9 s:10 b:11 + │ ├── right columns: k:13 i:14 f:15 s:16 b:17 │ ├── index-join p - │ │ ├── columns: i:7!null f:8 s:9 b:10 rowid:11!null - │ │ ├── key: (11) - │ │ ├── fd: ()-->(7), (11)-->(8-10) + │ │ ├── columns: k:7!null i:8!null f:9 s:10 b:11 + │ │ ├── key: (7) + │ │ ├── fd: ()-->(8), (7)-->(9-11) │ │ └── scan p@idx_i - │ │ ├── columns: i:7!null rowid:11!null - │ │ ├── constraint: /7/11: [/10 - /10] - │ │ ├── key: (11) - │ │ └── fd: ()-->(7) + │ │ ├── columns: k:7!null i:8!null + │ │ ├── constraint: /8/7: [/10 - /10] + │ │ ├── key: (7) + │ │ └── fd: ()-->(8) │ └── index-join p - │ ├── columns: i:13 f:14 s:15!null b:16 rowid:17!null - │ ├── key: (17) - │ ├── fd: (17)-->(13-16) + │ ├── columns: k:13!null i:14 f:15 s:16!null b:17 + │ ├── key: (13) + │ ├── fd: (13)-->(14-17) │ └── scan p@idx_f,partial - │ ├── columns: f:14 rowid:17!null - │ ├── key: (17) - │ └── fd: (17)-->(14) + │ ├── columns: k:13!null f:15 + │ ├── key: (13) + │ └── fd: (13)-->(15) └── aggregations - ├── const-agg [as=i:1, outer=(1)] - │ └── i:1 - ├── const-agg [as=f:2, outer=(2)] - │ └── f:2 - ├── const-agg [as=s:3, outer=(3)] - │ └── s:3 - └── const-agg [as=b:4, outer=(4)] - └── b:4 + ├── const-agg [as=i:2, outer=(2)] + │ └── i:2 + ├── const-agg [as=f:3, outer=(3)] + │ └── f:3 + ├── const-agg [as=s:4, outer=(4)] + │ └── s:4 + └── const-agg [as=b:5, outer=(5)] + └── b:5 # Apply when one side of the disjunction can be "constrained" by an # unconstrained partial index scan with remaining filters. opt expect=SplitDisjunctionAddKey -SELECT * FROM p WHERE i = 10 OR s = 'foo' +SELECT i, s, f, b FROM p WHERE i = 10 OR s = 'foo' ---- project - ├── columns: i:1 f:2 s:3 b:4 + ├── columns: i:2 s:4 f:3 b:5 └── distinct-on - ├── columns: i:1 f:2 s:3 b:4 rowid:5!null - ├── grouping columns: rowid:5!null - ├── key: (5) - ├── fd: (5)-->(1-4) + ├── columns: k:1!null i:2 f:3 s:4 b:5 + ├── grouping columns: k:1!null + ├── key: (1) + ├── fd: (1)-->(2-5) ├── union-all - │ ├── columns: i:1 f:2 s:3 b:4 rowid:5!null - │ ├── left columns: i:7 f:8 s:9 b:10 rowid:11 - │ ├── right columns: i:13 f:14 s:15 b:16 rowid:17 + │ ├── columns: k:1!null i:2 f:3 s:4 b:5 + │ ├── left columns: k:7 i:8 f:9 s:10 b:11 + │ ├── right columns: k:13 i:14 f:15 s:16 b:17 │ ├── index-join p - │ │ ├── columns: i:7!null f:8 s:9 b:10 rowid:11!null - │ │ ├── key: (11) - │ │ ├── fd: ()-->(7), (11)-->(8-10) + │ │ ├── columns: k:7!null i:8!null f:9 s:10 b:11 + │ │ ├── key: (7) + │ │ ├── fd: ()-->(8), (7)-->(9-11) │ │ └── scan p@idx_i - │ │ ├── columns: i:7!null rowid:11!null - │ │ ├── constraint: /7/11: [/10 - /10] - │ │ ├── key: (11) - │ │ └── fd: ()-->(7) + │ │ ├── columns: k:7!null i:8!null + │ │ ├── constraint: /8/7: [/10 - /10] + │ │ ├── key: (7) + │ │ └── fd: ()-->(8) │ └── select - │ ├── columns: i:13 f:14 s:15!null b:16 rowid:17!null - │ ├── key: (17) - │ ├── fd: ()-->(15), (17)-->(13,14,16) + │ ├── columns: k:13!null i:14 f:15 s:16!null b:17 + │ ├── key: (13) + │ ├── fd: ()-->(16), (13)-->(14,15,17) │ ├── index-join p - │ │ ├── columns: i:13 f:14 s:15 b:16 rowid:17!null - │ │ ├── key: (17) - │ │ ├── fd: (17)-->(13-16) + │ │ ├── columns: k:13!null i:14 f:15 s:16 b:17 + │ │ ├── key: (13) + │ │ ├── fd: (13)-->(14-17) │ │ └── scan p@idx_f,partial - │ │ ├── columns: f:14 rowid:17!null - │ │ ├── key: (17) - │ │ └── fd: (17)-->(14) + │ │ ├── columns: k:13!null f:15 + │ │ ├── key: (13) + │ │ └── fd: (13)-->(15) │ └── filters - │ └── s:15 = 'foo' [outer=(15), constraints=(/15: [/'foo' - /'foo']; tight), fd=()-->(15)] + │ └── s:16 = 'foo' [outer=(16), constraints=(/16: [/'foo' - /'foo']; tight), fd=()-->(16)] └── aggregations - ├── const-agg [as=i:1, outer=(1)] - │ └── i:1 - ├── const-agg [as=f:2, outer=(2)] - │ └── f:2 - ├── const-agg [as=s:3, outer=(3)] - │ └── s:3 - └── const-agg [as=b:4, outer=(4)] - └── b:4 + ├── const-agg [as=i:2, outer=(2)] + │ └── i:2 + ├── const-agg [as=f:3, outer=(3)] + │ └── f:3 + ├── const-agg [as=s:4, outer=(4)] + │ └── s:4 + └── const-agg [as=b:5, outer=(5)] + └── b:5