From 2b6f52b326e4f26d82c6fd26ba58c1b2368adb52 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 28 Dec 2020 10:29:10 -0800 Subject: [PATCH 1/6] opt: do not derive prune columns for Upsert, Update, Delete We no longer derive output prune columns for Upsert, Update, and Delete ops in `DerivePruneCols`. There are no PruneCols rules for these operators, so deriving their prune columns was only performing unnecessary work. There are other rules that prune the fetch and return columns for these operators. These rules do not rely on `DerivePruneCols`. Release note: None --- pkg/sql/opt/memo/testdata/logprops/delete | 3 --- pkg/sql/opt/norm/prune_cols_funcs.go | 22 +--------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/pkg/sql/opt/memo/testdata/logprops/delete b/pkg/sql/opt/memo/testdata/logprops/delete index a499a2b85e91..013da35d6772 100644 --- a/pkg/sql/opt/memo/testdata/logprops/delete +++ b/pkg/sql/opt/memo/testdata/logprops/delete @@ -64,7 +64,6 @@ project ├── volatile, mutations ├── key: (5) ├── fd: ()-->(1), (5)-->(2-4) - ├── prune: (1-4) └── select ├── columns: a:8(int!null) b:9(int) c:10(int!null) d:11(int) rowid:12(int!null) e:13(int) crdb_internal_mvcc_timestamp:14(decimal) ├── key: (12) @@ -107,7 +106,6 @@ project ├── volatile, mutations ├── key: () ├── fd: ()-->(1-5) - ├── prune: (1-4) └── select ├── columns: a:8(int!null) b:9(int) c:10(int!null) d:11(int) rowid:12(int!null) e:13(int) crdb_internal_mvcc_timestamp:14(decimal) ├── cardinality: [0 - 1] @@ -148,7 +146,6 @@ project ├── volatile, mutations ├── key: (5) ├── fd: (2)==(3), (3)==(2), (5)-->(1-4) - ├── prune: (1-4) └── select ├── columns: a:8(int!null) b:9(int!null) c:10(int!null) d:11(int) rowid:12(int!null) e:13(int) crdb_internal_mvcc_timestamp:14(decimal) ├── key: (12) diff --git a/pkg/sql/opt/norm/prune_cols_funcs.go b/pkg/sql/opt/norm/prune_cols_funcs.go index ca0cf56b424b..831f982ba586 100644 --- a/pkg/sql/opt/norm/prune_cols_funcs.go +++ b/pkg/sql/opt/norm/prune_cols_funcs.go @@ -89,17 +89,8 @@ func (c *CustomFuncs) NeededMutationCols( func (c *CustomFuncs) NeededMutationFetchCols( op opt.Operator, private *memo.MutationPrivate, ) opt.ColSet { - return neededMutationFetchCols(c.mem, op, private) -} - -// neededMutationFetchCols returns the set of columns needed by the given -// mutation operator. -func neededMutationFetchCols( - mem *memo.Memo, op opt.Operator, private *memo.MutationPrivate, -) opt.ColSet { - var cols opt.ColSet - tabMeta := mem.Metadata().TableMeta(private.Table) + tabMeta := c.mem.Metadata().TableMeta(private.Table) // familyCols returns the columns in the given family. familyCols := func(fam cat.Family) opt.ColSet { @@ -579,17 +570,6 @@ func DerivePruneCols(e memo.RelExpr) opt.ColSet { relProps.Rule.PruneCols.DifferenceWith(w.ScalarProps().OuterCols) } - case opt.UpdateOp, opt.UpsertOp, opt.DeleteOp: - // Find the columns that would need to be fetched, if no returning - // clause were present. - withoutReturningPrivate := *e.Private().(*memo.MutationPrivate) - withoutReturningPrivate.ReturnCols = opt.OptionalColList{} - neededCols := neededMutationFetchCols(e.Memo(), e.Op(), &withoutReturningPrivate) - - // Only the "free" RETURNING columns can be pruned away (i.e. the columns - // required by the mutation only because they're being returned). - relProps.Rule.PruneCols = relProps.OutputCols.Difference(neededCols) - case opt.WithOp: // WithOp passes through its input unchanged, so it has the same pruning // characteristics as its input. From 07e93642b95bcdcb7af4fc5442fd5b936e8699b5 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 28 Dec 2020 15:26:10 -0800 Subject: [PATCH 2/6] sql: remove logic to determine fetch cols in row updater Previously, the `row.MakeUpdater` function had logic to determine the fetch columns required for an update operation. This is not necessary because the cost based optimizer already determines the necessary fetch columns and plumbs them to `MakeUpdater` as the `requestedCols` argument. Release note: None --- pkg/sql/backfill/backfill.go | 3 +- pkg/sql/row/updater.go | 67 ++---------------------------------- 2 files changed, 5 insertions(+), 65 deletions(-) diff --git a/pkg/sql/backfill/backfill.go b/pkg/sql/backfill/backfill.go index 6038384b545c..030cab6a678a 100644 --- a/pkg/sql/backfill/backfill.go +++ b/pkg/sql/backfill/backfill.go @@ -253,9 +253,10 @@ func (cb *ColumnBackfiller) RunColumnBackfillChunk( ) (roachpb.Key, error) { // TODO(dan): Tighten up the bound on the requestedCols parameter to // makeRowUpdater. - requestedCols := make([]descpb.ColumnDescriptor, 0, len(tableDesc.Columns)+len(cb.added)) + requestedCols := make([]descpb.ColumnDescriptor, 0, len(tableDesc.Columns)+len(cb.added)+len(cb.dropped)) requestedCols = append(requestedCols, tableDesc.Columns...) requestedCols = append(requestedCols, cb.added...) + requestedCols = append(requestedCols, cb.dropped...) ru, err := row.MakeUpdater( ctx, txn, diff --git a/pkg/sql/row/updater.go b/pkg/sql/row/updater.go index 065480a37ed3..ebb451dce3d9 100644 --- a/pkg/sql/row/updater.go +++ b/pkg/sql/row/updater.go @@ -149,9 +149,6 @@ func MakeUpdater( } } - // Columns of the table to update, including those in delete/write-only state - tableCols := tableDesc.DeletableColumns() - var deleteOnlyIndexes []descpb.IndexDescriptor for _, idx := range tableDesc.DeleteOnlyIndexes() { if needsUpdate(idx) { @@ -172,6 +169,8 @@ func MakeUpdater( ru := Updater{ Helper: newRowHelper(codec, tableDesc, includeIndexes), DeleteHelper: deleteOnlyHelper, + FetchCols: requestedCols, + FetchColIDtoRowIndex: ColIDtoRowIndexFromCols(requestedCols), UpdateCols: updateCols, UpdateColIDtoRowIndex: updateColIDtoRowIndex, primaryKeyColChange: primaryKeyColChange, @@ -185,73 +184,13 @@ func MakeUpdater( // When changing the primary key, we delete the old values and reinsert // them, so request them all. var err error + tableCols := tableDesc.DeletableColumns() ru.rd = MakeDeleter(codec, tableDesc, tableCols) - ru.FetchCols = ru.rd.FetchCols - ru.FetchColIDtoRowIndex = ColIDtoRowIndexFromCols(ru.FetchCols) if ru.ri, err = MakeInserter( ctx, txn, codec, tableDesc, tableCols, alloc, ); err != nil { return Updater{}, err } - } else { - ru.FetchCols = requestedCols[:len(requestedCols):len(requestedCols)] - ru.FetchColIDtoRowIndex = ColIDtoRowIndexFromCols(ru.FetchCols) - - // maybeAddCol adds the provided column to ru.FetchCols and - // ru.FetchColIDtoRowIndex if it isn't already present. - maybeAddCol := func(colID descpb.ColumnID) error { - if _, ok := ru.FetchColIDtoRowIndex.Get(colID); !ok { - col, _, err := tableDesc.FindReadableColumnByID(colID) - if err != nil { - return err - } - ru.FetchColIDtoRowIndex.Set(col.ID, len(ru.FetchCols)) - ru.FetchCols = append(ru.FetchCols, *col) - } - return nil - } - - // Fetch all columns in the primary key so that we can construct the - // keys when writing out the new kvs to the primary index. - for _, colID := range tableDesc.GetPrimaryIndex().ColumnIDs { - if err := maybeAddCol(colID); err != nil { - return Updater{}, err - } - } - - // If any part of a column family is being updated, fetch all columns in - // that column family so that we can reconstruct the column family with - // the updated columns before writing it. - for i := range tableDesc.Families { - family := &tableDesc.Families[i] - familyBeingUpdated := false - for _, colID := range family.ColumnIDs { - if _, ok := ru.UpdateColIDtoRowIndex.Get(colID); ok { - familyBeingUpdated = true - break - } - } - if familyBeingUpdated { - for _, colID := range family.ColumnIDs { - if err := maybeAddCol(colID); err != nil { - return Updater{}, err - } - } - } - } - - // Fetch all columns from indices that are being update so that they can - // be used to create the new kv pairs for those indices. - for _, index := range includeIndexes { - if err := index.RunOverAllColumns(maybeAddCol); err != nil { - return Updater{}, err - } - } - for _, index := range deleteOnlyIndexes { - if err := index.RunOverAllColumns(maybeAddCol); err != nil { - return Updater{}, err - } - } } // If we are fetching from specific families, we might get From c5e72abb2755dcb7a3556cc5743438540e303826 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Tue, 29 Dec 2020 16:57:43 -0800 Subject: [PATCH 3/6] opt: safer access to partial index predicates in TableMeta Previously, partial index predicate expressions in TableMeta were the source-of-truth used within the optimizer to determine if an index is a partial index. However, partial index predicates are not added to TableMeta for all types of statements in optbuilder. Therefore, it was not safe to assume this was a source-of-truth. This commit unexports the map of partial index predicates in TableMeta. Access to partial index predicates must now be done via `TableMeta.PartialIndexPredicate`. This function checks the catalog to determine if an index is a partial index, and panics if there is not a corresponding predicate expression in the partial index predicate map. This makes the function an actual a source-of-truth. Release note: None --- pkg/sql/opt/memo/expr.go | 2 +- pkg/sql/opt/memo/expr_format.go | 9 ++++--- pkg/sql/opt/metadata.go | 10 +++---- pkg/sql/opt/metadata_test.go | 7 ++--- pkg/sql/opt/norm/reject_nulls_funcs.go | 8 +++--- pkg/sql/opt/optbuilder/partial_index.go | 2 +- pkg/sql/opt/table_meta.go | 36 +++++++++++++++++++++---- pkg/sql/opt/xform/scan_index_iter.go | 2 +- pkg/sql/opt/xform/select_funcs.go | 2 +- 9 files changed, 54 insertions(+), 24 deletions(-) diff --git a/pkg/sql/opt/memo/expr.go b/pkg/sql/opt/memo/expr.go index d91af9257fae..5439b4fdbe39 100644 --- a/pkg/sql/opt/memo/expr.go +++ b/pkg/sql/opt/memo/expr.go @@ -658,7 +658,7 @@ func (s *ScanPrivate) IsLocking() bool { // index, nil is returned. func (s *ScanPrivate) PartialIndexPredicate(md *opt.Metadata) FiltersExpr { tabMeta := md.TableMeta(s.Table) - p, ok := tabMeta.PartialIndexPredicates[s.Index] + p, ok := tabMeta.PartialIndexPredicate(s.Index) if !ok { // The index is not a partial index. return nil diff --git a/pkg/sql/opt/memo/expr_format.go b/pkg/sql/opt/memo/expr_format.go index 38c8ac33ea1d..abcc749f2343 100644 --- a/pkg/sql/opt/memo/expr_format.go +++ b/pkg/sql/opt/memo/expr_format.go @@ -352,17 +352,18 @@ func (f *ExprFmtCtx) formatRelational(e RelExpr, tp treeprinter.Node) { f.formatExpr(tab.ComputedCols[col], c.Child(f.ColumnString(col))) } } - if tab.PartialIndexPredicates != nil { + partialIndexPredicates := tab.PartialIndexPredicatesForFormattingOnly() + if partialIndexPredicates != nil { c := tp.Child("partial index predicates") - indexOrds := make([]cat.IndexOrdinal, 0, len(tab.PartialIndexPredicates)) - for ord := range tab.PartialIndexPredicates { + indexOrds := make([]cat.IndexOrdinal, 0, len(partialIndexPredicates)) + for ord := range partialIndexPredicates { indexOrds = append(indexOrds, ord) } sort.Ints(indexOrds) for _, ord := range indexOrds { name := string(tab.Table.Index(ord).Name()) f.Buffer.Reset() - f.formatScalarWithLabel(name, tab.PartialIndexPredicates[ord], c) + f.formatScalarWithLabel(name, partialIndexPredicates[ord], c) } } } diff --git a/pkg/sql/opt/metadata.go b/pkg/sql/opt/metadata.go index 6a4ef3af62bd..51c644c37053 100644 --- a/pkg/sql/opt/metadata.go +++ b/pkg/sql/opt/metadata.go @@ -396,7 +396,7 @@ func (md *Metadata) AddTable(tab cat.Table, alias *tree.TableName) TableID { // ScalarExpr to new column IDs. It takes as arguments a ScalarExpr and a // mapping of old column IDs to new column IDs, and returns a new ScalarExpr. // This function is used when duplicating Constraints, ComputedCols, and -// PartialIndexPredicates. DuplicateTable requires this callback function, +// partialIndexPredicates. DuplicateTable requires this callback function, // rather than performing the remapping itself, because remapping column IDs // requires constructing new expressions with norm.Factory. The norm package // depends on opt, and cannot be imported here. @@ -450,9 +450,9 @@ func (md *Metadata) DuplicateTable( // Create new partial index predicate expressions by remapping the column // IDs in each ScalarExpr. var partialIndexPredicates map[cat.IndexOrdinal]ScalarExpr - if len(tabMeta.PartialIndexPredicates) > 0 { - partialIndexPredicates = make(map[cat.IndexOrdinal]ScalarExpr, len(tabMeta.PartialIndexPredicates)) - for idxOrd, e := range tabMeta.PartialIndexPredicates { + if len(tabMeta.partialIndexPredicates) > 0 { + partialIndexPredicates = make(map[cat.IndexOrdinal]ScalarExpr, len(tabMeta.partialIndexPredicates)) + for idxOrd, e := range tabMeta.partialIndexPredicates { partialIndexPredicates[idxOrd] = remapColumnIDs(e, colMap) } } @@ -464,7 +464,7 @@ func (md *Metadata) DuplicateTable( IgnoreForeignKeys: tabMeta.IgnoreForeignKeys, Constraints: constraints, ComputedCols: computedCols, - PartialIndexPredicates: partialIndexPredicates, + partialIndexPredicates: partialIndexPredicates, }) return newTabID diff --git a/pkg/sql/opt/metadata_test.go b/pkg/sql/opt/metadata_test.go index 6f98e25eff39..c1e4c3b45390 100644 --- a/pkg/sql/opt/metadata_test.go +++ b/pkg/sql/opt/metadata_test.go @@ -281,7 +281,7 @@ func TestIndexColumns(t *testing.T) { // TestDuplicateTable tests that we can extract a set of columns from an index ordinal. func TestDuplicateTable(t *testing.T) { cat := testcat.New() - _, err := cat.ExecuteDDL("CREATE TABLE a (b BOOL, b2 BOOL)") + _, err := cat.ExecuteDDL("CREATE TABLE a (b BOOL, b2 BOOL, INDEX (b2) WHERE b)") if err != nil { t.Fatal(err) } @@ -332,11 +332,12 @@ func TestDuplicateTable(t *testing.T) { t.Errorf("expected computed column to reference new column ID %d, got %d", dupB, col) } - if dupTabMeta.PartialIndexPredicates == nil || dupTabMeta.PartialIndexPredicates[1] == nil { + pred, isPartialIndex := dupTabMeta.PartialIndexPredicate(1) + if !isPartialIndex { t.Fatalf("expected partial index predicates to be duplicated") } - col = dupTabMeta.PartialIndexPredicates[1].(*memo.VariableExpr).Col + col = pred.(*memo.VariableExpr).Col if col == b { t.Errorf("expected partial index predicate to reference new column ID %d, got %d", dupB, col) } diff --git a/pkg/sql/opt/norm/reject_nulls_funcs.go b/pkg/sql/opt/norm/reject_nulls_funcs.go index 8ce83632baf1..b75eea8de5d9 100644 --- a/pkg/sql/opt/norm/reject_nulls_funcs.go +++ b/pkg/sql/opt/norm/reject_nulls_funcs.go @@ -331,9 +331,11 @@ func deriveScanRejectNullCols(in memo.RelExpr) opt.ColSet { scan := in.(*memo.ScanExpr) var rejectNullCols opt.ColSet - for _, pred := range md.TableMeta(scan.Table).PartialIndexPredicates { - predFilters := *pred.(*memo.FiltersExpr) - rejectNullCols.UnionWith(isNotNullCols(predFilters)) + for i, n := 0, md.Table(scan.Table).IndexCount(); i < n; i++ { + if pred, isPartialIndex := md.TableMeta(scan.Table).PartialIndexPredicate(i); isPartialIndex { + predFilters := *pred.(*memo.FiltersExpr) + rejectNullCols.UnionWith(isNotNullCols(predFilters)) + } } return rejectNullCols diff --git a/pkg/sql/opt/optbuilder/partial_index.go b/pkg/sql/opt/optbuilder/partial_index.go index 9e78103943fb..f2f99f9b5bac 100644 --- a/pkg/sql/opt/optbuilder/partial_index.go +++ b/pkg/sql/opt/optbuilder/partial_index.go @@ -25,7 +25,7 @@ import ( // Note: This function should only be used to build partial index or arbiter // predicate expressions that have only a table's ordinary columns in scope and // that are not part of the relational expression tree. For example, this is -// used to populate the TableMeta.PartialIndexPredicates cache and for +// used to populate the partial index predicates map in TableMeta and for // determining arbiter indexes in UPSERT and INSERT ON CONFLICT mutations. But // it is not used for building synthesized mutation columns that determine // whether to issue PUT or DEL operations on a partial index for a mutated row; diff --git a/pkg/sql/opt/table_meta.go b/pkg/sql/opt/table_meta.go index 35ada6385979..c9c67b92f768 100644 --- a/pkg/sql/opt/table_meta.go +++ b/pkg/sql/opt/table_meta.go @@ -156,11 +156,11 @@ type TableMeta struct { // Computed columns with non-immutable operators are omitted. ComputedCols map[ColumnID]ScalarExpr - // PartialIndexPredicates is a map from index ordinals on the table to + // partialIndexPredicates is a map from index ordinals on the table to // *FiltersExprs representing the predicate on the corresponding partial // index. If an index is not a partial index, it will not have an entry in // the map. - PartialIndexPredicates map[cat.IndexOrdinal]ScalarExpr + partialIndexPredicates map[cat.IndexOrdinal]ScalarExpr // anns annotates the table metadata with arbitrary data. anns [maxTableAnnIDCount]interface{} @@ -254,10 +254,36 @@ func (tm *TableMeta) AddComputedCol(colID ColumnID, computedCol ScalarExpr) { // AddPartialIndexPredicate adds a partial index predicate to the table's // metadata. func (tm *TableMeta) AddPartialIndexPredicate(ord cat.IndexOrdinal, pred ScalarExpr) { - if tm.PartialIndexPredicates == nil { - tm.PartialIndexPredicates = make(map[cat.IndexOrdinal]ScalarExpr) + if tm.partialIndexPredicates == nil { + tm.partialIndexPredicates = make(map[cat.IndexOrdinal]ScalarExpr) } - tm.PartialIndexPredicates[ord] = pred + tm.partialIndexPredicates[ord] = pred +} + +// PartialIndexPredicate returns the given index's predicate scalar expression, +// if the index is a partial index. Returns ok=false if the index is not a +// partial index. Panics if the index is a partial index but a predicate scalar +// expression does not exist in the table metadata. +func (tm *TableMeta) PartialIndexPredicate(ord cat.IndexOrdinal) (pred ScalarExpr, ok bool) { + if _, isPartialIndex := tm.Table.Index(ord).Predicate(); !isPartialIndex { + return nil, false + } + pred, ok = tm.partialIndexPredicates[ord] + if !ok { + panic(errors.AssertionFailedf("partial index predicate does not exist in table metadata")) + } + return pred, true +} + +// PartialIndexPredicatesForFormattingOnly returns the partialIndexPredicates +// map. +// +// WARNING: The returned map is NOT a source-of-truth for determining if an +// index is a partial index. This function should only be used to show the +// partial index expressions that have been built for a table when formatting +// opt expressions. Use PartialIndexPredicate in all other cases. +func (tm *TableMeta) PartialIndexPredicatesForFormattingOnly() map[cat.IndexOrdinal]ScalarExpr { + return tm.partialIndexPredicates } // TableAnnotation returns the given annotation that is associated with the diff --git a/pkg/sql/opt/xform/scan_index_iter.go b/pkg/sql/opt/xform/scan_index_iter.go index 99aef8acde30..c694369c1aee 100644 --- a/pkg/sql/opt/xform/scan_index_iter.go +++ b/pkg/sql/opt/xform/scan_index_iter.go @@ -213,7 +213,7 @@ func (it *scanIndexIter) ForEachStartingAfter(ord int, f enumerateIndexFunc) { continue } - pred, isPartialIndex := it.tabMeta.PartialIndexPredicates[ord] + pred, isPartialIndex := it.tabMeta.PartialIndexPredicate(ord) // Skip over partial indexes if rejectPartialIndexes is set. if it.hasRejectFlags(rejectPartialIndexes) && isPartialIndex { diff --git a/pkg/sql/opt/xform/select_funcs.go b/pkg/sql/opt/xform/select_funcs.go index 3a7eea0e7495..69c196d9ef9d 100644 --- a/pkg/sql/opt/xform/select_funcs.go +++ b/pkg/sql/opt/xform/select_funcs.go @@ -1655,7 +1655,7 @@ func (c *CustomFuncs) canMaybeConstrainIndexWithCols( // possible to generate an unconstrained partial index scan, which may // lead to better query plans. if _, isPartialIndex := index.Predicate(); isPartialIndex { - p, ok := tabMeta.PartialIndexPredicates[i] + p, ok := tabMeta.PartialIndexPredicate(i) if !ok { // A partial index predicate expression was not built for the // partial index. See Builder.buildScan for details on when this From 62ab822bc623c71f4c756ac80c18634e96f2e65d Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Tue, 29 Dec 2020 16:58:23 -0800 Subject: [PATCH 4/6] opt: move addPartialIndexPredicatesForTable to optbuilder/partial_index.go Release note: None --- pkg/sql/opt/optbuilder/partial_index.go | 75 +++++++++++++++++++++++++ pkg/sql/opt/optbuilder/select.go | 74 ------------------------ 2 files changed, 75 insertions(+), 74 deletions(-) diff --git a/pkg/sql/opt/optbuilder/partial_index.go b/pkg/sql/opt/optbuilder/partial_index.go index f2f99f9b5bac..413bae2e302f 100644 --- a/pkg/sql/opt/optbuilder/partial_index.go +++ b/pkg/sql/opt/optbuilder/partial_index.go @@ -13,11 +13,86 @@ package optbuilder import ( "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" + "github.com/cockroachdb/cockroach/pkg/sql/parser" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/errors" ) +// addPartialIndexPredicatesForTable finds all partial indexes in the table and +// adds their predicates to the table metadata (see +// TableMeta.partialIndexPredicates). The predicates are converted from strings +// to ScalarExprs here. +// +// The predicates are used as "known truths" about table data. Any predicates +// containing non-immutable operators are omitted. +// +// scan is an optional argument that is a Scan expression on the table. If scan +// outputs all the ordinary columns in the table, we avoid constructing a new +// scan. A scan and its logical properties are required in order to fully +// normalize the partial index predicates. +func (b *Builder) addPartialIndexPredicatesForTable(tabMeta *opt.TableMeta, scan memo.RelExpr) { + tab := tabMeta.Table + + // Find the first partial index. + numIndexes := tab.IndexCount() + indexOrd := 0 + for ; indexOrd < numIndexes; indexOrd++ { + if _, ok := tab.Index(indexOrd).Predicate(); ok { + break + } + } + + // Return early if there are no partial indexes. Only partial indexes have + // predicates. + if indexOrd == numIndexes { + return + } + + // Construct a scan as the tableScope expr so that logical properties of the + // scan can be used to fully normalize the index predicate. + tableScope := b.allocScope() + tableScope.appendOrdinaryColumnsFromTable(tabMeta, &tabMeta.Alias) + + // If the optional scan argument was provided and it outputs all of the + // ordinary table columns, we use it as tableScope.expr. Otherwise, we must + // construct a new scan. Attaching a scan to tableScope.expr is required to + // fully normalize the partial index predicates with logical properties of + // the scan. + if scan != nil && tableScope.colSet().SubsetOf(scan.Relational().OutputCols) { + tableScope.expr = scan + } else { + tableScope.expr = b.factory.ConstructScan(&memo.ScanPrivate{ + Table: tabMeta.MetaID, + Cols: tableScope.colSet(), + }) + } + + // Skip to the first partial index we found above. + for ; indexOrd < numIndexes; indexOrd++ { + index := tab.Index(indexOrd) + pred, ok := index.Predicate() + + // If the index is not a partial index, do nothing. + if !ok { + continue + } + + expr, err := parser.ParseExpr(pred) + if err != nil { + panic(err) + } + + // Build the partial index predicate as a memo.FiltersExpr and add it + // to the table metadata. + predExpr, err := b.buildPartialIndexPredicate(tableScope, expr, "index predicate") + if err != nil { + panic(err) + } + tabMeta.AddPartialIndexPredicate(indexOrd, &predExpr) + } +} + // buildPartialIndexPredicate builds a memo.FiltersExpr from the given // tree.Expr. The expression must be of type Bool and it must be immutable. // Returns an error if any non-immutable operators are found. diff --git a/pkg/sql/opt/optbuilder/select.go b/pkg/sql/opt/optbuilder/select.go index 6604913ce782..d155bfe68c42 100644 --- a/pkg/sql/opt/optbuilder/select.go +++ b/pkg/sql/opt/optbuilder/select.go @@ -688,80 +688,6 @@ func (b *Builder) addComputedColsForTable(tabMeta *opt.TableMeta) { } } -// addPartialIndexPredicatesForTable finds all partial indexes in the table and -// adds their predicates to the table metadata (see -// TableMeta.PartialIndexPredicates). The predicates are converted from strings -// to ScalarExprs here. -// -// The predicates are used as "known truths" about table data. Any predicates -// containing non-immutable operators are omitted. -// -// scan is an optional argument that is a Scan expression on the table. If scan -// outputs all the ordinary columns in the table, we avoid constructing a new -// scan. A scan and its logical properties are required in order to fully -// normalize the partial index predicates. -func (b *Builder) addPartialIndexPredicatesForTable(tabMeta *opt.TableMeta, scan memo.RelExpr) { - tab := tabMeta.Table - - // Find the first partial index. - numIndexes := tab.IndexCount() - indexOrd := 0 - for ; indexOrd < numIndexes; indexOrd++ { - if _, ok := tab.Index(indexOrd).Predicate(); ok { - break - } - } - - // Return early if there are no partial indexes. Only partial indexes have - // predicates. - if indexOrd == numIndexes { - return - } - - // Construct a scan as the tableScope expr so that logical properties of the - // scan can be used to fully normalize the index predicate. - tableScope := b.allocScope() - tableScope.appendOrdinaryColumnsFromTable(tabMeta, &tabMeta.Alias) - - // If the optional scan argument was provided and it outputs all of the - // ordinary table columns, we use it as tableScope.expr. Otherwise, we must - // construct a new scan. Attaching a scan to tableScope.expr is required to - // fully normalize the partial index predicates with logical properties of - // the scan. - if scan != nil && tableScope.colSet().SubsetOf(scan.Relational().OutputCols) { - tableScope.expr = scan - } else { - tableScope.expr = b.factory.ConstructScan(&memo.ScanPrivate{ - Table: tabMeta.MetaID, - Cols: tableScope.colSet(), - }) - } - - // Skip to the first partial index we found above. - for ; indexOrd < numIndexes; indexOrd++ { - index := tab.Index(indexOrd) - pred, ok := index.Predicate() - - // If the index is not a partial index, do nothing. - if !ok { - continue - } - - expr, err := parser.ParseExpr(pred) - if err != nil { - panic(err) - } - - // Build the partial index predicate as a memo.FiltersExpr and add it - // to the table metadata. - predExpr, err := b.buildPartialIndexPredicate(tableScope, expr, "index predicate") - if err != nil { - panic(err) - } - tabMeta.AddPartialIndexPredicate(indexOrd, &predExpr) - } -} - func (b *Builder) buildSequenceSelect( seq cat.Sequence, seqName *tree.TableName, inScope *scope, ) (outScope *scope) { From 4e140b62fbe766ea1bb735700e9e564686baa67b Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Tue, 29 Dec 2020 12:02:45 -0800 Subject: [PATCH 5/6] opt: prune update/upsert fetch columns not needed for partial indexes Indexed columns of partial indexes are now only fetched for UPDATE and UPSERT operations when needed. They are pruned in cases where it is guaranteed that they are not needed to build old or new index entries. For example, consider the table and UPDATE: CREATE TABLE t ( a INT PRIMARY KEY, b INT, c INT, d INT, INDEX (b) WHERE c > 0, FAMILY (a), FAMILY (b), FAMILY (c), FAMILY (d) ) UPDATE t SET d = d + 1 WHERE a = 1 The partial index is guaranteed not to change with this UPDATE because neither its indexed columns nor the columns referenced in its predicate are mutating. Therefore, the existing values of b do not need to be fetched to maintain the state of the partial index. Furthermore, the primary index does require the existing values of b because no columns in b's family are mutating. So, b can be pruned from the UPDATE's fetch columns. Release note (performance improvement): Previously, indexed columns of partial indexes were always fetched for UPDATEs and UPSERTs. Now they are only fetched if they are required for maintaining the state of the index. If an UPDATE or UPSERT mutates columns that are neither indexed by a partial index nor referenced in a partial index predicate, they will no longer be fetched (assuming that they are not needed to maintain the state of other indexes, including the primary index). --- .../testdata/logic_test/partial_index | 48 +++ pkg/sql/opt/norm/prune_cols_funcs.go | 28 +- pkg/sql/opt/norm/testdata/rules/prune_cols | 393 +++++++++++++++++- pkg/sql/opt/optbuilder/insert.go | 5 + pkg/sql/opt/optbuilder/partial_index.go | 11 +- pkg/sql/opt/optbuilder/select.go | 7 +- pkg/sql/opt/optbuilder/update.go | 5 + 7 files changed, 459 insertions(+), 38 deletions(-) diff --git a/pkg/sql/logictest/testdata/logic_test/partial_index b/pkg/sql/logictest/testdata/logic_test/partial_index index c838197fdc9b..b3b5b152ec6a 100644 --- a/pkg/sql/logictest/testdata/logic_test/partial_index +++ b/pkg/sql/logictest/testdata/logic_test/partial_index @@ -1281,6 +1281,54 @@ SELECT * FROM inv_c@i WHERE j @> '{"x": "y"}' AND s IN ('foo', 'bar') ORDER BY k 1 {"num": 1, "x": "y"} foo 3 {"num": 3, "x": "y"} bar +# Updates and Upserts with fetch columns pruned. + +statement ok +CREATE TABLE prune ( + a INT PRIMARY KEY, + b INT, + c INT, + d INT, + INDEX idx (b) WHERE c > 0, + FAMILY (a), + FAMILY (b), + FAMILY (c), + FAMILY (d) +) + +statement ok +INSERT INTO prune (a, b, c, d) VALUES (1, 2, 3, 4) + +# Test that an update is successful when fetch columns b and c are pruned +# because an update to idx is not required. +statement ok +UPDATE prune SET d = d + 1 WHERE a = 1 + +query IIII rowsort +SELECT * FROM prune@idx WHERE c > 0 +---- +1 2 3 5 + +# Test that an upsert is successful when fetch columns b and c are pruned +# because an update to idx is not required. +statement ok +UPSERT INTO prune (a, d) VALUES (1, 6) + +query IIII rowsort +SELECT * FROM prune@idx WHERE c > 0 +---- +1 2 3 6 + +# Test that an upsert is successful when fetch columns b and c are pruned +# because an update to idx is not required. +statement ok +INSERT INTO prune (a, d) VALUES (1, 6) ON CONFLICT (a) DO UPDATE SET d = 7 + +query IIII rowsort +SELECT * FROM prune@idx WHERE c > 0 +---- +1 2 3 7 + # Regression tests for #52318. Mutations on partial indexes in the # DELETE_AND_WRITE_ONLY state should update the indexes correctly. subtest regression_52318 diff --git a/pkg/sql/opt/norm/prune_cols_funcs.go b/pkg/sql/opt/norm/prune_cols_funcs.go index 831f982ba586..f65672f32d5d 100644 --- a/pkg/sql/opt/norm/prune_cols_funcs.go +++ b/pkg/sql/opt/norm/prune_cols_funcs.go @@ -144,26 +144,26 @@ func (c *CustomFuncs) NeededMutationFetchCols( // Make sure to consider indexes that are being added or dropped. for i, n := 0, tabMeta.Table.DeletableIndexCount(); i < n; i++ { - // If the columns being updated are not part of the index and the - // index is not a partial index, then the update does not require - // changes to the index. Partial indexes may be updated (even when a - // column in the index is not changing) when rows that were not - // previously in the index must be added to the index because they - // now satisfy the partial index predicate. + // If the columns being updated are not part of the index, then the + // update does not require changes to the index. Partial indexes may + // be updated (even when a column in the index is not changing) when + // the predicate references columns that are being updated. For + // example, rows that were not previously in the index must be added + // to the index because they now satisfy the partial index + // predicate, requiring the index columns to be fetched. // // Note that we use the set of index columns where the virtual // columns have been mapped to their source columns. Virtual columns // are never part of the updated columns. Updates to source columns // trigger index changes. - // - // TODO(mgartner): Index columns are not necessary when neither the - // index columns nor the columns referenced in the partial index - // predicate are being updated. We should prune mutation fetch - // columns when this is the case, rather than always marking index - // columns of partial indexes as "needed". indexCols := tabMeta.IndexColumnsMapVirtual(i) - _, isPartialIndex := tabMeta.Table.Index(i).Predicate() - if !indexCols.Intersects(updateCols) && !isPartialIndex { + pred, isPartialIndex := tabMeta.PartialIndexPredicate(i) + indexAndPredCols := indexCols.Copy() + if isPartialIndex { + predFilters := *pred.(*memo.FiltersExpr) + indexAndPredCols.UnionWith(predFilters.OuterCols()) + } + if !indexAndPredCols.Intersects(updateCols) { continue } diff --git a/pkg/sql/opt/norm/testdata/rules/prune_cols b/pkg/sql/opt/norm/testdata/rules/prune_cols index bde5476683f1..ef54a006d307 100644 --- a/pkg/sql/opt/norm/testdata/rules/prune_cols +++ b/pkg/sql/opt/norm/testdata/rules/prune_cols @@ -48,9 +48,11 @@ CREATE TABLE partial_indexes ( a INT PRIMARY KEY, b INT, c STRING, + d INT, FAMILY (a), FAMILY (b), FAMILY (c), + FAMILY (d), INDEX (c) WHERE b > 1 ) ---- @@ -2058,51 +2060,404 @@ update computed ├── c_new:13 + 1 [as=column14:14, outer=(13), immutable] └── c_new:13 + 10 [as=column15:15, outer=(13), immutable] +# Prune UPDATE fetch columns when the partial index indexes the column but +# neither the column nor the columns referenced in the partial index predicate +# are mutating. +norm expect=(PruneMutationFetchCols,PruneMutationInputCols) +UPDATE partial_indexes SET d = d + 1 WHERE a = 1 +---- +update partial_indexes + ├── columns: + ├── fetch columns: a:6 d:9 + ├── update-mapping: + │ └── d_new:11 => d:4 + ├── partial index put columns: partial_index_put1:12 + ├── partial index del columns: partial_index_put1:12 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:12 d_new:11 a:6!null d:9 + ├── cardinality: [0 - 1] + ├── immutable + ├── key: () + ├── fd: ()-->(6,9,11,12) + ├── select + │ ├── columns: a:6!null b:7 d:9 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(6,7,9) + │ ├── scan partial_indexes + │ │ ├── columns: a:6!null b:7 d:9 + │ │ ├── partial index predicates + │ │ │ └── secondary: filters + │ │ │ └── b:7 > 1 [outer=(7), constraints=(/7: [/2 - ]; tight)] + │ │ ├── key: (6) + │ │ └── fd: (6)-->(7,9) + │ └── filters + │ └── a:6 = 1 [outer=(6), constraints=(/6: [/1 - /1]; tight), fd=()-->(6)] + └── projections + ├── b:7 > 1 [as=partial_index_put1:12, outer=(7)] + └── d:9 + 1 [as=d_new:11, outer=(9), immutable] + +# Do not prune the indexed column c when a column in the partial index +# predicate, b, is being updated. +norm expect-not=PruneMutationFetchCols +UPDATE partial_indexes SET d = d + 1, b = 2 WHERE a = 1 +---- +update partial_indexes + ├── columns: + ├── fetch columns: a:6 b:7 c:8 d:9 + ├── update-mapping: + │ ├── b_new:12 => b:2 + │ └── d_new:11 => d:4 + ├── partial index put columns: partial_index_put1:13 + ├── partial index del columns: partial_index_del1:14 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:13!null partial_index_del1:14 d_new:11 b_new:12!null a:6!null b:7 c:8 d:9 + ├── cardinality: [0 - 1] + ├── immutable + ├── key: () + ├── fd: ()-->(6-9,11-14) + ├── select + │ ├── columns: a:6!null b:7 c:8 d:9 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(6-9) + │ ├── scan partial_indexes + │ │ ├── columns: a:6!null b:7 c:8 d:9 + │ │ ├── partial index predicates + │ │ │ └── secondary: filters + │ │ │ └── b:7 > 1 [outer=(7), constraints=(/7: [/2 - ]; tight)] + │ │ ├── key: (6) + │ │ └── fd: (6)-->(7-9) + │ └── filters + │ └── a:6 = 1 [outer=(6), constraints=(/6: [/1 - /1]; tight), fd=()-->(6)] + └── projections + ├── true [as=partial_index_put1:13] + ├── b:7 > 1 [as=partial_index_del1:14, outer=(7)] + ├── d:9 + 1 [as=d_new:11, outer=(9), immutable] + └── 2 [as=b_new:12] + +# Prune UPSERT fetch columns when the partial index indexes the column but +# neither the column nor the columns referenced in the partial index predicate +# are mutating. +norm expect=(PruneMutationFetchCols,PruneMutationInputCols) +UPSERT INTO partial_indexes (a, d) VALUES (1, 2) +---- +upsert partial_indexes + ├── columns: + ├── arbiter indexes: primary + ├── canary column: a:10 + ├── fetch columns: a:10 d:13 + ├── insert-mapping: + │ ├── column1:6 => a:1 + │ ├── column8:8 => b:2 + │ ├── column9:9 => c:3 + │ └── column2:7 => d:4 + ├── update-mapping: + │ └── column2:7 => d:4 + ├── partial index put columns: partial_index_put1:18 + ├── partial index del columns: partial_index_del1:19 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:18 partial_index_del1:19 column1:6!null column2:7!null column8:8 column9:9 a:10 d:13 + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(6-10,13,18,19) + ├── left-join (cross) + │ ├── columns: column1:6!null column2:7!null column8:8 column9:9 a:10 b:11 d:13 + │ ├── cardinality: [1 - 1] + │ ├── multiplicity: left-rows(exactly-one), right-rows(exactly-one) + │ ├── key: () + │ ├── fd: ()-->(6-11,13) + │ ├── values + │ │ ├── columns: column1:6!null column2:7!null column8:8 column9:9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(6-9) + │ │ └── (1, 2, NULL, NULL) + │ ├── select + │ │ ├── columns: a:10!null b:11 d:13 + │ │ ├── cardinality: [0 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(10,11,13) + │ │ ├── scan partial_indexes + │ │ │ ├── columns: a:10!null b:11 d:13 + │ │ │ ├── partial index predicates + │ │ │ │ └── secondary: filters + │ │ │ │ └── b:11 > 1 [outer=(11), constraints=(/11: [/2 - ]; tight)] + │ │ │ ├── key: (10) + │ │ │ └── fd: (10)-->(11,13) + │ │ └── filters + │ │ └── a:10 = 1 [outer=(10), constraints=(/10: [/1 - /1]; tight), fd=()-->(10)] + │ └── filters (true) + └── projections + ├── CASE WHEN a:10 IS NULL THEN column8:8 ELSE b:11 END > 1 [as=partial_index_put1:18, outer=(8,10,11)] + └── b:11 > 1 [as=partial_index_del1:19, outer=(11)] + +# Do not prune the indexed column c when a column in the partial index +# predicate, b, is being updated. +norm expect-not=PruneMutationFetchCols +UPSERT INTO partial_indexes (a, b, d) VALUES (1, 2, 3) +---- +upsert partial_indexes + ├── columns: + ├── arbiter indexes: primary + ├── canary column: a:10 + ├── fetch columns: a:10 b:11 c:12 d:13 + ├── insert-mapping: + │ ├── column1:6 => a:1 + │ ├── column2:7 => b:2 + │ ├── column9:9 => c:3 + │ └── column3:8 => d:4 + ├── update-mapping: + │ ├── column2:7 => b:2 + │ └── column3:8 => d:4 + ├── partial index put columns: partial_index_put1:17 + ├── partial index del columns: partial_index_del1:18 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:17!null partial_index_del1:18 column1:6!null column2:7!null column3:8!null column9:9 a:10 b:11 c:12 d:13 + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(6-13,17,18) + ├── left-join (cross) + │ ├── columns: column1:6!null column2:7!null column3:8!null column9:9 a:10 b:11 c:12 d:13 + │ ├── cardinality: [1 - 1] + │ ├── multiplicity: left-rows(exactly-one), right-rows(exactly-one) + │ ├── key: () + │ ├── fd: ()-->(6-13) + │ ├── values + │ │ ├── columns: column1:6!null column2:7!null column3:8!null column9:9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(6-9) + │ │ └── (1, 2, 3, NULL) + │ ├── select + │ │ ├── columns: a:10!null b:11 c:12 d:13 + │ │ ├── cardinality: [0 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(10-13) + │ │ ├── scan partial_indexes + │ │ │ ├── columns: a:10!null b:11 c:12 d:13 + │ │ │ ├── partial index predicates + │ │ │ │ └── secondary: filters + │ │ │ │ └── b:11 > 1 [outer=(11), constraints=(/11: [/2 - ]; tight)] + │ │ │ ├── key: (10) + │ │ │ └── fd: (10)-->(11-13) + │ │ └── filters + │ │ └── a:10 = 1 [outer=(10), constraints=(/10: [/1 - /1]; tight), fd=()-->(10)] + │ └── filters (true) + └── projections + ├── column2:7 > 1 [as=partial_index_put1:17, outer=(7)] + └── b:11 > 1 [as=partial_index_del1:18, outer=(11)] + +# Prune INSERT ON CONFLICT DO UPDATE fetch columns when the partial index +# indexes the column but neither the column nor the columns referenced in the +# partial index predicate are mutating. +norm expect=(PruneMutationFetchCols,PruneMutationInputCols) +INSERT INTO partial_indexes (a, d) VALUES (1, 2) ON CONFLICT (a) DO UPDATE SET d = 3 +---- +upsert partial_indexes + ├── columns: + ├── arbiter indexes: primary + ├── canary column: a:10 + ├── fetch columns: a:10 d:13 + ├── insert-mapping: + │ ├── column1:6 => a:1 + │ ├── column8:8 => b:2 + │ ├── column9:9 => c:3 + │ └── column2:7 => d:4 + ├── update-mapping: + │ └── upsert_d:19 => d:4 + ├── partial index put columns: partial_index_put1:20 + ├── partial index del columns: partial_index_del1:21 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:20 partial_index_del1:21 upsert_d:19!null column1:6!null column2:7!null column8:8 column9:9 a:10 d:13 + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(6-10,13,19-21) + ├── left-join (cross) + │ ├── columns: column1:6!null column2:7!null column8:8 column9:9 a:10 b:11 d:13 + │ ├── cardinality: [1 - 1] + │ ├── multiplicity: left-rows(exactly-one), right-rows(exactly-one) + │ ├── key: () + │ ├── fd: ()-->(6-11,13) + │ ├── values + │ │ ├── columns: column1:6!null column2:7!null column8:8 column9:9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(6-9) + │ │ └── (1, 2, NULL, NULL) + │ ├── select + │ │ ├── columns: a:10!null b:11 d:13 + │ │ ├── cardinality: [0 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(10,11,13) + │ │ ├── scan partial_indexes + │ │ │ ├── columns: a:10!null b:11 d:13 + │ │ │ ├── partial index predicates + │ │ │ │ └── secondary: filters + │ │ │ │ └── b:11 > 1 [outer=(11), constraints=(/11: [/2 - ]; tight)] + │ │ │ ├── key: (10) + │ │ │ └── fd: (10)-->(11,13) + │ │ └── filters + │ │ └── a:10 = 1 [outer=(10), constraints=(/10: [/1 - /1]; tight), fd=()-->(10)] + │ └── filters (true) + └── projections + ├── CASE WHEN a:10 IS NULL THEN column8:8 ELSE b:11 END > 1 [as=partial_index_put1:20, outer=(8,10,11)] + ├── b:11 > 1 [as=partial_index_del1:21, outer=(11)] + └── CASE WHEN a:10 IS NULL THEN column2:7 ELSE 3 END [as=upsert_d:19, outer=(7,10)] + +# Do not prune the indexed column c when a column in the partial index +# predicate, b, is being updated. +norm expect-not=PruneMutationFetchCols +INSERT INTO partial_indexes (a, d) VALUES (1, 2) ON CONFLICT (a) DO UPDATE SET b = 3, d = 4 +---- +upsert partial_indexes + ├── columns: + ├── arbiter indexes: primary + ├── canary column: a:10 + ├── fetch columns: a:10 b:11 c:12 d:13 + ├── insert-mapping: + │ ├── column1:6 => a:1 + │ ├── column8:8 => b:2 + │ ├── column9:9 => c:3 + │ └── column2:7 => d:4 + ├── update-mapping: + │ ├── upsert_b:18 => b:2 + │ └── upsert_d:20 => d:4 + ├── partial index put columns: partial_index_put1:21 + ├── partial index del columns: partial_index_del1:22 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:21 partial_index_del1:22 column1:6!null column2:7!null column8:8 column9:9 a:10 b:11 c:12 d:13 upsert_b:18 upsert_d:20!null + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(6-13,18,20-22) + ├── project + │ ├── columns: upsert_b:18 upsert_d:20!null column1:6!null column2:7!null column8:8 column9:9 a:10 b:11 c:12 d:13 + │ ├── cardinality: [1 - 1] + │ ├── key: () + │ ├── fd: ()-->(6-13,18,20) + │ ├── left-join (cross) + │ │ ├── columns: column1:6!null column2:7!null column8:8 column9:9 a:10 b:11 c:12 d:13 + │ │ ├── cardinality: [1 - 1] + │ │ ├── multiplicity: left-rows(exactly-one), right-rows(exactly-one) + │ │ ├── key: () + │ │ ├── fd: ()-->(6-13) + │ │ ├── values + │ │ │ ├── columns: column1:6!null column2:7!null column8:8 column9:9 + │ │ │ ├── cardinality: [1 - 1] + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(6-9) + │ │ │ └── (1, 2, NULL, NULL) + │ │ ├── select + │ │ │ ├── columns: a:10!null b:11 c:12 d:13 + │ │ │ ├── cardinality: [0 - 1] + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(10-13) + │ │ │ ├── scan partial_indexes + │ │ │ │ ├── columns: a:10!null b:11 c:12 d:13 + │ │ │ │ ├── partial index predicates + │ │ │ │ │ └── secondary: filters + │ │ │ │ │ └── b:11 > 1 [outer=(11), constraints=(/11: [/2 - ]; tight)] + │ │ │ │ ├── key: (10) + │ │ │ │ └── fd: (10)-->(11-13) + │ │ │ └── filters + │ │ │ └── a:10 = 1 [outer=(10), constraints=(/10: [/1 - /1]; tight), fd=()-->(10)] + │ │ └── filters (true) + │ └── projections + │ ├── CASE WHEN a:10 IS NULL THEN column8:8 ELSE 3 END [as=upsert_b:18, outer=(8,10)] + │ └── CASE WHEN a:10 IS NULL THEN column2:7 ELSE 4 END [as=upsert_d:20, outer=(7,10)] + └── projections + ├── upsert_b:18 > 1 [as=partial_index_put1:21, outer=(18)] + └── b:11 > 1 [as=partial_index_del1:22, outer=(11)] + +# Do not prune DELETE fetch columns. +norm +DELETE FROM partial_indexes WHERE a = 1 +---- +delete partial_indexes + ├── columns: + ├── fetch columns: a:6 c:8 + ├── partial index del columns: partial_index_del1:11 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_del1:11 a:6!null c:8 + ├── cardinality: [0 - 1] + ├── key: () + ├── fd: ()-->(6,8,11) + ├── select + │ ├── columns: a:6!null b:7 c:8 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(6-8) + │ ├── scan partial_indexes + │ │ ├── columns: a:6!null b:7 c:8 + │ │ ├── partial index predicates + │ │ │ └── secondary: filters + │ │ │ └── b:7 > 1 [outer=(7), constraints=(/7: [/2 - ]; tight)] + │ │ ├── key: (6) + │ │ └── fd: (6)-->(7,8) + │ └── filters + │ └── a:6 = 1 [outer=(6), constraints=(/6: [/1 - /1]; tight), fd=()-->(6)] + └── projections + └── b:7 > 1 [as=partial_index_del1:11, outer=(7)] + # Do not prune columns that are required for evaluating partial index # predicates. -norm expect-not=PruneMutationFetchCols +norm UPDATE partial_indexes SET b = b + 1 WHERE a = 1 ---- update partial_indexes ├── columns: - ├── fetch columns: a:5 b:6 c:7 + ├── fetch columns: a:6 b:7 c:8 ├── update-mapping: - │ └── b_new:9 => b:2 - ├── partial index put columns: partial_index_put1:10 - ├── partial index del columns: partial_index_del1:11 + │ └── b_new:11 => b:2 + ├── partial index put columns: partial_index_put1:12 + ├── partial index del columns: partial_index_del1:13 ├── cardinality: [0 - 0] ├── volatile, mutations └── project - ├── columns: partial_index_put1:10 partial_index_del1:11 a:5!null b:6 c:7 b_new:9 + ├── columns: partial_index_put1:12 partial_index_del1:13 a:6!null b:7 c:8 b_new:11 ├── cardinality: [0 - 1] ├── immutable ├── key: () - ├── fd: ()-->(5-7,9-11) + ├── fd: ()-->(6-8,11-13) ├── project - │ ├── columns: b_new:9 a:5!null b:6 c:7 + │ ├── columns: b_new:11 a:6!null b:7 c:8 │ ├── cardinality: [0 - 1] │ ├── immutable │ ├── key: () - │ ├── fd: ()-->(5-7,9) + │ ├── fd: ()-->(6-8,11) │ ├── select - │ │ ├── columns: a:5!null b:6 c:7 + │ │ ├── columns: a:6!null b:7 c:8 │ │ ├── cardinality: [0 - 1] │ │ ├── key: () - │ │ ├── fd: ()-->(5-7) + │ │ ├── fd: ()-->(6-8) │ │ ├── scan partial_indexes - │ │ │ ├── columns: a:5!null b:6 c:7 + │ │ │ ├── columns: a:6!null b:7 c:8 │ │ │ ├── partial index predicates │ │ │ │ └── secondary: filters - │ │ │ │ └── b:6 > 1 [outer=(6), constraints=(/6: [/2 - ]; tight)] - │ │ │ ├── key: (5) - │ │ │ └── fd: (5)-->(6,7) + │ │ │ │ └── b:7 > 1 [outer=(7), constraints=(/7: [/2 - ]; tight)] + │ │ │ ├── key: (6) + │ │ │ └── fd: (6)-->(7,8) │ │ └── filters - │ │ └── a:5 = 1 [outer=(5), constraints=(/5: [/1 - /1]; tight), fd=()-->(5)] + │ │ └── a:6 = 1 [outer=(6), constraints=(/6: [/1 - /1]; tight), fd=()-->(6)] │ └── projections - │ └── b:6 + 1 [as=b_new:9, outer=(6), immutable] + │ └── b:7 + 1 [as=b_new:11, outer=(7), immutable] └── projections - ├── b_new:9 > 1 [as=partial_index_put1:10, outer=(9)] - └── b:6 > 1 [as=partial_index_del1:11, outer=(6)] + ├── b_new:11 > 1 [as=partial_index_put1:12, outer=(11)] + └── b:7 > 1 [as=partial_index_del1:13, outer=(7)] # Prune secondary family column not needed for the update. norm expect=(PruneMutationFetchCols,PruneMutationInputCols) diff --git a/pkg/sql/opt/optbuilder/insert.go b/pkg/sql/opt/optbuilder/insert.go index aa929e258714..cdf095caad30 100644 --- a/pkg/sql/opt/optbuilder/insert.go +++ b/pkg/sql/opt/optbuilder/insert.go @@ -1061,6 +1061,11 @@ func (mb *mutationBuilder) buildUpsert(returning tree.ReturningExprs) { // Add any check constraint boolean columns to the input. mb.addCheckConstraintCols() + // Add the partial index predicate expressions to the table metadata. + // These expressions are used to prune fetch columns during + // normalization. + mb.b.addPartialIndexPredicatesForTable(mb.md.TableMeta(mb.tabID), nil /* scan */, true /* includeDeletable */) + // Project partial index PUT and DEL boolean columns. // // In some cases existing rows may not be fetched for an UPSERT (see diff --git a/pkg/sql/opt/optbuilder/partial_index.go b/pkg/sql/opt/optbuilder/partial_index.go index 413bae2e302f..c60cc64136c7 100644 --- a/pkg/sql/opt/optbuilder/partial_index.go +++ b/pkg/sql/opt/optbuilder/partial_index.go @@ -31,11 +31,18 @@ import ( // outputs all the ordinary columns in the table, we avoid constructing a new // scan. A scan and its logical properties are required in order to fully // normalize the partial index predicates. -func (b *Builder) addPartialIndexPredicatesForTable(tabMeta *opt.TableMeta, scan memo.RelExpr) { +func (b *Builder) addPartialIndexPredicatesForTable( + tabMeta *opt.TableMeta, scan memo.RelExpr, includeDeletable bool, +) { tab := tabMeta.Table + var numIndexes int + if includeDeletable { + numIndexes = tab.DeletableIndexCount() + } else { + numIndexes = tab.IndexCount() + } // Find the first partial index. - numIndexes := tab.IndexCount() indexOrd := 0 for ; indexOrd < numIndexes; indexOrd++ { if _, ok := tab.Index(indexOrd).Predicate(); ok { diff --git a/pkg/sql/opt/optbuilder/select.go b/pkg/sql/opt/optbuilder/select.go index d155bfe68c42..3a74a991df95 100644 --- a/pkg/sql/opt/optbuilder/select.go +++ b/pkg/sql/opt/optbuilder/select.go @@ -536,9 +536,10 @@ func (b *Builder) buildScan( outScope.expr = b.factory.ConstructScan(&private) // Add the partial indexes after constructing the scan so we can use the - // logical properties of the scan to fully normalize the index - // predicates. - b.addPartialIndexPredicatesForTable(tabMeta, outScope.expr) + // logical properties of the scan to fully normalize the index predicates. + // We don't need to add deletable partial index predicates in the context of + // a scan. + b.addPartialIndexPredicatesForTable(tabMeta, outScope.expr, false /* includeDeletable */) if !virtualColIDs.Empty() { // Project the expressions for the virtual columns (and pass through all diff --git a/pkg/sql/opt/optbuilder/update.go b/pkg/sql/opt/optbuilder/update.go index e91158ef185a..a38f3370a795 100644 --- a/pkg/sql/opt/optbuilder/update.go +++ b/pkg/sql/opt/optbuilder/update.go @@ -336,6 +336,11 @@ func (mb *mutationBuilder) buildUpdate(returning tree.ReturningExprs) { mb.addCheckConstraintCols() + // Add the partial index predicate expressions to the table metadata. + // These expressions are used to prune fetch columns during + // normalization. + mb.b.addPartialIndexPredicatesForTable(mb.md.TableMeta(mb.tabID), nil /* scan */, true /* includeDeletable */) + // Project partial index PUT and DEL boolean columns. mb.projectPartialIndexPutAndDelCols(preCheckScope, mb.fetchScope) From 039fb1b669ba25f10733eba8dc3ec28fca7a1a24 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Tue, 29 Dec 2020 16:30:58 -0800 Subject: [PATCH 6/6] opt: normalize partial index PUT/DEL projections to false The `SimplifyPartialIndexProjections` normalization rule has been added that normalizes synthesized partial index PUT and DEL columns to False when it is guaranteed that a mutation will not require changes to the associated partial index. This normalization can lead to further normalizations, such as pruning columns that the synthesized projections relied on. The motivation for this change is to allow fully disjoint updates to different columns in the same row, when the columns are split across different families. By pruning columns not needed to maintain a partial index, we're not forced to scan all column families. This can ultimately reduce contention during updates. Release note (performance improvement): UPDATE operations on tables with partial indexes no longer evaluate partial index predicate expressions when it is guaranteed that the operation will not alter the state of the partial index. In some cases, this can eliminate fetching the existing value of columns that are referenced in partial index predicates. --- pkg/sql/opt/norm/BUILD.bazel | 1 + pkg/sql/opt/norm/mutation_funcs.go | 119 ++++++++++ pkg/sql/opt/norm/prune_cols_funcs.go | 15 +- pkg/sql/opt/norm/rules/mutation.opt | 38 +++ pkg/sql/opt/norm/testdata/rules/mutation | 258 +++++++++++++++++++++ pkg/sql/opt/norm/testdata/rules/prune_cols | 12 +- 6 files changed, 435 insertions(+), 8 deletions(-) create mode 100644 pkg/sql/opt/norm/mutation_funcs.go create mode 100644 pkg/sql/opt/norm/testdata/rules/mutation diff --git a/pkg/sql/opt/norm/BUILD.bazel b/pkg/sql/opt/norm/BUILD.bazel index 28af1cffb2d6..1355bdbd12fb 100644 --- a/pkg/sql/opt/norm/BUILD.bazel +++ b/pkg/sql/opt/norm/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "join_funcs.go", "limit_funcs.go", "list_sorter.go", + "mutation_funcs.go", "ordering_funcs.go", "project_builder.go", "project_funcs.go", diff --git a/pkg/sql/opt/norm/mutation_funcs.go b/pkg/sql/opt/norm/mutation_funcs.go new file mode 100644 index 000000000000..c1800fc60f33 --- /dev/null +++ b/pkg/sql/opt/norm/mutation_funcs.go @@ -0,0 +1,119 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package norm + +import ( + "github.com/cockroachdb/cockroach/pkg/sql/opt" + "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" +) + +// SimplifiablePartialIndexProjectCols returns the set of projected partial +// index PUT and DEL columns with expressions that can be simplified to false. +// These projected expressions can only be simplified to false when an UPDATE +// mutates neither the associated index's columns nor the columns referenced in +// the partial index predicate. +func (c *CustomFuncs) SimplifiablePartialIndexProjectCols( + private *memo.MutationPrivate, + uniqueChecks memo.UniqueChecksExpr, + fkChecks memo.FKChecksExpr, + projections memo.ProjectionsExpr, +) opt.ColSet { + tabMeta := c.mem.Metadata().TableMeta(private.Table) + + // Determine the set of target table columns that need to be updated. Notice + // that we collect the target table column IDs, not the update column IDs. + var updateCols opt.ColSet + for ord, col := range private.UpdateCols { + if col != 0 { + updateCols.Add(tabMeta.MetaID.ColumnID(ord)) + } + } + + // Determine the set of columns needed for the mutation operator, excluding + // the partial index PUT and DEL columns. + neededMutationCols := c.neededMutationCols(private, uniqueChecks, fkChecks, false /* includePartialIndexCols */) + + // Determine the set of project columns that are already simplified to + // false. + var simplifiedProjectCols opt.ColSet + for i := range projections { + project := &projections[i] + if project.Element == memo.FalseSingleton { + simplifiedProjectCols.Add(project.Col) + } + } + + // Columns that are required by the mutation operator and columns that + // have already been simplified to false are ineligible to be simplified. + ineligibleCols := neededMutationCols.Union(simplifiedProjectCols) + + // ord is an ordinal into the mutation's PartialIndexPutCols and + // PartialIndexDelCols, which both have entries for each partial index + // defined on the table. + ord := -1 + var cols opt.ColSet + for i, n := 0, tabMeta.Table.DeletableIndexCount(); i < n; i++ { + pred, isPartialIndex := tabMeta.PartialIndexPredicate(i) + + // Skip non-partial indexes. + if !isPartialIndex { + continue + } + ord++ + + // If the columns being updated are part of the index or referenced in + // the partial index predicate, then updates to the index may be + // required. Therefore, the partial index PUT and DEL columns cannot be + // simplified. + // + // Note that we use the set of index columns where the virtual + // columns have been mapped to their source columns. Virtual columns + // are never part of the updated columns. Updates to source columns + // trigger index changes. + predFilters := *pred.(*memo.FiltersExpr) + indexAndPredCols := tabMeta.IndexColumnsMapVirtual(i) + indexAndPredCols.UnionWith(predFilters.OuterCols()) + if indexAndPredCols.Intersects(updateCols) { + continue + } + + // Add the projected PUT column if it is eligible to be simplified. + putCol := private.PartialIndexPutCols[ord] + if !ineligibleCols.Contains(putCol) { + cols.Add(putCol) + } + + // Add the projected DEL column if it is eligible to be simplified. + delCol := private.PartialIndexDelCols[ord] + if !ineligibleCols.Contains(delCol) { + cols.Add(delCol) + } + } + + return cols +} + +// SimplifyPartialIndexProjections returns a new projection expression with any +// projected column's expression simplified to false if the column exists in +// simplifiableCols. +func (c *CustomFuncs) SimplifyPartialIndexProjections( + projections memo.ProjectionsExpr, simplifiableCols opt.ColSet, +) memo.ProjectionsExpr { + simplified := make(memo.ProjectionsExpr, len(projections)) + for i := range projections { + if col := projections[i].Col; simplifiableCols.Contains(col) { + simplified[i] = c.f.ConstructProjectionsItem(memo.FalseSingleton, col) + } else { + simplified[i] = projections[i] + } + } + return simplified +} diff --git a/pkg/sql/opt/norm/prune_cols_funcs.go b/pkg/sql/opt/norm/prune_cols_funcs.go index f65672f32d5d..c7aa2303aacf 100644 --- a/pkg/sql/opt/norm/prune_cols_funcs.go +++ b/pkg/sql/opt/norm/prune_cols_funcs.go @@ -44,6 +44,15 @@ func (c *CustomFuncs) NeededExplainCols(private *memo.ExplainPrivate) opt.ColSet // in turn trigger the PruneMutationInputCols rule. func (c *CustomFuncs) NeededMutationCols( private *memo.MutationPrivate, uniqueChecks memo.UniqueChecksExpr, fkChecks memo.FKChecksExpr, +) opt.ColSet { + return c.neededMutationCols(private, uniqueChecks, fkChecks, true /* includePartialIndexCols */) +} + +func (c *CustomFuncs) neededMutationCols( + private *memo.MutationPrivate, + uniqueChecks memo.UniqueChecksExpr, + fkChecks memo.FKChecksExpr, + includePartialIndexCols bool, ) opt.ColSet { var cols opt.ColSet @@ -60,8 +69,10 @@ func (c *CustomFuncs) NeededMutationCols( addCols(private.FetchCols) addCols(private.UpdateCols) addCols(private.CheckCols) - addCols(private.PartialIndexPutCols) - addCols(private.PartialIndexDelCols) + if includePartialIndexCols { + addCols(private.PartialIndexPutCols) + addCols(private.PartialIndexDelCols) + } addCols(private.ReturnCols) addCols(opt.OptionalColList(private.PassthroughCols)) if private.CanaryCol != 0 { diff --git a/pkg/sql/opt/norm/rules/mutation.opt b/pkg/sql/opt/norm/rules/mutation.opt index e69de29bb2d1..e66c33278474 100644 --- a/pkg/sql/opt/norm/rules/mutation.opt +++ b/pkg/sql/opt/norm/rules/mutation.opt @@ -0,0 +1,38 @@ +# ============================================================================= +# mutation.opt contains normalization rules for the mutation operators. +# ============================================================================= + +# SimplifyPartialIndexProjections converts partial index PUT and DEL projected +# expressions to false when it is guaranteed that the mutation will not require +# changes to the associated partial index. These projected expressions can only +# be simplified to false when an UPDATE mutates neither the associated index's +# columns nor the columns referenced in the partial index predicate. +[SimplifyPartialIndexProjections, Normalize] +(Update + $project:(Project $input:* $projections:* $passthrough:*) + $uniqueChecks:* + $fkChecks:* + $mutationPrivate:* & + ^(ColsAreEmpty + $simplifiableCols:(SimplifiablePartialIndexProjectCols + $mutationPrivate + $uniqueChecks + $fkChecks + $projections + ) + ) +) +=> +(Update + (Project + $input + (SimplifyPartialIndexProjections + $projections + $simplifiableCols + ) + $passthrough + ) + $uniqueChecks + $fkChecks + $mutationPrivate +) diff --git a/pkg/sql/opt/norm/testdata/rules/mutation b/pkg/sql/opt/norm/testdata/rules/mutation new file mode 100644 index 000000000000..5c4962779360 --- /dev/null +++ b/pkg/sql/opt/norm/testdata/rules/mutation @@ -0,0 +1,258 @@ +# -------------------------------------------------- +# SimplifyPartialIndexProjections +# -------------------------------------------------- + +exec-ddl +CREATE TABLE t ( + k INT PRIMARY KEY, + a INT, + b INT, + c INT, + d INT, + e INT, + f INT, + g INT, + h BOOL, + INDEX (a), + INDEX (c) WHERE d > 1, + INDEX (e) WHERE f > 1 AND g > 1, + INDEX (b), + INDEX (d) WHERE c > 1 +) +---- + +# Simplify UPDATE partial index put/del column to false when the indexed columns +# and columns referenced in predicates are not mutating. +norm expect=SimplifyPartialIndexProjections +UPDATE t SET a = 2, b = 2 WHERE k = 1 +---- +update t + ├── columns: + ├── fetch columns: k:11 a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── update-mapping: + │ ├── a_new:21 => a:2 + │ └── a_new:21 => b:3 + ├── partial index put columns: partial_index_put1:22 partial_index_put2:23 partial_index_put3:24 + ├── partial index del columns: partial_index_put1:22 partial_index_put2:23 partial_index_put3:24 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:22!null partial_index_put2:23!null partial_index_put3:24!null a_new:21!null k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── cardinality: [0 - 1] + ├── key: () + ├── fd: ()-->(11-19,21-24) + ├── select + │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(11-19) + │ ├── scan t + │ │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ │ ├── partial index predicates + │ │ │ ├── secondary: filters + │ │ │ │ └── d:15 > 1 [outer=(15), constraints=(/15: [/2 - ]; tight)] + │ │ │ ├── secondary: filters + │ │ │ │ ├── f:17 > 1 [outer=(17), constraints=(/17: [/2 - ]; tight)] + │ │ │ │ └── g:18 > 1 [outer=(18), constraints=(/18: [/2 - ]; tight)] + │ │ │ └── secondary: filters + │ │ │ └── c:14 > 1 [outer=(14), constraints=(/14: [/2 - ]; tight)] + │ │ ├── key: (11) + │ │ └── fd: (11)-->(12-19) + │ └── filters + │ └── k:11 = 1 [outer=(11), constraints=(/11: [/1 - /1]; tight), fd=()-->(11)] + └── projections + ├── false [as=partial_index_put1:22] + ├── false [as=partial_index_put2:23] + ├── false [as=partial_index_put3:24] + └── 2 [as=a_new:21] + +# Simplify UPDATE partial index put/del column to false for second partial index +# only. +norm expect=SimplifyPartialIndexProjections +UPDATE t SET a = 2, d = 2 WHERE k = 1 +---- +update t + ├── columns: + ├── fetch columns: k:11 a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── update-mapping: + │ ├── a_new:21 => a:2 + │ └── a_new:21 => d:5 + ├── partial index put columns: partial_index_put1:22 partial_index_put2:24 partial_index_put3:25 + ├── partial index del columns: partial_index_del1:23 partial_index_put2:24 partial_index_put3:25 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:22!null partial_index_del1:23 partial_index_put2:24!null partial_index_put3:25 a_new:21!null k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── cardinality: [0 - 1] + ├── key: () + ├── fd: ()-->(11-19,21-25) + ├── select + │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(11-19) + │ ├── scan t + │ │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ │ ├── partial index predicates + │ │ │ ├── secondary: filters + │ │ │ │ └── d:15 > 1 [outer=(15), constraints=(/15: [/2 - ]; tight)] + │ │ │ ├── secondary: filters + │ │ │ │ ├── f:17 > 1 [outer=(17), constraints=(/17: [/2 - ]; tight)] + │ │ │ │ └── g:18 > 1 [outer=(18), constraints=(/18: [/2 - ]; tight)] + │ │ │ └── secondary: filters + │ │ │ └── c:14 > 1 [outer=(14), constraints=(/14: [/2 - ]; tight)] + │ │ ├── key: (11) + │ │ └── fd: (11)-->(12-19) + │ └── filters + │ └── k:11 = 1 [outer=(11), constraints=(/11: [/1 - /1]; tight), fd=()-->(11)] + └── projections + ├── true [as=partial_index_put1:22] + ├── d:15 > 1 [as=partial_index_del1:23, outer=(15)] + ├── false [as=partial_index_put2:24] + ├── c:14 > 1 [as=partial_index_put3:25, outer=(14)] + └── 2 [as=a_new:21] + +# Do not simplify partial index put/del column to false when the indexed columns +# are mutating. +norm expect-not=SimplifyPartialIndexProjections +UPDATE t SET c = 1, e = 1 WHERE k = 1 +---- +update t + ├── columns: + ├── fetch columns: k:11 a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── update-mapping: + │ ├── c_new:21 => c:4 + │ └── c_new:21 => e:6 + ├── partial index put columns: partial_index_put1:22 partial_index_put2:23 partial_index_put3:24 + ├── partial index del columns: partial_index_put1:22 partial_index_put2:23 partial_index_del3:25 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:22 partial_index_put2:23 partial_index_put3:24!null partial_index_del3:25 c_new:21!null k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── cardinality: [0 - 1] + ├── key: () + ├── fd: ()-->(11-19,21-25) + ├── select + │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(11-19) + │ ├── scan t + │ │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ │ ├── partial index predicates + │ │ │ ├── secondary: filters + │ │ │ │ └── d:15 > 1 [outer=(15), constraints=(/15: [/2 - ]; tight)] + │ │ │ ├── secondary: filters + │ │ │ │ ├── f:17 > 1 [outer=(17), constraints=(/17: [/2 - ]; tight)] + │ │ │ │ └── g:18 > 1 [outer=(18), constraints=(/18: [/2 - ]; tight)] + │ │ │ └── secondary: filters + │ │ │ └── c:14 > 1 [outer=(14), constraints=(/14: [/2 - ]; tight)] + │ │ ├── key: (11) + │ │ └── fd: (11)-->(12-19) + │ └── filters + │ └── k:11 = 1 [outer=(11), constraints=(/11: [/1 - /1]; tight), fd=()-->(11)] + └── projections + ├── d:15 > 1 [as=partial_index_put1:22, outer=(15)] + ├── (f:17 > 1) AND (g:18 > 1) [as=partial_index_put2:23, outer=(17,18)] + ├── false [as=partial_index_put3:24] + ├── c:14 > 1 [as=partial_index_del3:25, outer=(14)] + └── 1 [as=c_new:21] + +# Do not simplify partial index put/del column to false when the columns +# referenced in partial index predicates are mutating. +norm expect-not=SimplifyPartialIndexProjections +UPDATE t SET d = d + 1, g = g + 1 WHERE k = 1 +---- +update t + ├── columns: + ├── fetch columns: k:11 a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── update-mapping: + │ ├── d_new:21 => d:5 + │ └── g_new:22 => g:8 + ├── partial index put columns: partial_index_put1:23 partial_index_put2:25 partial_index_put3:27 + ├── partial index del columns: partial_index_del1:24 partial_index_del2:26 partial_index_put3:27 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:23 partial_index_del1:24 partial_index_put2:25 partial_index_del2:26 partial_index_put3:27 k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 d_new:21 g_new:22 + ├── cardinality: [0 - 1] + ├── immutable + ├── key: () + ├── fd: ()-->(11-19,21-27) + ├── project + │ ├── columns: d_new:21 g_new:22 k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ ├── cardinality: [0 - 1] + │ ├── immutable + │ ├── key: () + │ ├── fd: ()-->(11-19,21,22) + │ ├── select + │ │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ │ ├── cardinality: [0 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(11-19) + │ │ ├── scan t + │ │ │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ │ │ ├── partial index predicates + │ │ │ │ ├── secondary: filters + │ │ │ │ │ └── d:15 > 1 [outer=(15), constraints=(/15: [/2 - ]; tight)] + │ │ │ │ ├── secondary: filters + │ │ │ │ │ ├── f:17 > 1 [outer=(17), constraints=(/17: [/2 - ]; tight)] + │ │ │ │ │ └── g:18 > 1 [outer=(18), constraints=(/18: [/2 - ]; tight)] + │ │ │ │ └── secondary: filters + │ │ │ │ └── c:14 > 1 [outer=(14), constraints=(/14: [/2 - ]; tight)] + │ │ │ ├── key: (11) + │ │ │ └── fd: (11)-->(12-19) + │ │ └── filters + │ │ └── k:11 = 1 [outer=(11), constraints=(/11: [/1 - /1]; tight), fd=()-->(11)] + │ └── projections + │ ├── d:15 + 1 [as=d_new:21, outer=(15), immutable] + │ └── g:18 + 1 [as=g_new:22, outer=(18), immutable] + └── projections + ├── d_new:21 > 1 [as=partial_index_put1:23, outer=(21)] + ├── d:15 > 1 [as=partial_index_del1:24, outer=(15)] + ├── (f:17 > 1) AND (g_new:22 > 1) [as=partial_index_put2:25, outer=(17,22)] + ├── (f:17 > 1) AND (g:18 > 1) [as=partial_index_del2:26, outer=(17,18)] + └── c:14 > 1 [as=partial_index_put3:27, outer=(14)] + +# Do not simplify partial index put/del column to false when it is also an +# update column (h_new). +norm +UPDATE t SET h = d > 1 WHERE k = 1 +---- +update t + ├── columns: + ├── fetch columns: k:11 a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── update-mapping: + │ └── h_new:21 => h:9 + ├── partial index put columns: h_new:21 partial_index_put2:22 partial_index_put3:23 + ├── partial index del columns: h_new:21 partial_index_put2:22 partial_index_put3:23 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put2:22!null partial_index_put3:23!null h_new:21 k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── cardinality: [0 - 1] + ├── key: () + ├── fd: ()-->(11-19,21-23) + ├── select + │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(11-19) + │ ├── scan t + │ │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ │ ├── partial index predicates + │ │ │ ├── secondary: filters + │ │ │ │ └── d:15 > 1 [outer=(15), constraints=(/15: [/2 - ]; tight)] + │ │ │ ├── secondary: filters + │ │ │ │ ├── f:17 > 1 [outer=(17), constraints=(/17: [/2 - ]; tight)] + │ │ │ │ └── g:18 > 1 [outer=(18), constraints=(/18: [/2 - ]; tight)] + │ │ │ └── secondary: filters + │ │ │ └── c:14 > 1 [outer=(14), constraints=(/14: [/2 - ]; tight)] + │ │ ├── key: (11) + │ │ └── fd: (11)-->(12-19) + │ └── filters + │ └── k:11 = 1 [outer=(11), constraints=(/11: [/1 - /1]; tight), fd=()-->(11)] + └── projections + ├── false [as=partial_index_put2:22] + ├── false [as=partial_index_put3:23] + └── d:15 > 1 [as=h_new:21, outer=(15)] diff --git a/pkg/sql/opt/norm/testdata/rules/prune_cols b/pkg/sql/opt/norm/testdata/rules/prune_cols index ef54a006d307..d07165b60127 100644 --- a/pkg/sql/opt/norm/testdata/rules/prune_cols +++ b/pkg/sql/opt/norm/testdata/rules/prune_cols @@ -2076,27 +2076,27 @@ update partial_indexes ├── cardinality: [0 - 0] ├── volatile, mutations └── project - ├── columns: partial_index_put1:12 d_new:11 a:6!null d:9 + ├── columns: partial_index_put1:12!null d_new:11 a:6!null d:9 ├── cardinality: [0 - 1] ├── immutable ├── key: () ├── fd: ()-->(6,9,11,12) ├── select - │ ├── columns: a:6!null b:7 d:9 + │ ├── columns: a:6!null d:9 │ ├── cardinality: [0 - 1] │ ├── key: () - │ ├── fd: ()-->(6,7,9) + │ ├── fd: ()-->(6,9) │ ├── scan partial_indexes - │ │ ├── columns: a:6!null b:7 d:9 + │ │ ├── columns: a:6!null d:9 │ │ ├── partial index predicates │ │ │ └── secondary: filters │ │ │ └── b:7 > 1 [outer=(7), constraints=(/7: [/2 - ]; tight)] │ │ ├── key: (6) - │ │ └── fd: (6)-->(7,9) + │ │ └── fd: (6)-->(9) │ └── filters │ └── a:6 = 1 [outer=(6), constraints=(/6: [/1 - /1]; tight), fd=()-->(6)] └── projections - ├── b:7 > 1 [as=partial_index_put1:12, outer=(7)] + ├── false [as=partial_index_put1:12] └── d:9 + 1 [as=d_new:11, outer=(9), immutable] # Do not prune the indexed column c when a column in the partial index