From 8549ad2ee25f66cfc963c805ee11ebf322ed0272 Mon Sep 17 00:00:00 2001 From: wenyihu3 Date: Sun, 17 Jul 2022 14:59:13 -0400 Subject: [PATCH] sql: add not visible index to optimizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds the logic of the invisible index feature to the optimizer. After this commit has been merged, the invisible index feature should be fully functional with `CREATE INDEX` and `CREATE TABLE`. Assists: https://github.com/cockroachdb/cockroach/issues/72576 See also: https://github.com/cockroachdb/cockroach/pull/85239 Release note (sql change): creating a not visible index using `CREATE TABLE … (INDEX … NOT VISIBLE)` or `CREATE INDEX … NOT VISIBLE` is now supported. --- pkg/sql/alter_table.go | 10 + pkg/sql/catalog/descpb/index.go | 9 + pkg/sql/catalog/descpb/structured.go | 34 +- pkg/sql/catalog/table_elements.go | 1 + pkg/sql/catalog/tabledesc/BUILD.bazel | 1 + pkg/sql/catalog/tabledesc/index.go | 6 + pkg/sql/catalog/tabledesc/validate.go | 48 + pkg/sql/create_index.go | 15 +- pkg/sql/create_table.go | 21 +- .../execbuilder/testdata/not_visible_index | 897 +++++++++++++++++- pkg/sql/opt/optbuilder/select.go | 4 + pkg/sql/opt/xform/scan_index_iter.go | 17 +- 12 files changed, 1018 insertions(+), 45 deletions(-) diff --git a/pkg/sql/alter_table.go b/pkg/sql/alter_table.go index dc3db15d0862..de94ed3142a3 100644 --- a/pkg/sql/alter_table.go +++ b/pkg/sql/alter_table.go @@ -859,6 +859,16 @@ func (n *alterTableNode) startExec(params runParams) error { return err } } + + // Warn against dropping an index if there exists a NotVisible index that may + // be used for constraint check behind the scene. + if notVisibleIndexNotice := tabledesc.ValidateNotVisibleIndexWithinTable(n.tableDesc); notVisibleIndexNotice != nil { + params.p.BufferClientNotice( + params.ctx, + notVisibleIndexNotice, + ) + } + // Were some changes made? // // This is only really needed for the unittests that add dummy mutations diff --git a/pkg/sql/catalog/descpb/index.go b/pkg/sql/catalog/descpb/index.go index 63ed04b26f86..df5963e2f1e4 100644 --- a/pkg/sql/catalog/descpb/index.go +++ b/pkg/sql/catalog/descpb/index.go @@ -64,6 +64,15 @@ func (desc *IndexDescriptor) FillColumns(elems tree.IndexElemList) error { return nil } +// IsHelpfulOriginIndex returns whether the index may become a helpful index for +// foreign key constraint check on the child table. Given the originColIDs for +// foreign key constraint, the index could be useful for FK check if it has +// overlaps (intersection) with the originColIDs. +// TODO(wenyihu6): check if checking HasIntersection is correct here. +func (desc *IndexDescriptor) IsHelpfulOriginIndex(originColIDs ColumnIDs) bool { + return !desc.IsPartial() && ColumnIDs(desc.KeyColumnIDs).HasIntersection(originColIDs) +} + // IsValidOriginIndex returns whether the index can serve as an origin index for a foreign // key constraint with the provided set of originColIDs. func (desc *IndexDescriptor) IsValidOriginIndex(originColIDs ColumnIDs) bool { diff --git a/pkg/sql/catalog/descpb/structured.go b/pkg/sql/catalog/descpb/structured.go index 29f0ea1fa00f..fb4f8336e3a7 100644 --- a/pkg/sql/catalog/descpb/structured.go +++ b/pkg/sql/catalog/descpb/structured.go @@ -131,6 +131,28 @@ func (c ColumnIDs) HasPrefix(input ColumnIDs) bool { return true } +// UtilFastIntSetOperation takes in a list of ColumnIDs and performs the given +// operation op after transforming ColumnIDs to FastIntSet. +func (c ColumnIDs) UtilFastIntSetOperation( + input ColumnIDs, op func(util.FastIntSet, util.FastIntSet) bool, +) bool { + ourColsSet := util.MakeFastIntSet() + for _, col := range c { + ourColsSet.Add(int(col)) + } + + inputColsSet := util.MakeFastIntSet() + for _, inputCol := range input { + inputColsSet.Add(int(inputCol)) + } + return op(ourColsSet, inputColsSet) +} + +// HasIntersection returns true if the input list has intersection with columnIDs. +func (c ColumnIDs) HasIntersection(input ColumnIDs) bool { + return c.UtilFastIntSetOperation(input, util.FastIntSet.Intersects) +} + // Equals returns true if the input list is equal to this list. func (c ColumnIDs) Equals(input ColumnIDs) bool { if len(input) != len(c) { @@ -147,17 +169,7 @@ func (c ColumnIDs) Equals(input ColumnIDs) bool { // PermutationOf returns true if this list and the input list contain the same // set of column IDs in any order. Duplicate ColumnIDs have no effect. func (c ColumnIDs) PermutationOf(input ColumnIDs) bool { - ourColsSet := util.MakeFastIntSet() - for _, col := range c { - ourColsSet.Add(int(col)) - } - - inputColsSet := util.MakeFastIntSet() - for _, inputCol := range input { - inputColsSet.Add(int(inputCol)) - } - - return inputColsSet.Equals(ourColsSet) + return c.UtilFastIntSetOperation(input, util.FastIntSet.Equals) } // Contains returns whether this list contains the input ID. diff --git a/pkg/sql/catalog/table_elements.go b/pkg/sql/catalog/table_elements.go index e3e03a1946d1..113e883ec888 100644 --- a/pkg/sql/catalog/table_elements.go +++ b/pkg/sql/catalog/table_elements.go @@ -151,6 +151,7 @@ type Index interface { GetShardColumnName() string IsValidOriginIndex(originColIDs descpb.ColumnIDs) bool + IsHelpfulOriginIndex(originColIDs descpb.ColumnIDs) bool IsValidReferencedUniqueConstraint(referencedColIDs descpb.ColumnIDs) bool GetPartitioning() Partitioning diff --git a/pkg/sql/catalog/tabledesc/BUILD.bazel b/pkg/sql/catalog/tabledesc/BUILD.bazel index 1f94aec128e6..f743c75a9e18 100644 --- a/pkg/sql/catalog/tabledesc/BUILD.bazel +++ b/pkg/sql/catalog/tabledesc/BUILD.bazel @@ -40,6 +40,7 @@ go_library( "//pkg/sql/parser", "//pkg/sql/pgwire/pgcode", "//pkg/sql/pgwire/pgerror", + "//pkg/sql/pgwire/pgnotice", "//pkg/sql/privilege", "//pkg/sql/rowenc", "//pkg/sql/schemachanger/scpb", diff --git a/pkg/sql/catalog/tabledesc/index.go b/pkg/sql/catalog/tabledesc/index.go index fa0c386754b9..193393c23b38 100644 --- a/pkg/sql/catalog/tabledesc/index.go +++ b/pkg/sql/catalog/tabledesc/index.go @@ -142,6 +142,12 @@ func (w index) IsValidOriginIndex(originColIDs descpb.ColumnIDs) bool { return w.desc.IsValidOriginIndex(originColIDs) } +// IsHelpfulOriginIndex returns whether the index may become a helpful index for +// foreign key constraint check on the child table. +func (w index) IsHelpfulOriginIndex(originColIDs descpb.ColumnIDs) bool { + return w.desc.IsHelpfulOriginIndex(originColIDs) +} + // IsValidReferencedUniqueConstraint returns whether the index can serve as a // referenced index for a foreign key constraint with the provided set of // referencedColumnIDs. diff --git a/pkg/sql/catalog/tabledesc/validate.go b/pkg/sql/catalog/tabledesc/validate.go index 70810e025b87..fb44d80c6baa 100644 --- a/pkg/sql/catalog/tabledesc/validate.go +++ b/pkg/sql/catalog/tabledesc/validate.go @@ -28,6 +28,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/parser" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgnotice" "github.com/cockroachdb/cockroach/pkg/sql/privilege" "github.com/cockroachdb/cockroach/pkg/sql/rowenc" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" @@ -792,6 +793,53 @@ func (desc *wrapper) ValidateSelf(vea catalog.ValidationErrorAccumulator) { ValidateOnUpdate(desc, vea.Report) } +// NotVisible indexes may still be used to police unique or foreign key +// constraint check behind the scene. Hence, dropping the index might behave +// different from marking the index invisible. The following two cases are where +// NotVisible indexes might be used for constraint check: +// Case 1. if there exists unique and not-visible indexes in the given tableDesc. +// Case 2. if the given tableDesc is a child table and there exists not visible +// indexes that could be helpful for FK check in the table. +// Note that not visible indexes in a parent table may also be used for FK +// check, but they are not discussed in a separate case here because they will +// always be unique indexes (having unique indexes or constraint on a parent +// table was necessary to create the child table). +func validateNotVisibleIndex( + index catalog.Index, tableDesc catalog.TableDescriptor, +) pgnotice.Notice { + noticeStr := "not visible indexes may still be used for unique or " + + "foreign key constraint check, so the query plan may be different from " + + "dropping the index completely." + if index.IsUnique() { + return pgnotice.Newf(noticeStr) + } + + notice := tableDesc.ForeachOutboundFK(func(fk *descpb.ForeignKeyConstraint) error { + // IsHelpfulOriginIndex checks if the index columns has overlaps with the fk + // constraint origin columns. If it does, we know that this index might be + // helpful for fk checks. + if index.IsHelpfulOriginIndex(fk.OriginColumnIDs) { + return pgnotice.Newf(noticeStr) + } + return nil + }) + return notice +} + +// ValidateNotVisibleIndexWithinTable returns a notice when dropping an index +// may behave differently than marking the index invisible. +func ValidateNotVisibleIndexWithinTable(tableDesc catalog.TableDescriptor) pgnotice.Notice { + for _, idx := range tableDesc.AllIndexes() { + if idx.IsNotVisible() { + // Perform checks on every NotVisible indexes. + if notice := validateNotVisibleIndex(idx, tableDesc); notice != nil { + return notice + } + } + } + return nil +} + // ValidateOnUpdate returns an error if there is a column with both a foreign // key constraint and an ON UPDATE expression, nil otherwise. func ValidateOnUpdate(desc catalog.TableDescriptor, errReportFn func(err error)) { diff --git a/pkg/sql/create_index.go b/pkg/sql/create_index.go index 34e76cb95882..98e9c3650db2 100644 --- a/pkg/sql/create_index.go +++ b/pkg/sql/create_index.go @@ -736,12 +736,6 @@ func (n *createIndexNode) startExec(params runParams) error { ) } - if n.n.NotVisible { - return unimplemented.Newf( - "Not Visible Index", - "creating a not visible index is not supported yet") - } - // Warn against creating a non-partitioned index on a partitioned table, // which is undesirable in most cases. // Avoid the warning if we have PARTITION ALL BY as all indexes will implicitly @@ -813,6 +807,15 @@ func (n *createIndexNode) startExec(params runParams) error { return err } + // Warn against dropping an index if there exists a NotVisible index that may + // be used for constraint check behind the scene. + if notVisibleIndexNotice := tabledesc.ValidateNotVisibleIndexWithinTable(n.tableDesc); notVisibleIndexNotice != nil { + params.p.BufferClientNotice( + params.ctx, + notVisibleIndexNotice, + ) + } + // The index name may have changed as a result of // AllocateIDs(). Retrieve it for the event log below. index := n.tableDesc.Mutations[mutationIdx].GetIndex() diff --git a/pkg/sql/create_table.go b/pkg/sql/create_table.go index 71cbb9fd9189..f9889cff1bab 100644 --- a/pkg/sql/create_table.go +++ b/pkg/sql/create_table.go @@ -1784,11 +1784,7 @@ func NewTableDesc( return nil, pgerror.Newf(pgcode.DuplicateRelation, "duplicate index name: %q", d.Name) } } - if d.NotVisible { - return nil, unimplemented.Newf( - "Not Visible Index", - "creating a not visible index is not supported yet") - } + if err := validateColumnsAreAccessible(&desc, d.Columns); err != nil { return nil, err } @@ -1893,11 +1889,7 @@ func NewTableDesc( // We will add the unique constraint below. break } - if d.NotVisible { - return nil, unimplemented.Newf( - "Not Visible Index", - "creating a not visible index is not supported yet") - } + // If the index is named, ensure that the name is unique. Unnamed // indexes will be given a unique auto-generated name later on when // AllocateIDs is called. @@ -2225,6 +2217,15 @@ func NewTableDesc( return nil, onUpdateErr } + // Warn against dropping an index if there exists a NotVisible index that may + // be used for constraint check behind the scene. + if notVisibleIndexNotice := tabledesc.ValidateNotVisibleIndexWithinTable(&desc); notVisibleIndexNotice != nil { + evalCtx.ClientNoticeSender.BufferClientNotice( + ctx, + notVisibleIndexNotice, + ) + } + // AllocateIDs mutates its receiver. `return desc, desc.AllocateIDs()` // happens to work in gc, but does not work in gccgo. // diff --git a/pkg/sql/opt/exec/execbuilder/testdata/not_visible_index b/pkg/sql/opt/exec/execbuilder/testdata/not_visible_index index cc509036d0bc..976c5b5a22e4 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/not_visible_index +++ b/pkg/sql/opt/exec/execbuilder/testdata/not_visible_index @@ -1,22 +1,893 @@ # LogicTest: local +# This file will exercise these dimensions of not visible index feature: +# - Check show create table +# - Check system descriptors +# - Check show indexes +# - Check invisible index feature using EXPLAIN + +# In addition, this file should also check if we printing log notice correctly. +# - When dropping an index is not equivalent to marking an index as invisible, +# we should log a notice message. +# - Whenever a unique invisible index is created in CREATE INDEX or CREATE TABLE +# - Whenever an invisible index is created on a child table in CREATE INDEX or CREATE TABLE +# - Whenever a new foreign key constraint is added to a table with invisible index +# - Whenever they are using force index with invisible index + +statement ok +CREATE TABLE t1 (k INT PRIMARY KEY, v INT, i INT, INDEX idx_v_visible(v) VISIBLE, INDEX idx_i_invisible(i) NOT VISIBLE, FAMILY (k, v, i)) + +statement ok +CREATE INDEX idx_v_invisible ON t1(v) NOT VISIBLE + +# Test SHOW CREATE TABLE +query T +SELECT create_statement FROM [SHOW CREATE TABLE t1] +---- +CREATE TABLE public.t1 ( + k INT8 NOT NULL, + v INT8 NULL, + i INT8 NULL, + CONSTRAINT t1_pkey PRIMARY KEY (k ASC), + INDEX idx_v_visible (v ASC), + INDEX idx_i_invisible (i ASC) NOT VISIBLE, + INDEX idx_v_invisible (v ASC) NOT VISIBLE, + FAMILY fam_0_k_v_i (k, v, i) +) + +# Test SHOW INDEX, SHOW INDEXES, SHOW KEYS FROM TABLE +query TTB +SELECT index_name, column_name, visible FROM [SHOW INDEX FROM t1] ORDER BY index_name, seq_in_index +---- +idx_i_invisible i false +idx_i_invisible k false +idx_v_invisible v false +idx_v_invisible k false +idx_v_visible v true +idx_v_visible k true +t1_pkey k true +t1_pkey v true +t1_pkey i true + +query TTB +SELECT index_name, column_name, visible FROM [SHOW INDEXES FROM t1] ORDER BY index_name, seq_in_index +---- +idx_i_invisible i false +idx_i_invisible k false +idx_v_invisible v false +idx_v_invisible k false +idx_v_visible v true +idx_v_visible k true +t1_pkey k true +t1_pkey v true +t1_pkey i true + +query TTB +SELECT index_name, column_name, visible FROM [SHOW KEYS FROM t1] ORDER BY index_name, seq_in_index +---- +idx_i_invisible i false +idx_i_invisible k false +idx_v_invisible v false +idx_v_invisible k false +idx_v_visible v true +idx_v_visible k true +t1_pkey k true +t1_pkey v true +t1_pkey i true + +# Check System Descriptor +query TTBITTBBB colnames +SELECT * FROM [SHOW INDEX FROM system.descriptor] +---- +table_name index_name non_unique seq_in_index column_name direction storing implicit visible +descriptor primary false 1 id ASC false false true +descriptor primary false 2 descriptor N/A true false true + +query TT +SELECT cols.desc->>'name', cols.desc->>'notVisible' FROM ( + SELECT json_array_elements( + crdb_internal.pb_to_json('cockroach.sql.sqlbase.Descriptor', descriptor)->'table'->'indexes' + ) AS desc FROM system.descriptor WHERE id = 't1'::REGCLASS +) AS cols +---- +idx_v_visible NULL +idx_i_invisible true +idx_v_invisible true + +# Check crdb_internal.table_indexes +query TB colnames +SELECT index_name, is_visible FROM crdb_internal.table_indexes ORDER BY index_id +---- +index_name is_visible +t1_pkey true +idx_v_visible true +idx_i_invisible false +idx_v_invisible false + +# Check information_schema.statistics +query TTT colnames +SELECT index_name, column_name, is_visible FROM information_schema.statistics ORDER BY index_name, seq_in_index +---- +index_name column_name is_visible +idx_i_invisible i NO +idx_i_invisible k NO +idx_v_invisible v NO +idx_v_invisible k NO +idx_v_visible v YES +idx_v_visible k YES +t1_pkey k YES +t1_pkey v YES +t1_pkey i YES + +statement ok +DROP INDEX idx_v_visible; + +statement ok +DROP INDEX idx_i_invisible; + +statement ok +DROP INDEX idx_v_invisible; + +statement ok +DROP TABLE t1; + +# Test SHOW INDEX, SHOW INDEXES, SHOW KEYS FROM DATABASE +statement ok +CREATE DATABASE db; + +statement ok +CREATE TABLE db.t1 (k INT PRIMARY KEY, v INT, i INT, INDEX idx_v_visible(v) VISIBLE, INDEX idx_i_invisible(i) NOT VISIBLE) + +statement ok +CREATE TABLE db.t2 (k INT PRIMARY KEY, v INT, i INT, INDEX idx_v_visible(v) VISIBLE, INDEX idx_i_invisible(i) NOT VISIBLE) + +statement ok +CREATE INDEX idx_v_invisible ON db.t1(v) NOT VISIBLE + +statement ok +CREATE UNIQUE INDEX unique_idx_v_invisible ON db.t2(v) NOT VISIBLE + +query TTTB +SELECT table_name, index_name, column_name, visible FROM [SHOW INDEX FROM DATABASE db] ORDER BY table_name, index_name, seq_in_index +---- +t1 idx_i_invisible i false +t1 idx_i_invisible k false +t1 idx_v_invisible v false +t1 idx_v_invisible k false +t1 idx_v_visible v true +t1 idx_v_visible k true +t1 t1_pkey k true +t1 t1_pkey v true +t1 t1_pkey i true +t2 idx_i_invisible i false +t2 idx_i_invisible k false +t2 idx_v_visible v true +t2 idx_v_visible k true +t2 t2_pkey k true +t2 t2_pkey v true +t2 t2_pkey i true +t2 unique_idx_v_invisible v false +t2 unique_idx_v_invisible k false + +query TTTB +SELECT table_name, index_name, column_name, visible FROM [SHOW INDEXES FROM DATABASE db] ORDER BY table_name, index_name, seq_in_index +---- +t1 idx_i_invisible i false +t1 idx_i_invisible k false +t1 idx_v_invisible v false +t1 idx_v_invisible k false +t1 idx_v_visible v true +t1 idx_v_visible k true +t1 t1_pkey k true +t1 t1_pkey v true +t1 t1_pkey i true +t2 idx_i_invisible i false +t2 idx_i_invisible k false +t2 idx_v_visible v true +t2 idx_v_visible k true +t2 t2_pkey k true +t2 t2_pkey v true +t2 t2_pkey i true +t2 unique_idx_v_invisible v false +t2 unique_idx_v_invisible k false + +query TTTB +SELECT table_name, index_name, column_name, visible FROM [SHOW KEYS FROM DATABASE db] ORDER BY table_name, index_name, seq_in_index +---- +t1 idx_i_invisible i false +t1 idx_i_invisible k false +t1 idx_v_invisible v false +t1 idx_v_invisible k false +t1 idx_v_visible v true +t1 idx_v_visible k true +t1 t1_pkey k true +t1 t1_pkey v true +t1 t1_pkey i true +t2 idx_i_invisible i false +t2 idx_i_invisible k false +t2 idx_v_visible v true +t2 idx_v_visible k true +t2 t2_pkey k true +t2 t2_pkey v true +t2 t2_pkey i true +t2 unique_idx_v_invisible v false +t2 unique_idx_v_invisible k false + +statement ok +DROP DATABASE db; + +# The following tests check for not visible index feature using EXPLAIN. + +#################################################################### +# Invisible index is ignored during normal SELECT, UPDATE, DELETE. # +#################################################################### +statement ok +CREATE TABLE t1 (k INT PRIMARY KEY, v INT, INDEX idx_v_visible(v) VISIBLE) + +# idx_v_visible is selected if it is visible. +query T +EXPLAIN(OPT) SELECT * FROM t1 WHERE v = 2 +---- +scan t1@idx_v_visible + └── constraint: /2/1: [/2 - /2] + +statement ok +DROP INDEX t1@idx_v_visible + +statement ok +CREATE INDEX idx_v_invisible ON t1(v) NOT VISIBLE + +# After making idx_v_invisible invisible, SELECT ignores idx_v_invisible. +query T +EXPLAIN(OPT) SELECT v FROM t1 WHERE v = 2 +---- +select + ├── scan t1 + └── filters + └── v = 2 + +# More SELECT ignores idx_v_invisible. +query T +EXPLAIN(OPT) SELECT DISTINCT ON (v) t1 FROM t1 +---- +distinct-on + ├── project + │ ├── scan t1 + │ └── projections + │ └── ((k, v) AS k, v) + └── aggregations + └── first-agg + └── t1 + +# idx_v_invisible is ignored for normal UPDATE. +query T +EXPLAIN(OPT) UPDATE t1 SET k = 1 WHERE v > 0 +---- +update t1 + └── project + ├── select + │ ├── scan t1 + │ └── filters + │ └── v > 0 + └── projections + └── 1 + +# idx_v_invisible is ignored for normal DELETE. +query T +EXPLAIN(OPT) DELETE FROM t1 WHERE v > 0 +---- +delete t1 + └── select + ├── scan t1 + └── filters + └── v > 0 + +################################################################################## +# Check Force Index, Force Partial Index, Inverted Index, Partial Inverted Index # +################################################################################## +# Force index is still in effect, and idx_v_invisible is used. +query T +EXPLAIN(OPT) SELECT v FROM t1@idx_v_invisible WHERE v = 2 +---- +scan t1@idx_v_invisible + ├── constraint: /2/1: [/2 - /2] + └── flags: force-index=idx_v_invisible + +statement ok +DROP INDEX t1@idx_v_invisible + +statement ok +CREATE INDEX idx_v_invisible ON t1(v) WHERE v > 0 + +# Partial force index is still in effect, and idx_v_invisible is used when query filter implies predicate. +query T +EXPLAIN(OPT) SELECT * FROM t1 WHERE v > 10; +---- +scan t1@idx_v_invisible,partial + └── constraint: /2/1: [/11 - ] + +# Partial force index is still in effect, and idx_v_invisible is used when query filter implies predicate. +query T +EXPLAIN(OPT) SELECT * FROM t1@{FORCE_INDEX=idx_v_invisible} WHERE v > 10; +---- +scan t1@idx_v_invisible,partial + ├── constraint: /2/1: [/11 - ] + └── flags: force-index=idx_v_invisible + +# Partial force index is still in effect, and idx_v_invisible is not used if query filter does not imply predicate. +statement error pgcode 42809 index "idx_v_invisible" is a partial index that does not contain all the rows needed to execute this query +EXPLAIN SELECT * FROM t1@{FORCE_INDEX=idx_v_invisible} WHERE v < 0; + +statement ok +DROP TABLE t1 + +################################################################################## +# Check Invisible Inverted Index and Partial Inverted Index. +################################################################################## +statement ok +CREATE TABLE t1 (id INT, data JSONB, geom GEOMETRY, INVERTED INDEX idx_geom_visible(geom) VISIBLE); + +# idx_geom_visible is chosen because it is visible. +query T +EXPLAIN(OPT) SELECT * FROM t1 WHERE st_covers(geom, 'LINESTRING ( 0 0, 0 2 )'::geometry) +---- +select + ├── index-join t1 + │ └── inverted-filter + │ ├── inverted expression: /7 + │ │ ├── tight: false, unique: false + │ │ └── union spans + │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x01", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x01"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x04", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x04"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x10", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x10"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00@", "B\xfd\x10\x00\x00\x00\x00\x00\x00@"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x01\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x01\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x04\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x04\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x10\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x10\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00@\x00", "B\xfd\x10\x00\x00\x00\x00\x00@\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x01\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x01\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x04\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x04\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x10\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x10\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x00@\x00\x00", "B\xfd\x10\x00\x00\x00\x00@\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x01\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x01\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x04\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x04\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x00\x10\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x10\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x00@\x00\x00\x00", "B\xfd\x10\x00\x00\x00@\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x01\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x01\x00\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x04\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x04\x00\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x00\x10\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x10\x00\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x00@\x00\x00\x00\x00", "B\xfd\x10\x00\x00@\x00\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x01\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x01\x00\x00\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x04\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x04\x00\x00\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x00\x10\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x10\x00\x00\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x00@\x00\x00\x00\x00\x00", "B\xfd\x10\x00@\x00\x00\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x01\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x01\x00\x00\x00\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x04\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x04\x00\x00\x00\x00\x00\x00"] + │ │ ├── ["B\xfd\x10\x10\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x10\x00\x00\x00\x00\x00\x00"] + │ │ ├── ["B\xfd\x10@\x00\x00\x00\x00\x00\x00", "B\xfd\x10@\x00\x00\x00\x00\x00\x00"] + │ │ ├── ["B\xfd\x11\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x11\x00\x00\x00\x00\x00\x00\x00"] + │ │ └── ["B\xfd\x14\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x14\x00\x00\x00\x00\x00\x00\x00"] + │ ├── pre-filterer expression + │ │ └── st_coveredby('0102000000020000000000000000000000000000000000000000000000000000000000000000000040', geom) + │ └── scan t1@idx_geom_visible + │ └── inverted constraint: /7/4 + │ └── spans + │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] + │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x01", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x01"] + │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x04", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x04"] + │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x10", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x10"] + │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00@", "B\xfd\x10\x00\x00\x00\x00\x00\x00@"] + │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x01\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x01\x00"] + │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x04\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x04\x00"] + │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x10\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x10\x00"] + │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00@\x00", "B\xfd\x10\x00\x00\x00\x00\x00@\x00"] + │ ├── ["B\xfd\x10\x00\x00\x00\x00\x01\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x01\x00\x00"] + │ ├── ["B\xfd\x10\x00\x00\x00\x00\x04\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x04\x00\x00"] + │ ├── ["B\xfd\x10\x00\x00\x00\x00\x10\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x10\x00\x00"] + │ ├── ["B\xfd\x10\x00\x00\x00\x00@\x00\x00", "B\xfd\x10\x00\x00\x00\x00@\x00\x00"] + │ ├── ["B\xfd\x10\x00\x00\x00\x01\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x01\x00\x00\x00"] + │ ├── ["B\xfd\x10\x00\x00\x00\x04\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x04\x00\x00\x00"] + │ ├── ["B\xfd\x10\x00\x00\x00\x10\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x10\x00\x00\x00"] + │ ├── ["B\xfd\x10\x00\x00\x00@\x00\x00\x00", "B\xfd\x10\x00\x00\x00@\x00\x00\x00"] + │ ├── ["B\xfd\x10\x00\x00\x01\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x01\x00\x00\x00\x00"] + │ ├── ["B\xfd\x10\x00\x00\x04\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x04\x00\x00\x00\x00"] + │ ├── ["B\xfd\x10\x00\x00\x10\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x10\x00\x00\x00\x00"] + │ ├── ["B\xfd\x10\x00\x00@\x00\x00\x00\x00", "B\xfd\x10\x00\x00@\x00\x00\x00\x00"] + │ ├── ["B\xfd\x10\x00\x01\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x01\x00\x00\x00\x00\x00"] + │ ├── ["B\xfd\x10\x00\x04\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x04\x00\x00\x00\x00\x00"] + │ ├── ["B\xfd\x10\x00\x10\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x10\x00\x00\x00\x00\x00"] + │ ├── ["B\xfd\x10\x00@\x00\x00\x00\x00\x00", "B\xfd\x10\x00@\x00\x00\x00\x00\x00"] + │ ├── ["B\xfd\x10\x01\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x01\x00\x00\x00\x00\x00\x00"] + │ ├── ["B\xfd\x10\x04\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x04\x00\x00\x00\x00\x00\x00"] + │ ├── ["B\xfd\x10\x10\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x10\x00\x00\x00\x00\x00\x00"] + │ ├── ["B\xfd\x10@\x00\x00\x00\x00\x00\x00", "B\xfd\x10@\x00\x00\x00\x00\x00\x00"] + │ ├── ["B\xfd\x11\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x11\x00\x00\x00\x00\x00\x00\x00"] + │ └── ["B\xfd\x14\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x14\x00\x00\x00\x00\x00\x00\x00"] + └── filters + └── st_covers(geom, '0102000000020000000000000000000000000000000000000000000000000000000000000000000040') + +statement ok +DROP INDEX idx_geom_visible + +statement ok +CREATE INDEX idx_geom_invisible ON t1 USING GIN (geom) NOT VISIBLE; + +# Check invisible inverted index: ignored idx_geom_invisible. +query T +EXPLAIN(OPT) SELECT * FROM t1 WHERE st_covers(geom, 'LINESTRING ( 0 0, 0 2 )'::geometry) +---- +select + ├── scan t1 + └── filters + └── st_covers(geom, '0102000000020000000000000000000000000000000000000000000000000000000000000000000040') + +# Check JSONB +statement ok +CREATE INVERTED INDEX idx_data_invisible ON t1(data) WHERE id > 10 NOT VISIBLE + +# Check invisible inverted partial index: ignored idx_data_invisible. +query T +EXPLAIN(OPT) SELECT * FROM t1 WHERE data @> '{"foo": "1"}' AND id > 10 +---- +select + ├── scan t1 + │ └── partial index predicates + │ └── idx_data_invisible: filters + │ └── id > 10 + └── filters + ├── data @> '{"foo": "1"}' + └── id > 10 + +statement ok +DROP TABLE t1 + +################################################################################# +# These tests check for some other cases called from buildDataSource (buildJoin, +# buildInputForUpdate, buildFromWithLateral). +################################################################################# +statement ok +CREATE TABLE t4 (p INT PRIMARY KEY, other INT NOT NULL, INDEX idx_t4_invisible(other) NOT VISIBLE) + +statement ok +CREATE TABLE t5 (a INT PRIMARY KEY, other INT) + +statement ok +CREATE INDEX idx_t5_invisible ON t5(other) NOT VISIBLE + +# ignored idx_t4_invisible, idx_t5_invisible +query T +EXPLAIN(OPT) SELECT * FROM t4, t5 WHERE t4.other = t5.other +---- +inner-join (hash) + ├── scan t4 + ├── scan t5 + └── filters + └── t4.other = t5.other + +# called buildFromTablesRightDeep, buildJoin +# ignored idx_t4_invisible, idx_t5_invisible +query T +EXPLAIN(OPT) SELECT * FROM t4 AS a JOIN t5 as b USING(other) ORDER BY other +---- +project + └── inner-join (merge) + ├── sort + │ └── scan t4 [as=a] + ├── sort + │ └── scan t5 [as=b] + └── filters (true) + +# called buildFromWithLateral +# ignored idx_t4_invisible, idx_t5_invisible +query T +EXPLAIN(OPT) SELECT * FROM t4, LATERAL (SELECT * FROM t5 WHERE other > 0) WHERE t4.other > 0; +---- +inner-join (cross) + ├── select + │ ├── scan t4 + │ └── filters + │ └── t4.other > 0 + ├── select + │ ├── scan t5 + │ └── filters + │ └── t5.other > 0 + └── filters (true) + +statement ok +DROP TABLE t4 + +statement ok +DROP TABLE t5 + + +######################################################################################## +# Invisible index is still used to check for uniqueness. INSERT...ON CONFLICT, UPSERT. # +######################################################################################## +statement ok +CREATE TABLE t1 (k INT PRIMARY KEY, v INT) + +statement ok +CREATE UNIQUE INDEX idx_v_unique_invisible ON t1(v) NOT VISIBLE + +# idx_v_unique_invisible is ignored under normal condition. +query T +EXPLAIN(OPT) SELECT * FROM t1 WHERE v > 0 +---- +select + ├── scan t1 + └── filters + └── v > 0 + +# idx_v_unique_invisible is used to check uniqueness. +query T +EXPLAIN(OPT) INSERT INTO t1 VALUES (1, 2) ON CONFLICT DO NOTHING +---- +insert t1 + ├── arbiter indexes: t1_pkey idx_v_unique_invisible + └── anti-join (lookup t1@idx_v_unique_invisible) + ├── lookup columns are key + ├── anti-join (cross) + │ ├── values + │ │ └── (1, 2) + │ ├── scan t1 + │ │ └── constraint: /7: [/1 - /1] + │ └── filters (true) + └── filters (true) + +# idx_v_unique_invisible is used to check uniqueness. +query T +EXPLAIN(OPT) INSERT INTO t1 VALUES (1, 2) ON CONFLICT(v) DO UPDATE SET k = t1.v +---- +upsert t1 + ├── arbiter indexes: idx_v_unique_invisible + └── project + ├── left-join (cross) + │ ├── values + │ │ └── (1, 2) + │ ├── scan t1@idx_v_unique_invisible + │ │ ├── constraint: /8: [/2 - /2] + │ │ └── flags: disabled not visible index feature + │ └── filters (true) + └── projections + └── CASE WHEN k IS NULL THEN column1 ELSE v END + +# UPSERT uses primary index to check uniqueness, so idx_v_unique_invisible is not useful. +query T +EXPLAIN(OPT) UPSERT INTO t1(k, v) VALUES (1, 2) +---- +upsert t1 + ├── arbiter indexes: t1_pkey + └── left-join (cross) + ├── values + │ └── (1, 2) + ├── scan t1 + │ └── constraint: /7: [/1 - /1] + └── filters (true) + +statement ok +DROP TABLE t1 + +########################################################################################### +# Invisible index is still used to check for FK constraint. +# - When parent deletes or update +# - When child inserts, upserts, or update +########################################################################################### +statement ok +CREATE TABLE parent (p INT PRIMARY KEY) + +statement ok +CREATE TABLE child (c INT PRIMARY KEY, p INT NOT NULL REFERENCES parent(p), INDEX c_idx_invisible (p) NOT VISIBLE) + +# Part 1: When parent deletes or update, invisible indexes on the child table will be used. +# c_idx_invisible is invisible when no FK is involved (delete on a child table requires no FK check). +query T +EXPLAIN(OPT) DELETE FROM child WHERE p = 4 +---- +delete child + └── select + ├── scan child + └── filters + └── p = 4 + +# c_idx_invisible is used to perform constraint-check (delete on a parent table requires FK check). +query T +EXPLAIN(OPT) DELETE FROM parent where p = 2 +---- +delete parent + ├── scan parent + │ └── constraint: /4: [/2 - /2] + └── f-k-checks + └── f-k-checks-item: child(p) -> parent(p) + └── semi-join (lookup child@c_idx_invisible) + ├── with-scan &1 + └── filters (true) + +# c_idx_invisible is used to perform constraint-check (update on a parent table requires FK check). +query T +EXPLAIN(OPT) UPDATE parent SET p = p+1 +---- +update parent + ├── project + │ ├── scan parent + │ └── projections + │ └── parent.p + 1 + └── f-k-checks + └── f-k-checks-item: child(p) -> parent(p) + └── project + └── inner-join (hash) + ├── except-all + │ ├── with-scan &1 + │ └── with-scan &1 + ├── distinct-on + │ └── scan child@c_idx_invisible + │ └── flags: disabled not visible index feature + └── filters + └── p = child.p + +statement ok +DROP TABLE child + +statement ok +DROP TABLE parent + +# Part 2: When child insert, upsert, update, invisible indexes on the parent table will be used. +statement ok +CREATE TABLE parent (p INT PRIMARY KEY, other INT) + +statement ok +CREATE UNIQUE INDEX u_idx_invisible ON parent(other) NOT VISIBLE + +statement ok +CREATE TABLE child (c INT PRIMARY KEY, p INT NOT NULL REFERENCES parent(other)) + +# u_idx_invisible is invisible when no FK check is involved (select on a table requires no FK check). +query T +EXPLAIN(OPT) SELECT * FROM parent WHERE other > 0 +---- +select + ├── scan parent + └── filters + └── other > 0 + +# u_idx_invisible is used for FK check (insert on a child table requires FK check). +query T +EXPLAIN(OPT) INSERT INTO child VALUES (200, 1) +---- +insert child + ├── values + │ └── (200, 1) + └── f-k-checks + └── f-k-checks-item: child(p) -> parent(other) + └── anti-join (lookup parent@u_idx_invisible) + ├── lookup columns are key + ├── with-scan &1 + └── filters (true) + +# u_idx_invisible is used for FK check (upsert on a child table requires FK check). +query T +EXPLAIN(OPT) UPSERT INTO child VALUES (200, 1) +---- +upsert child + ├── values + │ └── (200, 1) + └── f-k-checks + └── f-k-checks-item: child(p) -> parent(other) + └── anti-join (lookup parent@u_idx_invisible) + ├── lookup columns are key + ├── with-scan &1 + └── filters (true) + +statement ok +DROP TABLE child + +statement ok +DROP TABLE parent + +################################################################################################### +# Invisible index is still used to check for FK constraint with ON CASCADE, SET DEFAULT, SET NULL. +# - When parent deletes or update +# - Since EXPLAIN does not show here. We will check the scan flag output under `opt` testdata to confirm. +################################################################################################### statement ok -CREATE TABLE t1 (k INT PRIMARY KEY, v INT, geom GEOMETRY) +CREATE TABLE parent (p INT PRIMARY KEY) -statement error pq: unimplemented: creating a not visible index is not supported yet -CREATE INDEX idx_k_invisible ON t1(v) NOT VISIBLE +statement ok +CREATE TABLE child_delete (c INT PRIMARY KEY, p INT NOT NULL REFERENCES parent(p) ON DELETE CASCADE, INDEX c_delete_idx_invisible (p) NOT VISIBLE) + +statement ok +CREATE TABLE child_update (c INT PRIMARY KEY, p INT NOT NULL REFERENCES parent(p) ON UPDATE CASCADE, INDEX c_update_idx_invisible (p) NOT VISIBLE) + +query TTB +SELECT index_name, column_name, visible FROM [SHOW INDEX FROM parent] ORDER BY index_name, seq_in_index +---- +parent_pkey p true + +query TTB +SELECT index_name, column_name, visible FROM [SHOW INDEX FROM child_delete] ORDER BY index_name, seq_in_index +---- +c_delete_idx_invisible p false +c_delete_idx_invisible c false +child_delete_pkey c true +child_delete_pkey p true + +query TTB +SELECT index_name, column_name, visible FROM [SHOW INDEX FROM child_update] ORDER BY index_name, seq_in_index +---- +c_update_idx_invisible p false +c_update_idx_invisible c false +child_update_pkey c true +child_update_pkey p true + +# c_update_idx_invisible is used for constraint check (delete on a parent table requires FK check). +query T +EXPLAIN(OPT) DELETE FROM parent where p = 2 +---- +delete parent + ├── scan parent + │ └── constraint: /4: [/2 - /2] + └── f-k-checks + └── f-k-checks-item: child_update(p) -> parent(p) + └── semi-join (lookup child_update@c_update_idx_invisible) + ├── with-scan &1 + └── filters (true) + +# c_delete_idx_invisible is used for constraint check (update on a parent table requires FK check). +query T +EXPLAIN(OPT) UPDATE parent SET p = p+1 +---- +update parent + ├── project + │ ├── scan parent + │ └── projections + │ └── parent.p + 1 + └── f-k-checks + └── f-k-checks-item: child_delete(p) -> parent(p) + └── project + └── inner-join (hash) + ├── except-all + │ ├── with-scan &1 + │ └── with-scan &1 + ├── distinct-on + │ └── scan child_delete@c_delete_idx_invisible + │ └── flags: disabled not visible index feature + └── filters + └── p = child_delete.p + +statement ok +DROP TABLE child_delete + +statement ok +DROP TABLE child_update + +statement ok +DROP TABLE parent + +############################################################################ +# We should log notices +# when dropping an index might be different from marking an index invisible +# - Invisible unique indexes are created +# - Invisible indexes are created in a child table +# - Invisible indexes are used with Force Index +############################################################################ +# Creating an invisble secondary index in CREATE TABLE does not log warning if it is not in a child table. +query T noticetrace +CREATE TABLE t1 (p INT PRIMARY KEY, INDEX idx_invisible (p) NOT VISIBLE) +---- + +# Creating an invisble secondary index in CREATE INDEX does not log warning. +query T noticetrace +CREATE INDEX t1_idx_invisible_unique ON t1(p) NOT VISIBLE +---- + +# Creating an invisble unique index in CREATE INDEX does log warning. +query T noticetrace +CREATE UNIQUE INDEX t1_idx_invisible ON t1(p) NOT VISIBLE +---- +NOTICE: not visible indexes may still be used for unique or foreign key constraint check, so the query plan may be different from dropping the index completely. + +# Creating an invisble unique index in CREATE TABLE does log warning. +# TODO (wenyihu6): try to find workaround to test for creating invisible unique indexes in CREATE TABLE +# (confirmed that the following statement works on a cluster) +# query T noticetrace +# CREATE TABLE t2 (p INT PRIMARY KEY, UNIQUE INDEX c_idx_invisible (p) NOT VISIBLE) +# ---- + +# Creating an invisble unique index in CREATE TABLE does log warning. +# (Confirmed that the following statement works on a cluster and only prints once.) +# query T noticetrace +# CREATE TABLE t3 (p INT PRIMARY KEY, UNIQUE INDEX c_idx_invisible1 (p) NOT VISIBLE, UNIQUE INDEX c_idx_invisible2 (p) NOT VISIBLE) +# ---- + +statement ok +DROP TABLE t1 + +############################################################################ +# Creating an invisble secondary index in CREATE TABLE does not log warning if it is not in a child table. +query T noticetrace +CREATE TABLE parent (p INT PRIMARY KEY, INDEX p_idx_invisible_1 (p) NOT VISIBLE) +---- + +# Creating an invisible index in a child table does log warning and only once. +query T noticetrace +CREATE TABLE child (c INT PRIMARY KEY, p INT NOT NULL REFERENCES parent(p), INDEX c_idx1_invisible (p) NOT VISIBLE, INDEX c_idx2_invisible (p) NOT VISIBLE) +---- +NOTICE: not visible indexes may still be used for unique or foreign key constraint check, so the query plan may be different from dropping the index completely. + +# Creating an invisible index in a child table does log warning. +query T noticetrace +CREATE INDEX c_idx3_invisible ON child(p) NOT VISIBLE +---- +NOTICE: not visible indexes may still be used for unique or foreign key constraint check, so the query plan may be different from dropping the index completely. + +# Creating a unique invisible index in a child table does log warning and only once. +query T noticetrace +CREATE UNIQUE INDEX c_idx_invisible_unique ON child(p) NOT VISIBLE +---- +NOTICE: not visible indexes may still be used for unique or foreign key constraint check, so the query plan may be different from dropping the index completely. + +# Creating an invisble secondary index in a parent table still does not log warning. +query T noticetrace +CREATE INDEX p_idx_invisible_2 ON parent(p) NOT VISIBLE +---- + +statement ok +DROP TABLE child + +statement ok +DROP TABLE parent -statement error pq: unimplemented: creating a not visible index is not supported yet -CREATE INVERTED INDEX idx_c_partial_invisible ON t1(geom) WHERE k >= v AND v = 3 NOT VISIBLE +############################################################################ +query T noticetrace +CREATE TABLE parent (p INT PRIMARY KEY) +---- -statement error pq: unimplemented: creating a not visible index is not supported yet -CREATE UNIQUE INDEX unique_idx ON t1(k) NOT VISIBLE +query T noticetrace +CREATE TABLE child (c INT PRIMARY KEY) +---- -statement error pq: unimplemented: creating a not visible index is not supported yet -CREATE TABLE t1_invisible (b INT, INDEX foo (b) WHERE b > 3 NOT VISIBLE) +# Adding a FK constraint to a table without invisible indexes does not log warning. +query T noticetrace +ALTER TABLE child ADD CONSTRAINT p_fk_1 FOREIGN KEY (c) REFERENCES parent(p); +---- -statement error pq: unimplemented: creating a not visible index is not supported yet -CREATE TABLE t2_invisible (k INT PRIMARY KEY, v INT, i INT, p INT, INDEX idx_v_invisible (v) NOT VISIBLE) +# Adding invisible indexes to a child table logs warning. +query T noticetrace +CREATE INDEX c_idx_invisible_2 ON child(c) NOT VISIBLE +---- +NOTICE: not visible indexes may still be used for unique or foreign key constraint check, so the query plan may be different from dropping the index completely. -statement error pq: unimplemented: creating a not visible index is not supported yet -CREATE TABLE t3_invisible (b INT, UNIQUE INDEX foo (b) WHERE b > 3 NOT VISIBLE) +# Adding a FK constraint to a table with invisible indexes should log warning. +query T noticetrace +ALTER TABLE child ADD CONSTRAINT p_fk_2 FOREIGN KEY (c) REFERENCES parent(p); +---- +NOTICE: not visible indexes may still be used for unique or foreign key constraint check, so the query plan may be different from dropping the index completely. + +# Adding multiple FK constraints in a table with invisible indexes should log only once. +query T noticetrace +ALTER TABLE child ADD CONSTRAINT fk1 FOREIGN KEY (c) REFERENCES parent(p), ADD CONSTRAINT fk2 FOREIGN KEY (c) REFERENCES child(c); +---- +NOTICE: not visible indexes may still be used for unique or foreign key constraint check, so the query plan may be different from dropping the index completely. + +# Having multiple FK constraints in CREATE TABLE with invisible indexes should log only once. +query T noticetrace +CREATE TABLE child2 (c INT PRIMARY KEY, p INT NOT NULL REFERENCES parent(p) ON DELETE CASCADE, h INT NOT NULL REFERENCES child(c) ON DELETE CASCADE, INDEX c1_idx_invisible (p) NOT VISIBLE) +---- +NOTICE: not visible indexes may still be used for unique or foreign key constraint check, so the query plan may be different from dropping the index completely. + +# Having FK constraints and multiple unique invisible indexes in CREATE TABLE should log only once. +# (Confirmed that the following statement works on a cluster and only prints once.) +# query T noticetrace +# CREATE TABLE child3 (c INT PRIMARY KEY, p INT NOT NULL REFERENCES parent(p) ON DELETE CASCADE, h INT NOT NULL REFERENCES child(c) ON DELETE CASCADE, UNIQUE INDEX c1_idx_invisible (p) NOT VISIBLE) +# ---- + +# 6. Force index with invisible index should also throw a notice + +statement ok +DROP TABLE child2 + +statement ok +DROP TABLE child + +statement ok +DROP TABLE parent diff --git a/pkg/sql/opt/optbuilder/select.go b/pkg/sql/opt/optbuilder/select.go index 9ce7d79aa6a6..77a5a5dde07c 100644 --- a/pkg/sql/opt/optbuilder/select.go +++ b/pkg/sql/opt/optbuilder/select.go @@ -527,6 +527,10 @@ func (b *Builder) buildScan( if tab.Index(i).Name() == tree.Name(indexFlags.Index) || tab.Index(i).ID() == cat.StableID(indexFlags.IndexID) { idx = i + + //if tab.Index(i).IsNotVisible() { + // // TODO(wenyihu6): Find a way to give notice here is force index is not visible. + //} break } } diff --git a/pkg/sql/opt/xform/scan_index_iter.go b/pkg/sql/opt/xform/scan_index_iter.go index b4d8950b97ed..d7763d0403d9 100644 --- a/pkg/sql/opt/xform/scan_index_iter.go +++ b/pkg/sql/opt/xform/scan_index_iter.go @@ -219,12 +219,19 @@ func (it *scanIndexIter) ForEachStartingAfter(ord int, f enumerateIndexFunc) { continue } - // If we are forcing a specific index, ignore all other indexes. - if it.scanPrivate.Flags.ForceIndex && ord != it.scanPrivate.Flags.Index { - continue - } - index := it.tabMeta.Table.Index(ord) + if it.scanPrivate.Flags.ForceIndex { + // If we are forcing a specific index, ignore all other indexes. + if ord != it.scanPrivate.Flags.Index { + continue + } + } else { + // If we are not forcing any specific index and not visible index feature is + // enabled here, ignore not visible indexes only. + if index.IsNotVisible() && !it.scanPrivate.Flags.DisableNotVisibleIndex { + continue + } + } // Skip over inverted indexes if rejectInvertedIndexes is set. if it.hasRejectFlags(rejectInvertedIndexes) && index.IsInverted() {