From abaf880bb7e78612546b18c85c7469a372a3070a Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Tue, 12 Jun 2018 18:30:20 +0200 Subject: [PATCH] sql: add proper support for hidden columns Prior to this patch, CockroachDB already had full support for marking arbitrary columns as "hidden" internally: a hidden column can be used for queries but does not get automatically selected by a `*` in a SELECT clause, and does not get automatically inserted/upserted into when INSERT/UPSERT do not list any columns. However, no control was given to users to determine which columns would be hidden. The only column that could be hidden in practice was the implicit `rowid` column created when CREATE does not specify a primary key. This patch complements the existing thorough support for hidden column by properly exposing the "hidden" attribute through SQL: - any column can now be marked as hidden in CREATE using the column attribute `NOT VISIBLE`. - the attribute can now be changed with `ALTER TABLE .. ALTER COLUMN .. SET [NOT] VISIBLE`. This change is further motivated by the need to annotate non-PK columns in `pg_catalog` vtables, for compatibility with PostgreSQL (this will be actually done in a later patch). The output of `SHOW CREATE` now displays all columns with their optional `NOT VISIBLE` attribute, instead of skipping over hidden column as previously. Note: the choice for the syntax "NOT VISIBLE" as opposed to "HIDDEN" is mandated by the fact that the first word of an attribute must be a reserved keyword, and adding reserved keywords is disruptive. This way, the patch is properly backward-compatible. Release note (sql change): clients can now set the visibility of a column using the `NOT VISIBLE` attribute in `CREATE TABLE` or `SET VISIBLE` / `SET NOT VISIBLE` in `ALTER TABLE .. ALTER COLUMN`. --- docs/generated/sql/bnf/alter_column.bnf | 8 ++ docs/generated/sql/bnf/alter_table.bnf | 4 +- docs/generated/sql/bnf/col_qualification.bnf | 2 + docs/generated/sql/bnf/stmt_block.bnf | 4 + pkg/cli/dump_test.go | 4 + pkg/cli/testdata/dump/flags | 4 + pkg/cli/testdata/dump/identifiers | 2 + pkg/cli/testdata/dump/inverted_index | 2 + pkg/cli/testdata/dump/multiple | 8 ++ pkg/cli/testdata/dump/reference_order | 4 + pkg/cli/testdata/dump/row | 3 +- pkg/sql/alter_table.go | 17 +++ pkg/sql/create_table.go | 6 +- pkg/sql/create_view.go | 6 +- pkg/sql/insert.go | 3 +- .../logictest/testdata/logic_test/alias_types | 12 ++- pkg/sql/logictest/testdata/logic_test/array | 24 +++-- .../testdata/logic_test/check_constraints | 36 ++++--- .../testdata/logic_test/collatedstring | 8 +- .../logictest/testdata/logic_test/computed | 78 ++++++++------ .../testdata/logic_test/create_statements | 8 ++ pkg/sql/logictest/testdata/logic_test/fk | 50 +++++---- .../testdata/logic_test/hidden_columns | 102 ++++++++++++++++++ .../testdata/logic_test/name_escapes | 2 + .../testdata/logic_test/schema_change_in_txn | 16 +-- pkg/sql/logictest/testdata/logic_test/table | 2 + pkg/sql/opt/testutils/testcat/create_table.go | 2 +- pkg/sql/parser/parse_test.go | 4 + pkg/sql/parser/sql.y | 20 +++- pkg/sql/sem/tree/alter_table.go | 24 +++++ pkg/sql/sem/tree/create.go | 10 ++ pkg/sql/show_create.go | 12 +-- pkg/sql/show_test.go | 14 +++ pkg/sql/sqlbase/structured.go | 3 + pkg/sql/sqlbase/table.go | 1 + 35 files changed, 392 insertions(+), 113 deletions(-) create mode 100644 pkg/sql/logictest/testdata/logic_test/hidden_columns diff --git a/docs/generated/sql/bnf/alter_column.bnf b/docs/generated/sql/bnf/alter_column.bnf index 1046df0d371e..358240bb3b28 100644 --- a/docs/generated/sql/bnf/alter_column.bnf +++ b/docs/generated/sql/bnf/alter_column.bnf @@ -3,6 +3,10 @@ alter_onetable_stmt ::= | 'ALTER' 'TABLE' table_name 'ALTER' 'COLUMN' column_name 'DROP' 'DEFAULT' | 'ALTER' 'TABLE' table_name 'ALTER' column_name 'SET' 'DEFAULT' a_expr | 'ALTER' 'TABLE' table_name 'ALTER' column_name 'DROP' 'DEFAULT' + | 'ALTER' 'TABLE' table_name 'ALTER' 'COLUMN' column_name 'SET' 'NOT' 'VISIBLE' + | 'ALTER' 'TABLE' table_name 'ALTER' column_name 'SET' 'NOT' 'VISIBLE' + | 'ALTER' 'TABLE' table_name 'ALTER' 'COLUMN' column_name 'SET' 'VISIBLE' + | 'ALTER' 'TABLE' table_name 'ALTER' column_name 'SET' 'VISIBLE' | 'ALTER' 'TABLE' table_name 'ALTER' 'COLUMN' column_name 'DROP' 'NOT' 'NULL' | 'ALTER' 'TABLE' table_name 'ALTER' column_name 'DROP' 'NOT' 'NULL' | 'ALTER' 'TABLE' table_name 'ALTER' 'COLUMN' column_name 'DROP' 'STORED' @@ -27,6 +31,10 @@ alter_onetable_stmt ::= | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'ALTER' 'COLUMN' column_name 'DROP' 'DEFAULT' | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'ALTER' column_name 'SET' 'DEFAULT' a_expr | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'ALTER' column_name 'DROP' 'DEFAULT' + | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'ALTER' 'COLUMN' column_name 'SET' 'NOT' 'VISIBLE' + | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'ALTER' column_name 'SET' 'NOT' 'VISIBLE' + | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'ALTER' 'COLUMN' column_name 'SET' 'VISIBLE' + | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'ALTER' column_name 'SET' 'VISIBLE' | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'ALTER' 'COLUMN' column_name 'DROP' 'NOT' 'NULL' | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'ALTER' column_name 'DROP' 'NOT' 'NULL' | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'ALTER' 'COLUMN' column_name 'DROP' 'STORED' diff --git a/docs/generated/sql/bnf/alter_table.bnf b/docs/generated/sql/bnf/alter_table.bnf index 7f6bc5c4de66..94b387724f65 100644 --- a/docs/generated/sql/bnf/alter_table.bnf +++ b/docs/generated/sql/bnf/alter_table.bnf @@ -1,3 +1,3 @@ alter_onetable_stmt ::= - 'ALTER' 'TABLE' table_name ( ( ( 'ADD' ( column_name typename col_qual_list ) | 'ADD' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DEFAULT' a_expr | 'DROP' 'DEFAULT' ) | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'NOT' 'NULL' | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'STORED' | 'DROP' ( 'COLUMN' | ) 'IF' 'EXISTS' column_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' ( 'COLUMN' | ) column_name ( 'CASCADE' | 'RESTRICT' | ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DATA' | ) 'TYPE' typename ( 'COLLATE' collation_name | ) ( 'USING' a_expr | ) | 'ADD' ( 'CONSTRAINT' constraint_name constraint_elem | constraint_elem ) | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' 'CONSTRAINT' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | partition_by | 'INJECT' 'STATISTICS' a_expr ) ) ( ( ',' ( 'ADD' ( column_name typename col_qual_list ) | 'ADD' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DEFAULT' a_expr | 'DROP' 'DEFAULT' ) | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'NOT' 'NULL' | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'STORED' | 'DROP' ( 'COLUMN' | ) 'IF' 'EXISTS' column_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' ( 'COLUMN' | ) column_name ( 'CASCADE' | 'RESTRICT' | ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DATA' | ) 'TYPE' typename ( 'COLLATE' collation_name | ) ( 'USING' a_expr | ) | 'ADD' ( 'CONSTRAINT' constraint_name constraint_elem | constraint_elem ) | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' 'CONSTRAINT' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | partition_by | 'INJECT' 'STATISTICS' a_expr ) ) )* ) - | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name ( ( ( 'ADD' ( column_name typename col_qual_list ) | 'ADD' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DEFAULT' a_expr | 'DROP' 'DEFAULT' ) | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'NOT' 'NULL' | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'STORED' | 'DROP' ( 'COLUMN' | ) 'IF' 'EXISTS' column_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' ( 'COLUMN' | ) column_name ( 'CASCADE' | 'RESTRICT' | ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DATA' | ) 'TYPE' typename ( 'COLLATE' collation_name | ) ( 'USING' a_expr | ) | 'ADD' ( 'CONSTRAINT' constraint_name constraint_elem | constraint_elem ) | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' 'CONSTRAINT' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | partition_by | 'INJECT' 'STATISTICS' a_expr ) ) ( ( ',' ( 'ADD' ( column_name typename col_qual_list ) | 'ADD' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DEFAULT' a_expr | 'DROP' 'DEFAULT' ) | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'NOT' 'NULL' | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'STORED' | 'DROP' ( 'COLUMN' | ) 'IF' 'EXISTS' column_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' ( 'COLUMN' | ) column_name ( 'CASCADE' | 'RESTRICT' | ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DATA' | ) 'TYPE' typename ( 'COLLATE' collation_name | ) ( 'USING' a_expr | ) | 'ADD' ( 'CONSTRAINT' constraint_name constraint_elem | constraint_elem ) | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' 'CONSTRAINT' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | partition_by | 'INJECT' 'STATISTICS' a_expr ) ) )* ) + 'ALTER' 'TABLE' table_name ( ( ( 'ADD' ( column_name typename col_qual_list ) | 'ADD' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DEFAULT' a_expr | 'DROP' 'DEFAULT' ) | 'ALTER' ( 'COLUMN' | ) column_name 'SET' 'NOT' 'VISIBLE' | 'ALTER' ( 'COLUMN' | ) column_name 'SET' 'VISIBLE' | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'NOT' 'NULL' | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'STORED' | 'DROP' ( 'COLUMN' | ) 'IF' 'EXISTS' column_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' ( 'COLUMN' | ) column_name ( 'CASCADE' | 'RESTRICT' | ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DATA' | ) 'TYPE' typename ( 'COLLATE' collation_name | ) ( 'USING' a_expr | ) | 'ADD' ( 'CONSTRAINT' constraint_name constraint_elem | constraint_elem ) | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' 'CONSTRAINT' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | partition_by | 'INJECT' 'STATISTICS' a_expr ) ) ( ( ',' ( 'ADD' ( column_name typename col_qual_list ) | 'ADD' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DEFAULT' a_expr | 'DROP' 'DEFAULT' ) | 'ALTER' ( 'COLUMN' | ) column_name 'SET' 'NOT' 'VISIBLE' | 'ALTER' ( 'COLUMN' | ) column_name 'SET' 'VISIBLE' | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'NOT' 'NULL' | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'STORED' | 'DROP' ( 'COLUMN' | ) 'IF' 'EXISTS' column_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' ( 'COLUMN' | ) column_name ( 'CASCADE' | 'RESTRICT' | ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DATA' | ) 'TYPE' typename ( 'COLLATE' collation_name | ) ( 'USING' a_expr | ) | 'ADD' ( 'CONSTRAINT' constraint_name constraint_elem | constraint_elem ) | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' 'CONSTRAINT' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | partition_by | 'INJECT' 'STATISTICS' a_expr ) ) )* ) + | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name ( ( ( 'ADD' ( column_name typename col_qual_list ) | 'ADD' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DEFAULT' a_expr | 'DROP' 'DEFAULT' ) | 'ALTER' ( 'COLUMN' | ) column_name 'SET' 'NOT' 'VISIBLE' | 'ALTER' ( 'COLUMN' | ) column_name 'SET' 'VISIBLE' | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'NOT' 'NULL' | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'STORED' | 'DROP' ( 'COLUMN' | ) 'IF' 'EXISTS' column_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' ( 'COLUMN' | ) column_name ( 'CASCADE' | 'RESTRICT' | ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DATA' | ) 'TYPE' typename ( 'COLLATE' collation_name | ) ( 'USING' a_expr | ) | 'ADD' ( 'CONSTRAINT' constraint_name constraint_elem | constraint_elem ) | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' 'CONSTRAINT' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | partition_by | 'INJECT' 'STATISTICS' a_expr ) ) ( ( ',' ( 'ADD' ( column_name typename col_qual_list ) | 'ADD' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' ( column_name typename col_qual_list ) | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' ( column_name typename col_qual_list ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DEFAULT' a_expr | 'DROP' 'DEFAULT' ) | 'ALTER' ( 'COLUMN' | ) column_name 'SET' 'NOT' 'VISIBLE' | 'ALTER' ( 'COLUMN' | ) column_name 'SET' 'VISIBLE' | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'NOT' 'NULL' | 'ALTER' ( 'COLUMN' | ) column_name 'DROP' 'STORED' | 'DROP' ( 'COLUMN' | ) 'IF' 'EXISTS' column_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' ( 'COLUMN' | ) column_name ( 'CASCADE' | 'RESTRICT' | ) | 'ALTER' ( 'COLUMN' | ) column_name ( 'SET' 'DATA' | ) 'TYPE' typename ( 'COLLATE' collation_name | ) ( 'USING' a_expr | ) | 'ADD' ( 'CONSTRAINT' constraint_name constraint_elem | constraint_elem ) | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'DROP' 'CONSTRAINT' constraint_name ( 'CASCADE' | 'RESTRICT' | ) | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | partition_by | 'INJECT' 'STATISTICS' a_expr ) ) )* ) diff --git a/docs/generated/sql/bnf/col_qualification.bnf b/docs/generated/sql/bnf/col_qualification.bnf index 053f879a2f71..55c64288333c 100644 --- a/docs/generated/sql/bnf/col_qualification.bnf +++ b/docs/generated/sql/bnf/col_qualification.bnf @@ -7,6 +7,7 @@ col_qualification ::= | 'CONSTRAINT' constraint_name 'DEFAULT' b_expr | 'CONSTRAINT' constraint_name 'REFERENCES' table_name opt_name_parens reference_actions | 'CONSTRAINT' constraint_name 'AS' '(' a_expr ')' 'STORED' + | 'CONSTRAINT' constraint_name 'NOT' 'VISIBLE' | 'NOT' 'NULL' | 'NULL' | 'UNIQUE' @@ -15,6 +16,7 @@ col_qualification ::= | 'DEFAULT' b_expr | 'REFERENCES' table_name opt_name_parens reference_actions | 'AS' '(' a_expr ')' 'STORED' + | 'NOT' 'VISIBLE' | 'COLLATE' collation_name | 'FAMILY' family_name | 'CREATE' 'FAMILY' family_name diff --git a/docs/generated/sql/bnf/stmt_block.bnf b/docs/generated/sql/bnf/stmt_block.bnf index 1edc552d596b..c75a048c64a8 100644 --- a/docs/generated/sql/bnf/stmt_block.bnf +++ b/docs/generated/sql/bnf/stmt_block.bnf @@ -842,6 +842,7 @@ unreserved_keyword ::= | 'VALIDATE' | 'VALUE' | 'VARYING' + | 'VISIBLE' | 'WITHIN' | 'WITHOUT' | 'WRITE' @@ -1769,6 +1770,8 @@ alter_table_cmd ::= | 'ADD' 'COLUMN' column_def | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_def | 'ALTER' opt_column column_name alter_column_default + | 'ALTER' opt_column column_name 'SET' 'NOT' 'VISIBLE' + | 'ALTER' opt_column column_name 'SET' 'VISIBLE' | 'ALTER' opt_column column_name 'DROP' 'NOT' 'NULL' | 'ALTER' opt_column column_name 'DROP' 'STORED' | 'DROP' opt_column 'IF' 'EXISTS' column_name opt_drop_behavior @@ -1991,6 +1994,7 @@ col_qualification_elem ::= | 'DEFAULT' b_expr | 'REFERENCES' table_name opt_name_parens reference_actions | 'AS' '(' a_expr ')' 'STORED' + | 'NOT' 'VISIBLE' family_name ::= name diff --git a/pkg/cli/dump_test.go b/pkg/cli/dump_test.go index 99554a6758dc..7b58138e89b2 100644 --- a/pkg/cli/dump_test.go +++ b/pkg/cli/dump_test.go @@ -390,6 +390,8 @@ func TestDumpAsOf(t *testing.T) { const want1 = `dump d t CREATE TABLE t ( i INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (i, rowid) ); @@ -412,7 +414,9 @@ INSERT INTO t (i) VALUES const want2 = `dump d t CREATE TABLE t ( i INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), j INT NULL DEFAULT 2:::INT, + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (i, rowid, j) ); diff --git a/pkg/cli/testdata/dump/flags b/pkg/cli/testdata/dump/flags index b728501b5ed1..90cb96ea3e0f 100644 --- a/pkg/cli/testdata/dump/flags +++ b/pkg/cli/testdata/dump/flags @@ -11,6 +11,8 @@ dump t f --dump-mode=both CREATE TABLE f ( x INT NULL, y INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (x, y, rowid) ); @@ -24,6 +26,8 @@ dump t f --dump-mode=schema CREATE TABLE f ( x INT NULL, y INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (x, y, rowid) ); diff --git a/pkg/cli/testdata/dump/identifiers b/pkg/cli/testdata/dump/identifiers index c2192ddef9ad..763c6ef0368a 100644 --- a/pkg/cli/testdata/dump/identifiers +++ b/pkg/cli/testdata/dump/identifiers @@ -13,6 +13,8 @@ dump d ---- CREATE TABLE ";" ( ";" INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), INDEX ";_;_idx" (";" ASC), FAMILY "primary" (";", rowid) ); diff --git a/pkg/cli/testdata/dump/inverted_index b/pkg/cli/testdata/dump/inverted_index index e4bbf1c89775..48da8948a4a2 100644 --- a/pkg/cli/testdata/dump/inverted_index +++ b/pkg/cli/testdata/dump/inverted_index @@ -18,6 +18,8 @@ dump d t CREATE TABLE t ( a JSON NULL, b JSON NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), INVERTED INDEX idx (a), INVERTED INDEX idx2 (b), FAMILY "primary" (a, b, rowid) diff --git a/pkg/cli/testdata/dump/multiple b/pkg/cli/testdata/dump/multiple index c8b1939fdc9f..bc9129a7e2d0 100644 --- a/pkg/cli/testdata/dump/multiple +++ b/pkg/cli/testdata/dump/multiple @@ -13,12 +13,16 @@ dump t f g CREATE TABLE f ( x INT NULL, y INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (x, y, rowid) ); CREATE TABLE g ( x INT NULL, y INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (x, y, rowid) ); @@ -36,12 +40,16 @@ dump t CREATE TABLE f ( x INT NULL, y INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (x, y, rowid) ); CREATE TABLE g ( x INT NULL, y INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (x, y, rowid) ); diff --git a/pkg/cli/testdata/dump/reference_order b/pkg/cli/testdata/dump/reference_order index 419681a3e4ad..9971c457fbb5 100644 --- a/pkg/cli/testdata/dump/reference_order +++ b/pkg/cli/testdata/dump/reference_order @@ -43,6 +43,8 @@ CREATE TABLE b ( CREATE TABLE a ( i INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), INDEX a_auto_index_fk_i_ref_b (i ASC), FAMILY "primary" (i, rowid) ); @@ -79,6 +81,8 @@ CREATE TABLE d ( CREATE TABLE c ( i INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), INDEX c_auto_index_fk_i_ref_d (i ASC), FAMILY "primary" (i, rowid) ); diff --git a/pkg/cli/testdata/dump/row b/pkg/cli/testdata/dump/row index db7784081367..f8b63eb376f1 100644 --- a/pkg/cli/testdata/dump/row +++ b/pkg/cli/testdata/dump/row @@ -77,6 +77,8 @@ CREATE TABLE t ( e1 DECIMAL(2) NULL, e2 DECIMAL(2,1) NULL, s1 STRING(1) NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (i, f, d, t, ts, n, o, u, ip, j, ary, tz, e1, e2, s1, rowid), FAMILY fam_1_s (s), FAMILY fam_2_b (b), @@ -91,4 +93,3 @@ INSERT INTO t (i, f, s, b, d, t, ts, n, o, e, u, ip, j, ary, tz, e1, e2, s1) VAL (NULL, 'NaN', NULL, NULL, NULL, NULL, NULL, NULL, NULL, 'NaN', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); ---- ---- - diff --git a/pkg/sql/alter_table.go b/pkg/sql/alter_table.go index 4c6bd5e6ce28..657696919711 100644 --- a/pkg/sql/alter_table.go +++ b/pkg/sql/alter_table.go @@ -567,6 +567,20 @@ func (n *alterTableNode) startExec(params runParams) error { return err } + case *tree.AlterTableSetHidden: + col, dropped, err := n.tableDesc.FindColumnByName(t.GetColumn()) + if err != nil { + return err + } + if dropped { + return fmt.Errorf("column %q in the middle of being dropped", t.GetColumn()) + } + if err := applyColumnMutation(n.tableDesc, &col, t, params); err != nil { + return err + } + n.tableDesc.UpdateColumnDescriptor(col) + descriptorChanged = true + case *tree.AlterTableInjectStats: sd, ok := n.statsData[i] if !ok { @@ -757,6 +771,9 @@ func applyColumnMutation( case *tree.AlterTableDropStored: col.ComputeExpr = nil + + case *tree.AlterTableSetHidden: + col.Hidden = t.Hidden } return nil } diff --git a/pkg/sql/create_table.go b/pkg/sql/create_table.go index eb0f0b010262..89e58ffbad23 100644 --- a/pkg/sql/create_table.go +++ b/pkg/sql/create_table.go @@ -921,7 +921,11 @@ func makeTableDescIfAs( if err != nil { return desc, err } - columnTableDef := tree.ColumnTableDef{Name: tree.Name(colRes.Name), Type: colType} + columnTableDef := tree.ColumnTableDef{ + Name: tree.Name(colRes.Name), + Hidden: colRes.Hidden, + Type: colType, + } columnTableDef.Nullable.Nullability = tree.SilentNull if len(p.AsColumnNames) > i { columnTableDef.Name = p.AsColumnNames[i] diff --git a/pkg/sql/create_view.go b/pkg/sql/create_view.go index 4cd953dc0e76..5cac0475c345 100644 --- a/pkg/sql/create_view.go +++ b/pkg/sql/create_view.go @@ -224,7 +224,11 @@ func (n *createViewNode) makeViewTableDesc( if err != nil { return desc, err } - columnTableDef := tree.ColumnTableDef{Name: tree.Name(colRes.Name), Type: colType} + columnTableDef := tree.ColumnTableDef{ + Name: tree.Name(colRes.Name), + Hidden: colRes.Hidden, + Type: colType, + } if len(columnNames) > i { columnTableDef.Name = columnNames[i] } diff --git a/pkg/sql/insert.go b/pkg/sql/insert.go index a48faff6b6b1..3385e3af4d04 100644 --- a/pkg/sql/insert.go +++ b/pkg/sql/insert.go @@ -638,8 +638,7 @@ func (p *planner) processColumns( if ensureColumns { // VisibleColumns is used here to prevent INSERT INTO VALUES (...) // (as opposed to INSERT INTO
(...) VALUES (...)) from writing - // hidden columns. At present, the only hidden column is the implicit rowid - // primary key column. + // hidden columns. return tableDesc.VisibleColumns(), nil } return nil, nil diff --git a/pkg/sql/logictest/testdata/logic_test/alias_types b/pkg/sql/logictest/testdata/logic_test/alias_types index 5780f8b14f06..2cce2d4bf3d6 100644 --- a/pkg/sql/logictest/testdata/logic_test/alias_types +++ b/pkg/sql/logictest/testdata/logic_test/alias_types @@ -13,11 +13,13 @@ SHOW CREATE TABLE aliases ---- Table CreateTable aliases CREATE TABLE aliases ( - a OID NULL, - b NAME NULL, - FAMILY "primary" (a, rowid), - FAMILY fam_1_b (b) - ) + a OID NULL, + b NAME NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (a, rowid), + FAMILY fam_1_b (b) +) statement ok INSERT INTO aliases VALUES (100, 'abc') diff --git a/pkg/sql/logictest/testdata/logic_test/array b/pkg/sql/logictest/testdata/logic_test/array index 4270b725ac22..749bbb09ecfd 100644 --- a/pkg/sql/logictest/testdata/logic_test/array +++ b/pkg/sql/logictest/testdata/logic_test/array @@ -381,9 +381,11 @@ query TT SHOW CREATE TABLE a ---- a CREATE TABLE a ( - b INT[] NULL, - FAMILY "primary" (b, rowid) - ) + b INT[] NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (b, rowid) +) statement ok DROP TABLE a @@ -452,9 +454,11 @@ query TT SHOW CREATE TABLE a ---- a CREATE TABLE a ( - b INT[] NULL, - FAMILY "primary" (b, rowid) - ) + b INT[] NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (b, rowid) +) statement error could not parse "foo" as type int INSERT INTO a VALUES (ARRAY['foo']) @@ -523,9 +527,11 @@ query TT SHOW CREATE TABLE a ---- a CREATE TABLE a ( - b SMALLINT[] NULL, - FAMILY "primary" (b, rowid) - ) + b SMALLINT[] NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (b, rowid) +) statement error integer out of range for type SMALLINT \(column "b"\) INSERT INTO a VALUES (ARRAY[100000]) diff --git a/pkg/sql/logictest/testdata/logic_test/check_constraints b/pkg/sql/logictest/testdata/logic_test/check_constraints index a7eff95742e0..2f321a5892a6 100644 --- a/pkg/sql/logictest/testdata/logic_test/check_constraints +++ b/pkg/sql/logictest/testdata/logic_test/check_constraints @@ -219,16 +219,18 @@ query TT SHOW CREATE TABLE t7 ---- t7 CREATE TABLE t7 ( - x INT NULL, - y INT NULL, - z INT NULL, - FAMILY "primary" (x, y, z, rowid), - CONSTRAINT check_x CHECK (x > 0), - CONSTRAINT check_x_y CHECK ((x + y) > 0), - CONSTRAINT check_y_z CHECK ((y + z) > 0), - CONSTRAINT check_y_z1 CHECK ((y + z) = 0), - CONSTRAINT named_constraint CHECK (z = 1) - ) + x INT NULL, + y INT NULL, + z INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (x, y, z, rowid), + CONSTRAINT check_x CHECK (x > 0), + CONSTRAINT check_x_y CHECK ((x + y) > 0), + CONSTRAINT check_y_z CHECK ((y + z) > 0), + CONSTRAINT check_y_z1 CHECK ((y + z) = 0), + CONSTRAINT named_constraint CHECK (z = 1) +) # Check that table references are dequalified in their stored representation. @@ -256,12 +258,14 @@ query TT SHOW CREATE TABLE t8 ---- t8 CREATE TABLE t8 ( - a INT NULL, - FAMILY "primary" (a, rowid), - CONSTRAINT check_a CHECK (a > 0), - CONSTRAINT check_a1 CHECK (a > 0), - CONSTRAINT check_a2 CHECK (a > 0) - ) + a INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (a, rowid), + CONSTRAINT check_a CHECK (a > 0), + CONSTRAINT check_a1 CHECK (a > 0), + CONSTRAINT check_a2 CHECK (a > 0) +) statement ok CREATE DATABASE test2 diff --git a/pkg/sql/logictest/testdata/logic_test/collatedstring b/pkg/sql/logictest/testdata/logic_test/collatedstring index cd9d1439c399..ab2f051e2377 100644 --- a/pkg/sql/logictest/testdata/logic_test/collatedstring +++ b/pkg/sql/logictest/testdata/logic_test/collatedstring @@ -203,9 +203,11 @@ query TT SHOW CREATE TABLE t ---- t CREATE TABLE t ( - a STRING COLLATE en NULL, - FAMILY "primary" (a, rowid) - ) + a STRING COLLATE en NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (a, rowid) +) statement ok INSERT INTO t VALUES diff --git a/pkg/sql/logictest/testdata/logic_test/computed b/pkg/sql/logictest/testdata/logic_test/computed index 0dbdecc90a64..a7696f10ce15 100644 --- a/pkg/sql/logictest/testdata/logic_test/computed +++ b/pkg/sql/logictest/testdata/logic_test/computed @@ -10,12 +10,14 @@ CREATE TABLE with_no_column_refs ( query TT SHOW CREATE TABLE with_no_column_refs ---- -with_no_column_refs CREATE TABLE with_no_column_refs ( - a INT NULL, - b INT NULL, - c INT NULL AS (3) STORED, - FAMILY "primary" (a, b, c, rowid) - ) +with_no_column_refs CREATE TABLE with_no_column_refs ( + a INT NULL, + b INT NULL, + c INT NULL AS (3) STORED, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (a, b, c, rowid) +) statement ok CREATE TABLE extra_parens ( @@ -27,12 +29,14 @@ CREATE TABLE extra_parens ( query TT SHOW CREATE TABLE extra_parens ---- -extra_parens CREATE TABLE extra_parens ( - a INT NULL, - b INT NULL, - c INT NULL AS ((3)) STORED, - FAMILY "primary" (a, b, c, rowid) - ) +extra_parens CREATE TABLE extra_parens ( + a INT NULL, + b INT NULL, + c INT NULL AS ((3)) STORED, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (a, b, c, rowid) +) statement error cannot write directly to computed column "c" @@ -79,13 +83,15 @@ CREATE TABLE x ( query TT SHOW CREATE TABLE x ---- -x CREATE TABLE x ( - a INT NULL DEFAULT 3:::INT, - b INT NULL DEFAULT 7:::INT, - c INT NULL AS (a) STORED, - d INT NULL AS (a + b) STORED, - FAMILY "primary" (a, b, c, d, rowid) - ) +x CREATE TABLE x ( + a INT NULL DEFAULT 3:::INT, + b INT NULL DEFAULT 7:::INT, + c INT NULL AS (a) STORED, + d INT NULL AS (a + b) STORED, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (a, b, c, d, rowid) +) statement error cannot write directly to computed column "c" INSERT INTO x (c) VALUES (1) @@ -587,11 +593,13 @@ CREATE TABLE x ( query TT SHOW CREATE TABLE x ---- -x CREATE TABLE x ( - a INT NULL, - b INT NULL AS (a) STORED, - FAMILY "primary" (a, b, rowid) - ) +x CREATE TABLE x ( + a INT NULL, + b INT NULL AS (a) STORED, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (a, b, rowid) +) statement ok DROP TABLE x @@ -609,11 +617,13 @@ ALTER TABLE x RENAME COLUMN a TO c query TT SHOW CREATE TABLE x ---- -x CREATE TABLE x ( - c INT NULL, - b INT NULL AS (c) STORED, - FAMILY "primary" (c, b, rowid) - ) +x CREATE TABLE x ( + c INT NULL, + b INT NULL AS (c) STORED, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (c, b, rowid) +) statement ok DROP TABLE x @@ -648,10 +658,12 @@ query TT SHOW CREATE TABLE x ---- x CREATE TABLE x ( - a INT NULL, - b INT NULL AS (a * 2) STORED, - c INT NULL AS (a + 4) STORED, - FAMILY "primary" (a, b, rowid, c) + a INT NULL, + b INT NULL AS (a * 2) STORED, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + c INT NULL AS (a + 4) STORED, + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (a, b, rowid, c) ) statement ok diff --git a/pkg/sql/logictest/testdata/logic_test/create_statements b/pkg/sql/logictest/testdata/logic_test/create_statements index cde5f909a2f5..0d90fd558e09 100644 --- a/pkg/sql/logictest/testdata/logic_test/create_statements +++ b/pkg/sql/logictest/testdata/logic_test/create_statements @@ -12,17 +12,23 @@ SELECT create_statement, create_nofks, alter_statements, validate_statements FRO create_statement create_nofks alter_statements validate_statements CREATE TABLE t ( a INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), CONSTRAINT fk_a_ref_t FOREIGN KEY (a) REFERENCES t (rowid), INDEX t_auto_index_fk_a_ref_t (a ASC), FAMILY "primary" (a, rowid) ) CREATE TABLE t ( a INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), INDEX t_auto_index_fk_a_ref_t (a ASC), FAMILY "primary" (a, rowid) ) {"ALTER TABLE t ADD CONSTRAINT fk_a_ref_t FOREIGN KEY (a) REFERENCES t (rowid)"} {"ALTER TABLE t VALIDATE CONSTRAINT fk_a_ref_t"} CREATE TABLE v ( "'" INT NULL, s STRING NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), CONSTRAINT fk_s_ref_v FOREIGN KEY (s) REFERENCES v (s), UNIQUE INDEX v_s_key (s ASC), CONSTRAINT "fk_'_ref_t" FOREIGN KEY ("'") REFERENCES t (rowid), @@ -31,6 +37,8 @@ CREATE TABLE v ( ) CREATE TABLE v ( "'" INT NULL, s STRING NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), UNIQUE INDEX v_s_key (s ASC), INDEX "v_auto_index_fk_'_ref_t" ("'" ASC), FAMILY "primary" ("'", s, rowid) diff --git a/pkg/sql/logictest/testdata/logic_test/fk b/pkg/sql/logictest/testdata/logic_test/fk index c47f2cd0b93f..810dc73c14c1 100644 --- a/pkg/sql/logictest/testdata/logic_test/fk +++ b/pkg/sql/logictest/testdata/logic_test/fk @@ -237,15 +237,17 @@ query TT SHOW CREATE TABLE delivery ---- delivery CREATE TABLE delivery ( - ts TIMESTAMP NULL DEFAULT now(), - "order" INT NULL, - shipment INT NULL, - item STRING NULL, - CONSTRAINT fk_item_ref_products FOREIGN KEY (item) REFERENCES products (upc), - INDEX delivery_item_idx (item ASC), - CONSTRAINT fk_order_ref_orders FOREIGN KEY ("order", shipment) REFERENCES orders (id, shipment), - INDEX delivery_auto_index_fk_order_ref_orders ("order" ASC, shipment ASC), - FAMILY "primary" (ts, "order", shipment, item, rowid) + ts TIMESTAMP NULL DEFAULT now(), + "order" INT NULL, + shipment INT NULL, + item STRING NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + CONSTRAINT fk_item_ref_products FOREIGN KEY (item) REFERENCES products (upc), + INDEX delivery_item_idx (item ASC), + CONSTRAINT fk_order_ref_orders FOREIGN KEY ("order", shipment) REFERENCES orders (id, shipment), + INDEX delivery_auto_index_fk_order_ref_orders ("order" ASC, shipment ASC), + FAMILY "primary" (ts, "order", shipment, item, rowid) ) statement ok @@ -591,12 +593,14 @@ query TT SHOW CREATE TABLE refpairs ---- refpairs CREATE TABLE refpairs ( - a INT NULL, - b STRING NULL, - c INT NULL, - CONSTRAINT fk_a_ref_pairs FOREIGN KEY (a, b) REFERENCES pairs (src, dest) ON UPDATE RESTRICT, - INDEX refpairs_a_b_c_idx (a ASC, b ASC, c ASC), - FAMILY "primary" (a, b, c, rowid) + a INT NULL, + b STRING NULL, + c INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + CONSTRAINT fk_a_ref_pairs FOREIGN KEY (a, b) REFERENCES pairs (src, dest) ON UPDATE RESTRICT, + INDEX refpairs_a_b_c_idx (a ASC, b ASC, c ASC), + FAMILY "primary" (a, b, c, rowid) ) statement error pgcode 23503 foreign key violation: value \[100 'two'\] not found in pairs@pairs_src_dest_key \[src dest\] @@ -803,13 +807,15 @@ query TT SHOW CREATE TABLE refers ---- refers CREATE TABLE refers ( - a INT NULL, - b INT NULL, - INDEX another_idx (b ASC), - CONSTRAINT fk_a_ref_referee FOREIGN KEY (a) REFERENCES referee (id), - INDEX refers_auto_index_fk_a_ref_referee (a ASC), - INDEX foo (a ASC), - FAMILY "primary" (a, b, rowid) + a INT NULL, + b INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + INDEX another_idx (b ASC), + CONSTRAINT fk_a_ref_referee FOREIGN KEY (a) REFERENCES referee (id), + INDEX refers_auto_index_fk_a_ref_referee (a ASC), + INDEX foo (a ASC), + FAMILY "primary" (a, b, rowid) ) statement ok diff --git a/pkg/sql/logictest/testdata/logic_test/hidden_columns b/pkg/sql/logictest/testdata/logic_test/hidden_columns new file mode 100644 index 000000000000..a7e33b71f7bb --- /dev/null +++ b/pkg/sql/logictest/testdata/logic_test/hidden_columns @@ -0,0 +1,102 @@ +# LogicTest: local local-opt local-parallel-stmts fakedist fakedist-opt fakedist-metadata + +statement ok +CREATE TABLE t (x INT NOT VISIBLE); + CREATE TABLE kv ( + k INT PRIMARY KEY NOT VISIBLE, + v INT NOT VISIBLE + ) + +# Verify that hidden columns can be explicitly inserted into. + +statement ok +INSERT INTO t(x) VALUES (123) + +statement ok +INSERT INTO kv(k,v) VALUES (123,456); + +# Verify that hidden columns cannot be implicitly inserted into + +statement error INSERT has more expressions than target columns, 1 expressions for 0 targets +INSERT INTO t VALUES (123) + +statement error INSERT has more expressions than target columns, 2 expressions for 0 targets +INSERT INTO kv VALUES (111, 222) + +# Verify the right columns are hidden. + +query TT +SHOW CREATE TABLE t +---- +t CREATE TABLE t ( + x INT NOT VISIBLE NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (x, rowid) +) + +query TT +SHOW CREATE TABLE t +---- +t CREATE TABLE t ( + x INT NOT VISIBLE NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (x, rowid) +) + +# Check that stars expand to no columns. + +query I +SELECT 42, * FROM t +---- +42 + +query I +SELECT 42, * FROM kv +---- +42 + +# Check that the hidden column can be selected explicitly. + +query II +SELECT 42, x FROM t +---- +42 123 + +# Check that ALTER can make hidden columns visible. + +statement ok +ALTER TABLE kv ALTER v SET VISIBLE + +query TT +SHOW CREATE TABLE kv +---- +kv CREATE TABLE kv ( + k INT NOT VISIBLE NOT NULL, + v INT NULL, + CONSTRAINT "primary" PRIMARY KEY (k ASC), + FAMILY "primary" (k, v) +) + +# Check that ALTER can even make rowid visible. + +statement ok +ALTER TABLE t ALTER rowid SET VISIBLE + +query TT +SHOW CREATE TABLE t +---- +t CREATE TABLE t ( + x INT NOT VISIBLE NULL, + rowid INT NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (x, rowid) +) + +# Check that rowid is properly auto-selected by star expansion. + +query I +SELECT rowid-rowid FROM (SELECT * FROM t) +---- +0 diff --git a/pkg/sql/logictest/testdata/logic_test/name_escapes b/pkg/sql/logictest/testdata/logic_test/name_escapes index 52c077ffb9d2..3223e118320e 100644 --- a/pkg/sql/logictest/testdata/logic_test/name_escapes +++ b/pkg/sql/logictest/testdata/logic_test/name_escapes @@ -61,5 +61,7 @@ SHOW CREATE TABLE ";--dontask" a INT NULL, b INT NULL, c INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (a, b, c, rowid) ) diff --git a/pkg/sql/logictest/testdata/logic_test/schema_change_in_txn b/pkg/sql/logictest/testdata/logic_test/schema_change_in_txn index c25617bbe06a..6574ffca5f51 100644 --- a/pkg/sql/logictest/testdata/logic_test/schema_change_in_txn +++ b/pkg/sql/logictest/testdata/logic_test/schema_change_in_txn @@ -98,13 +98,15 @@ query TT SHOW CREATE TABLE b ---- b CREATE TABLE b ( - parent_id INT NULL, - d INT NULL DEFAULT 23:::INT, - CONSTRAINT fk_parent_id_ref_parent FOREIGN KEY (parent_id) REFERENCES parent (id), - INDEX b_auto_index_fk_parent_id_ref_parent (parent_id ASC), - INDEX foo (parent_id ASC), - UNIQUE INDEX bar (parent_id ASC), - FAMILY "primary" (parent_id, rowid, d) + parent_id INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + d INT NULL DEFAULT 23:::INT, + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + CONSTRAINT fk_parent_id_ref_parent FOREIGN KEY (parent_id) REFERENCES parent (id), + INDEX b_auto_index_fk_parent_id_ref_parent (parent_id ASC), + INDEX foo (parent_id ASC), + UNIQUE INDEX bar (parent_id ASC), + FAMILY "primary" (parent_id, rowid, d) ) # table b is not visible to the transaction #17949 diff --git a/pkg/sql/logictest/testdata/logic_test/table b/pkg/sql/logictest/testdata/logic_test/table index e801060ea952..f0ad2484b073 100644 --- a/pkg/sql/logictest/testdata/logic_test/table +++ b/pkg/sql/logictest/testdata/logic_test/table @@ -449,6 +449,8 @@ SHOW CREATE TABLE test.null_default ---- test.public.null_default CREATE TABLE null_default ( ts TIMESTAMP NULL DEFAULT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (ts, rowid) ) diff --git a/pkg/sql/opt/testutils/testcat/create_table.go b/pkg/sql/opt/testutils/testcat/create_table.go index d0ce45437b35..68e82c466fec 100644 --- a/pkg/sql/opt/testutils/testcat/create_table.go +++ b/pkg/sql/opt/testutils/testcat/create_table.go @@ -116,7 +116,7 @@ func (tc *Catalog) qualifyTableName(name *tree.TableName) { func (tt *Table) addColumn(def *tree.ColumnTableDef) { nullable := !def.PrimaryKey && def.Nullable.Nullability != tree.NotNull typ := coltypes.CastTargetToDatumType(def.Type) - col := &Column{Name: string(def.Name), Type: typ, Nullable: nullable} + col := &Column{Name: string(def.Name), Type: typ, Nullable: nullable, Hidden: def.Hidden} tt.Columns = append(tt.Columns, col) if def.PrimaryKey { diff --git a/pkg/sql/parser/parse_test.go b/pkg/sql/parser/parse_test.go index 83fc6ad2a894..09eaa78a9bca 100644 --- a/pkg/sql/parser/parse_test.go +++ b/pkg/sql/parser/parse_test.go @@ -121,6 +121,8 @@ func TestParse(t *testing.T) { {`CREATE TABLE a (a INT DEFAULT 1 CONSTRAINT positive CHECK (a > 0))`}, {`CREATE TABLE a (a INT CONSTRAINT one DEFAULT 1 CONSTRAINT positive CHECK (a > 0))`}, {`CREATE TABLE a (a INT CONSTRAINT one CHECK (a > 0) CONSTRAINT two CHECK (a < 10))`}, + {`CREATE TABLE a (b INT NOT VISIBLE)`}, + {`CREATE TABLE a (b INT NOT VISIBLE NULL)`}, // "0" lost quotes previously. {`CREATE TABLE a (b INT, c TEXT, PRIMARY KEY (b, c, "0"))`}, {`CREATE TABLE a (b INT, c TEXT, FOREIGN KEY (b) REFERENCES other)`}, @@ -894,6 +896,8 @@ func TestParse(t *testing.T) { {`ALTER TABLE a ALTER COLUMN b DROP NOT NULL`}, {`ALTER TABLE a ALTER b DROP NOT NULL`}, {`ALTER TABLE a ALTER COLUMN b DROP STORED`}, + {`ALTER TABLE a ALTER COLUMN b SET VISIBLE`}, + {`ALTER TABLE a ALTER COLUMN b SET NOT VISIBLE`}, {`ALTER TABLE a ALTER b TYPE INT`}, {`ALTER TABLE a ALTER COLUMN b SET DATA TYPE INT`}, diff --git a/pkg/sql/parser/sql.y b/pkg/sql/parser/sql.y index 1e6f2b1d714f..61e1c542fcdd 100644 --- a/pkg/sql/parser/sql.y +++ b/pkg/sql/parser/sql.y @@ -537,7 +537,7 @@ func newNameFromStr(s string) *tree.Name { %token UNBOUNDED UNCOMMITTED UNION UNIQUE UNKNOWN %token UPDATE UPSERT USE USER USERS USING UUID -%token VALID VALIDATE VALUE VALUES VARCHAR VARIADIC VIEW VARYING VIRTUAL +%token VALID VALIDATE VALUE VALUES VARCHAR VARIADIC VIEW VISIBLE VARYING VIRTUAL %token WHEN WHERE WINDOW WITH WITHIN WITHOUT WORK WRITE @@ -1104,6 +1104,7 @@ alter_ddl_stmt: // ALTER TABLE ... ALTER [COLUMN] {SET DEFAULT | DROP DEFAULT} // ALTER TABLE ... ALTER [COLUMN] DROP NOT NULL // ALTER TABLE ... ALTER [COLUMN] DROP STORED +// ALTER TABLE ... ALTER [COLUMN] SET [NOT] VISIBLE // ALTER TABLE ... ALTER [COLUMN] [SET DATA] TYPE [COLLATE ] // ALTER TABLE ... RENAME TO // ALTER TABLE ... RENAME [COLUMN] TO @@ -1372,7 +1373,17 @@ alter_table_cmd: { $$.val = &tree.AlterTableSetDefault{ColumnKeyword: $2.bool(), Column: tree.Name($3), Default: $4.expr()} } - // ALTER TABLE ALTER [COLUMN] DROP NOT NULL + // ALTER TABLE ALTER [COLUMN] SET NOT VISIBLE +| ALTER opt_column column_name SET NOT VISIBLE + { + $$.val = &tree.AlterTableSetHidden{Column: tree.Name($3), Hidden: true} + } + // ALTER TABLE ALTER [COLUMN] SET VISIBLE +| ALTER opt_column column_name SET VISIBLE + { + $$.val = &tree.AlterTableSetHidden{Column: tree.Name($3), Hidden: false} + } + // ALTER TABLE ALTER [COLUMN] DROP NOT NULL | ALTER opt_column column_name DROP NOT NULL { $$.val = &tree.AlterTableDropNotNull{ColumnKeyword: $2.bool(), Column: tree.Name($3)} @@ -3823,6 +3834,10 @@ col_qualification_elem: sqllex.Error("syntax error: use AS ( ) STORED") return 1 } +| NOT VISIBLE + { + $$.val = tree.HiddenConstraint{} + } index_def: INDEX opt_index_name '(' index_params ')' opt_storing opt_interleave opt_partition_by @@ -8129,6 +8144,7 @@ unreserved_keyword: | VALIDATE | VALUE | VARYING +| VISIBLE | WITHIN | WITHOUT | WRITE diff --git a/pkg/sql/sem/tree/alter_table.go b/pkg/sql/sem/tree/alter_table.go index 76162cd00ddf..f586a452c6a0 100644 --- a/pkg/sql/sem/tree/alter_table.go +++ b/pkg/sql/sem/tree/alter_table.go @@ -65,6 +65,7 @@ func (*AlterTableDropNotNull) alterTableCmd() {} func (*AlterTableDropStored) alterTableCmd() {} func (*AlterTableSetAudit) alterTableCmd() {} func (*AlterTableSetDefault) alterTableCmd() {} +func (*AlterTableSetHidden) alterTableCmd() {} func (*AlterTableValidateConstraint) alterTableCmd() {} func (*AlterTablePartitionBy) alterTableCmd() {} func (*AlterTableInjectStats) alterTableCmd() {} @@ -78,6 +79,7 @@ var _ AlterTableCmd = &AlterTableDropNotNull{} var _ AlterTableCmd = &AlterTableDropStored{} var _ AlterTableCmd = &AlterTableSetAudit{} var _ AlterTableCmd = &AlterTableSetDefault{} +var _ AlterTableCmd = &AlterTableSetHidden{} var _ AlterTableCmd = &AlterTableValidateConstraint{} var _ AlterTableCmd = &AlterTablePartitionBy{} var _ AlterTableCmd = &AlterTableInjectStats{} @@ -197,6 +199,28 @@ func (node *AlterTableDropColumn) Format(ctx *FmtCtx) { } } +// AlterTableSetHidden represents a SET [NOT] VISIBLE command. +type AlterTableSetHidden struct { + Column Name + Hidden bool +} + +// Format implements the NodeFormatter interface. +func (node *AlterTableSetHidden) Format(ctx *FmtCtx) { + ctx.WriteString(" ALTER COLUMN ") + ctx.FormatNode(&node.Column) + ctx.WriteString(" SET") + if node.Hidden { + ctx.WriteString(" NOT") + } + ctx.WriteString(" VISIBLE") +} + +// GetColumn implements the ColumnMutationCmd interface. +func (node *AlterTableSetHidden) GetColumn() Name { + return node.Column +} + // AlterTableDropConstraint represents a DROP CONSTRAINT command. type AlterTableDropConstraint struct { IfExists bool diff --git a/pkg/sql/sem/tree/create.go b/pkg/sql/sem/tree/create.go index 4641c48ee5d2..742854952715 100644 --- a/pkg/sql/sem/tree/create.go +++ b/pkg/sql/sem/tree/create.go @@ -194,6 +194,7 @@ const ( type ColumnTableDef struct { Name Name Type coltypes.T + Hidden bool Nullable struct { Nullability Nullability ConstraintName Name @@ -279,6 +280,8 @@ func NewColumnTableDef( } d.DefaultExpr.Expr = t.Expr d.DefaultExpr.ConstraintName = c.Name + case HiddenConstraint: + d.Hidden = true case NotNullConstraint: if d.Nullable.Nullability == Null { return nil, pgerror.NewErrorf(pgerror.CodeSyntaxError, @@ -365,6 +368,9 @@ func (node *ColumnTableDef) Format(ctx *FmtCtx) { ctx.WriteString(" CONSTRAINT ") ctx.FormatNode(&node.Nullable.ConstraintName) } + if node.Hidden { + ctx.WriteString(" NOT VISIBLE") + } switch node.Nullable.Nullability { case Null: ctx.WriteString(" NULL") @@ -442,6 +448,7 @@ func (ColumnCollation) columnQualification() {} func (*ColumnDefault) columnQualification() {} func (NotNullConstraint) columnQualification() {} func (NullConstraint) columnQualification() {} +func (HiddenConstraint) columnQualification() {} func (PrimaryKeyConstraint) columnQualification() {} func (UniqueConstraint) columnQualification() {} func (*ColumnCheckConstraint) columnQualification() {} @@ -463,6 +470,9 @@ type NotNullConstraint struct{} // NullConstraint represents NULL on a column. type NullConstraint struct{} +// HiddenConstraint represents HIDDEN on a column. +type HiddenConstraint struct{} + // PrimaryKeyConstraint represents NULL on a column. type PrimaryKeyConstraint struct{} diff --git a/pkg/sql/show_create.go b/pkg/sql/show_create.go index 3b467e11e851..b8e31ab26cb2 100644 --- a/pkg/sql/show_create.go +++ b/pkg/sql/show_create.go @@ -193,20 +193,16 @@ func (p *planner) showCreateTable( f.WriteString("CREATE TABLE ") f.FormatNode(tn) f.WriteString(" (") - primaryKeyIsOnVisibleColumn := false - for i, col := range desc.VisibleColumns() { + for i, col := range desc.Columns { if i != 0 { f.WriteString(",") } f.WriteString("\n\t") f.WriteString(col.SQLString()) - if desc.IsPhysicalTable() && desc.PrimaryIndex.ColumnIDs[0] == col.ID { - // Only set primaryKeyIsOnVisibleColumn to true if the primary key - // is on a visible column (not rowid). - primaryKeyIsOnVisibleColumn = true - } } - if primaryKeyIsOnVisibleColumn { + if len(desc.PrimaryIndex.ColumnIDs) > 0 { + // Virtual tables do not have PK columns. Do not print a + // constraint in this case. f.WriteString(",\n\tCONSTRAINT ") formatQuoteNames(f.Buffer, desc.PrimaryIndex.Name) f.WriteString(" ") diff --git a/pkg/sql/show_test.go b/pkg/sql/show_test.go index 2e19abfffa2b..ef4441e8c104 100644 --- a/pkg/sql/show_test.go +++ b/pkg/sql/show_test.go @@ -79,6 +79,8 @@ func TestShowCreateTable(t *testing.T) { s STRING NULL, v FLOAT NOT NULL, t TIMESTAMP NULL DEFAULT now(), + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (i, v, t, rowid), FAMILY fam_1_s (s), CONSTRAINT check_i CHECK (i > 0) @@ -98,6 +100,8 @@ func TestShowCreateTable(t *testing.T) { s STRING NULL, v FLOAT NOT NULL, t TIMESTAMP NULL DEFAULT now(), + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (i, v, t, rowid), FAMILY fam_1_s (s), CONSTRAINT check_i CHECK (i > 0) @@ -114,6 +118,8 @@ func TestShowCreateTable(t *testing.T) { expect: `CREATE TABLE %s ( i INT NULL, s STRING NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), FAMILY "primary" (i, rowid), FAMILY fam_1_s (s), CONSTRAINT ck CHECK (i > 0) @@ -142,6 +148,8 @@ func TestShowCreateTable(t *testing.T) { f FLOAT NULL, s STRING NULL, d DATE NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), INDEX idx_if (f ASC, i ASC) STORING (s, d), UNIQUE INDEX %[1]s_d_key (d ASC), FAMILY "primary" (i, f, d, rowid), @@ -164,6 +172,8 @@ func TestShowCreateTable(t *testing.T) { expect: `CREATE TABLE %s ( a INT NULL, b INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), INDEX c (a ASC, b DESC), FAMILY "primary" (a, b, rowid) )`, @@ -181,6 +191,8 @@ func TestShowCreateTable(t *testing.T) { i INT NULL, j INT NULL, k INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), CONSTRAINT fk_i_ref_items FOREIGN KEY (i, j) REFERENCES items (a, b), INDEX t7_auto_index_fk_i_ref_items (i ASC, j ASC), CONSTRAINT fk_k_ref_items FOREIGN KEY (k) REFERENCES items (c), @@ -197,6 +209,8 @@ func TestShowCreateTable(t *testing.T) { )`, expect: `CREATE TABLE %s ( x INT NULL, + rowid INT NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), CONSTRAINT fk_ref FOREIGN KEY (x) REFERENCES o.public.foo (x), INDEX t8_auto_index_fk_ref (x ASC), FAMILY "primary" (x, rowid) diff --git a/pkg/sql/sqlbase/structured.go b/pkg/sql/sqlbase/structured.go index 706de777679b..d6fd9a937ab4 100644 --- a/pkg/sql/sqlbase/structured.go +++ b/pkg/sql/sqlbase/structured.go @@ -2546,6 +2546,9 @@ func (desc *ColumnDescriptor) SQLString() string { f.FormatNameP(&desc.Name) f.WriteByte(' ') f.WriteString(desc.Type.SQLString()) + if desc.Hidden { + f.WriteString(" NOT VISIBLE") + } if desc.Nullable { f.WriteString(" NULL") } else { diff --git a/pkg/sql/sqlbase/table.go b/pkg/sql/sqlbase/table.go index b089a8796665..79e603882d6a 100644 --- a/pkg/sql/sqlbase/table.go +++ b/pkg/sql/sqlbase/table.go @@ -179,6 +179,7 @@ func MakeColumnDefDescs( ) (*ColumnDescriptor, *IndexDescriptor, tree.TypedExpr, error) { col := &ColumnDescriptor{ Name: string(d.Name), + Hidden: d.Hidden, Nullable: d.Nullable.Nullability != tree.NotNull && !d.PrimaryKey, }