From 50a3dcb05858b214a39909f0d9a4737a91f9059a Mon Sep 17 00:00:00 2001 From: Jordan Lewis Date: Fri, 10 Sep 2021 19:13:52 -0300 Subject: [PATCH] sql: add json{,b}_to_record{,set}; recordtype srfs This commit adds support for the family of four json{,b}_to_record{,set} functions, which deconstruct JSON input (either an object or an array) and return a row or set of rows with well-typed elements from the JSON, governed by the *table alias* that the function is invoked with. For example, the query below deconstructs the input JSON, returning each of the keys "requested" by the table alias definition. Missing keys are replaced with `NULL`. ``` SELECT * FROM json_to_record('{"a": "b", "c": true}') AS t(a TEXT, z INT) ---- b NULL ``` The logic that governs the type casting from JSON to SQL mimics the logic in the similar `json_populate_record` family of functions, and should be identical to Postgres's such logic. It's permissible to use the virtual table type to deconstruct a sub-object into a record field within the top level JSON, like this: ``` CREATE TABLE mytable (a INT, b TEXT) SELECT * FROM json_to_record('{"foo": {"a": 3, "b": "bar"}}') AS t(foo mytable) ---- (3,bar) ``` Functions like these ones that return record types in PostgreSQL must be aliased to a named and typed tuple (like `AS t(a INT, b INT)`) to be usable. As a result, this commit: 1. adds parser support for this form of alias, with types 2. pushes the alias information for an aliased expression down the optimizer's call stack so that the aliased expression can access the alias information. Previously, the aliased expression was blind to any aliases that were applied to it. 3. teaches the generator function factories to get the alias information plumbed in as well. Release note (sql change): add the json{,b}_to_record{,set} builtin functions, which transform JSON into structured SQL records. --- docs/generated/sql/bnf/BUILD.bazel | 2 +- .../sql/bnf/alter_table_partition_by.bnf | 4 +- docs/generated/sql/bnf/check_column_level.bnf | 2 +- docs/generated/sql/bnf/check_table_level.bnf | 2 +- .../{column_def.bnf => column_table_def.bnf} | 2 +- docs/generated/sql/bnf/create_table_stmt.bnf | 4 +- .../sql/bnf/default_value_column_level.bnf | 2 +- .../sql/bnf/foreign_key_column_level.bnf | 2 +- .../sql/bnf/foreign_key_table_level.bnf | 2 +- .../sql/bnf/not_null_column_level.bnf | 2 +- .../sql/bnf/primary_key_column_level.bnf | 2 +- .../sql/bnf/primary_key_table_level.bnf | 2 +- docs/generated/sql/bnf/stmt_block.bnf | 49 ++++-- docs/generated/sql/bnf/table_ref.bnf | 14 +- .../generated/sql/bnf/unique_column_level.bnf | 2 +- docs/generated/sql/bnf/unique_table_level.bnf | 2 +- docs/generated/sql/bnf/with_clause.bnf | 4 +- docs/generated/sql/functions.md | 8 + pkg/cmd/docgen/diagrams.go | 28 +-- pkg/gen/bnf.bzl | 2 +- pkg/gen/diagrams.bzl | 2 +- pkg/gen/docs.bzl | 2 +- pkg/internal/sqlsmith/relational.go | 16 +- pkg/sql/distsql_physical_planner.go | 8 +- pkg/sql/execinfrapb/processors_sql.proto | 4 + pkg/sql/logictest/logic.go | 3 +- .../testdata/logic_test/json_builtins | 110 ++++++++++++ pkg/sql/opt/optbuilder/scalar.go | 2 +- pkg/sql/opt/optbuilder/scope.go | 15 ++ pkg/sql/opt/optbuilder/select.go | 17 +- pkg/sql/opt/optbuilder/srfs.go | 37 +++- pkg/sql/opt/optbuilder/testdata/srfs | 89 ++++++++++ pkg/sql/opt/optbuilder/with.go | 2 +- pkg/sql/parser/parse_test.go | 2 - pkg/sql/parser/sql.y | 125 +++++++++++--- pkg/sql/parser/testdata/select_clauses | 8 + pkg/sql/rowexec/project_set.go | 5 + pkg/sql/sem/builtins/builtins.go | 3 - pkg/sql/sem/builtins/generator_builtins.go | 161 ++++++++++++++++++ pkg/sql/sem/eval/generators.go | 6 + pkg/sql/sem/tree/function_definition.go | 10 ++ pkg/sql/sem/tree/select.go | 45 ++++- 42 files changed, 697 insertions(+), 112 deletions(-) rename docs/generated/sql/bnf/{column_def.bnf => column_table_def.bnf} (70%) diff --git a/docs/generated/sql/bnf/BUILD.bazel b/docs/generated/sql/bnf/BUILD.bazel index 1fe54e78216f..4ca64e454dd2 100644 --- a/docs/generated/sql/bnf/BUILD.bazel +++ b/docs/generated/sql/bnf/BUILD.bazel @@ -68,7 +68,7 @@ FILES = [ "check_table_level", "close_cursor_stmt", "col_qualification", - "column_def", + "column_table_def", "comment", "commit_transaction", "copy_from_stmt", diff --git a/docs/generated/sql/bnf/alter_table_partition_by.bnf b/docs/generated/sql/bnf/alter_table_partition_by.bnf index b11ee35dc480..a828c0ce3a27 100644 --- a/docs/generated/sql/bnf/alter_table_partition_by.bnf +++ b/docs/generated/sql/bnf/alter_table_partition_by.bnf @@ -1,3 +1,3 @@ alter_onetable_stmt ::= - 'ALTER' 'TABLE' table_name 'PARTITION' 'ALL' 'BY' partition_by_inner ( ( ',' ( 'RENAME' opt_column column_name 'TO' column_name | 'RENAME' 'CONSTRAINT' column_name 'TO' column_name | 'ADD' column_def | 'ADD' 'IF' 'NOT' 'EXISTS' column_def | 'ADD' 'COLUMN' column_def | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_def | 'ALTER' opt_column column_name alter_column_default | 'ALTER' opt_column column_name alter_column_on_update | 'ALTER' opt_column column_name alter_column_visible | 'ALTER' opt_column column_name 'DROP' 'NOT' 'NULL' | 'ALTER' opt_column column_name 'DROP' 'STORED' | 'ALTER' opt_column column_name 'SET' 'NOT' 'NULL' | 'DROP' opt_column 'IF' 'EXISTS' column_name opt_drop_behavior | 'DROP' opt_column column_name opt_drop_behavior | 'ALTER' opt_column column_name opt_set_data 'TYPE' typename opt_collate opt_alter_column_using | 'ADD' table_constraint opt_validate_behavior | 'ADD' 'CONSTRAINT' 'IF' 'NOT' 'EXISTS' constraint_name constraint_elem opt_validate_behavior | 'ALTER' 'PRIMARY' 'KEY' 'USING' 'COLUMNS' '(' index_params ')' opt_hash_sharded opt_with_storage_parameter_list | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name opt_drop_behavior | 'DROP' 'CONSTRAINT' constraint_name opt_drop_behavior | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | ( 'PARTITION' 'BY' partition_by_inner | 'PARTITION' 'ALL' 'BY' partition_by_inner ) | 'SET' '(' storage_parameter_list ')' | 'RESET' '(' storage_parameter_key_list ')' ) ) )* - | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'PARTITION' 'ALL' 'BY' partition_by_inner ( ( ',' ( 'RENAME' opt_column column_name 'TO' column_name | 'RENAME' 'CONSTRAINT' column_name 'TO' column_name | 'ADD' column_def | 'ADD' 'IF' 'NOT' 'EXISTS' column_def | 'ADD' 'COLUMN' column_def | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_def | 'ALTER' opt_column column_name alter_column_default | 'ALTER' opt_column column_name alter_column_on_update | 'ALTER' opt_column column_name alter_column_visible | 'ALTER' opt_column column_name 'DROP' 'NOT' 'NULL' | 'ALTER' opt_column column_name 'DROP' 'STORED' | 'ALTER' opt_column column_name 'SET' 'NOT' 'NULL' | 'DROP' opt_column 'IF' 'EXISTS' column_name opt_drop_behavior | 'DROP' opt_column column_name opt_drop_behavior | 'ALTER' opt_column column_name opt_set_data 'TYPE' typename opt_collate opt_alter_column_using | 'ADD' table_constraint opt_validate_behavior | 'ADD' 'CONSTRAINT' 'IF' 'NOT' 'EXISTS' constraint_name constraint_elem opt_validate_behavior | 'ALTER' 'PRIMARY' 'KEY' 'USING' 'COLUMNS' '(' index_params ')' opt_hash_sharded opt_with_storage_parameter_list | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name opt_drop_behavior | 'DROP' 'CONSTRAINT' constraint_name opt_drop_behavior | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | ( 'PARTITION' 'BY' partition_by_inner | 'PARTITION' 'ALL' 'BY' partition_by_inner ) | 'SET' '(' storage_parameter_list ')' | 'RESET' '(' storage_parameter_key_list ')' ) ) )* + 'ALTER' 'TABLE' table_name 'PARTITION' 'ALL' 'BY' partition_by_inner ( ( ',' ( 'RENAME' opt_column column_name 'TO' column_name | 'RENAME' 'CONSTRAINT' column_name 'TO' column_name | 'ADD' column_table_def | 'ADD' 'IF' 'NOT' 'EXISTS' column_table_def | 'ADD' 'COLUMN' column_table_def | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_table_def | 'ALTER' opt_column column_name alter_column_default | 'ALTER' opt_column column_name alter_column_on_update | 'ALTER' opt_column column_name alter_column_visible | 'ALTER' opt_column column_name 'DROP' 'NOT' 'NULL' | 'ALTER' opt_column column_name 'DROP' 'STORED' | 'ALTER' opt_column column_name 'SET' 'NOT' 'NULL' | 'DROP' opt_column 'IF' 'EXISTS' column_name opt_drop_behavior | 'DROP' opt_column column_name opt_drop_behavior | 'ALTER' opt_column column_name opt_set_data 'TYPE' typename opt_collate opt_alter_column_using | 'ADD' table_constraint opt_validate_behavior | 'ADD' 'CONSTRAINT' 'IF' 'NOT' 'EXISTS' constraint_name constraint_elem opt_validate_behavior | 'ALTER' 'PRIMARY' 'KEY' 'USING' 'COLUMNS' '(' index_params ')' opt_hash_sharded opt_with_storage_parameter_list | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name opt_drop_behavior | 'DROP' 'CONSTRAINT' constraint_name opt_drop_behavior | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | ( 'PARTITION' 'BY' partition_by_inner | 'PARTITION' 'ALL' 'BY' partition_by_inner ) | 'SET' '(' storage_parameter_list ')' | 'RESET' '(' storage_parameter_key_list ')' ) ) )* + | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'PARTITION' 'ALL' 'BY' partition_by_inner ( ( ',' ( 'RENAME' opt_column column_name 'TO' column_name | 'RENAME' 'CONSTRAINT' column_name 'TO' column_name | 'ADD' column_table_def | 'ADD' 'IF' 'NOT' 'EXISTS' column_table_def | 'ADD' 'COLUMN' column_table_def | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_table_def | 'ALTER' opt_column column_name alter_column_default | 'ALTER' opt_column column_name alter_column_on_update | 'ALTER' opt_column column_name alter_column_visible | 'ALTER' opt_column column_name 'DROP' 'NOT' 'NULL' | 'ALTER' opt_column column_name 'DROP' 'STORED' | 'ALTER' opt_column column_name 'SET' 'NOT' 'NULL' | 'DROP' opt_column 'IF' 'EXISTS' column_name opt_drop_behavior | 'DROP' opt_column column_name opt_drop_behavior | 'ALTER' opt_column column_name opt_set_data 'TYPE' typename opt_collate opt_alter_column_using | 'ADD' table_constraint opt_validate_behavior | 'ADD' 'CONSTRAINT' 'IF' 'NOT' 'EXISTS' constraint_name constraint_elem opt_validate_behavior | 'ALTER' 'PRIMARY' 'KEY' 'USING' 'COLUMNS' '(' index_params ')' opt_hash_sharded opt_with_storage_parameter_list | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name opt_drop_behavior | 'DROP' 'CONSTRAINT' constraint_name opt_drop_behavior | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | ( 'PARTITION' 'BY' partition_by_inner | 'PARTITION' 'ALL' 'BY' partition_by_inner ) | 'SET' '(' storage_parameter_list ')' | 'RESET' '(' storage_parameter_key_list ')' ) ) )* diff --git a/docs/generated/sql/bnf/check_column_level.bnf b/docs/generated/sql/bnf/check_column_level.bnf index 68e1da18ce2f..9cf320eccee9 100644 --- a/docs/generated/sql/bnf/check_column_level.bnf +++ b/docs/generated/sql/bnf/check_column_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' column_name column_type 'CHECK' '(' check_expr ')' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')' + 'CREATE' 'TABLE' table_name '(' column_name column_type 'CHECK' '(' check_expr ')' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')' diff --git a/docs/generated/sql/bnf/check_table_level.bnf b/docs/generated/sql/bnf/check_table_level.bnf index 799741a33b87..4ba8e0c0c48f 100644 --- a/docs/generated/sql/bnf/check_table_level.bnf +++ b/docs/generated/sql/bnf/check_table_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' constraint_name | ) 'CHECK' '(' check_expr ')' ( table_constraints | ) ')' + 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' constraint_name | ) 'CHECK' '(' check_expr ')' ( table_constraints | ) ')' diff --git a/docs/generated/sql/bnf/column_def.bnf b/docs/generated/sql/bnf/column_table_def.bnf similarity index 70% rename from docs/generated/sql/bnf/column_def.bnf rename to docs/generated/sql/bnf/column_table_def.bnf index 8da88cb31903..24c5babebeee 100644 --- a/docs/generated/sql/bnf/column_def.bnf +++ b/docs/generated/sql/bnf/column_table_def.bnf @@ -1,2 +1,2 @@ -column_def ::= +column_table_def ::= column_name typename ( ( col_qualification ) )* diff --git a/docs/generated/sql/bnf/create_table_stmt.bnf b/docs/generated/sql/bnf/create_table_stmt.bnf index 46a04cc2cd88..ee40d76b05d7 100644 --- a/docs/generated/sql/bnf/create_table_stmt.bnf +++ b/docs/generated/sql/bnf/create_table_stmt.bnf @@ -1,3 +1,3 @@ create_table_stmt ::= - 'CREATE' opt_persistence_temp_table 'TABLE' table_name '(' ( ( ( ( column_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) ( ( ',' ( column_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) )* ) | ) ')' opt_partition_by_table ( opt_with_storage_parameter_list ) ( 'ON' 'COMMIT' 'PRESERVE' 'ROWS' ) opt_locality - | 'CREATE' opt_persistence_temp_table 'TABLE' 'IF' 'NOT' 'EXISTS' table_name '(' ( ( ( ( column_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) ( ( ',' ( column_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) )* ) | ) ')' opt_partition_by_table ( opt_with_storage_parameter_list ) ( 'ON' 'COMMIT' 'PRESERVE' 'ROWS' ) opt_locality + 'CREATE' opt_persistence_temp_table 'TABLE' table_name '(' ( ( ( ( column_table_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) ( ( ',' ( column_table_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) )* ) | ) ')' opt_partition_by_table ( opt_with_storage_parameter_list ) ( 'ON' 'COMMIT' 'PRESERVE' 'ROWS' ) opt_locality + | 'CREATE' opt_persistence_temp_table 'TABLE' 'IF' 'NOT' 'EXISTS' table_name '(' ( ( ( ( column_table_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) ( ( ',' ( column_table_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) )* ) | ) ')' opt_partition_by_table ( opt_with_storage_parameter_list ) ( 'ON' 'COMMIT' 'PRESERVE' 'ROWS' ) opt_locality diff --git a/docs/generated/sql/bnf/default_value_column_level.bnf b/docs/generated/sql/bnf/default_value_column_level.bnf index 84eee1641203..61b218b3a308 100644 --- a/docs/generated/sql/bnf/default_value_column_level.bnf +++ b/docs/generated/sql/bnf/default_value_column_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' column_name column_type 'DEFAULT' default_value ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')' + 'CREATE' 'TABLE' table_name '(' column_name column_type 'DEFAULT' default_value ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')' diff --git a/docs/generated/sql/bnf/foreign_key_column_level.bnf b/docs/generated/sql/bnf/foreign_key_column_level.bnf index a13da4f43917..d0aa363d3b5d 100644 --- a/docs/generated/sql/bnf/foreign_key_column_level.bnf +++ b/docs/generated/sql/bnf/foreign_key_column_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' column_name column_type 'REFERENCES' parent_table ( '(' ref_column_name ')' | ) ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')' + 'CREATE' 'TABLE' table_name '(' column_name column_type 'REFERENCES' parent_table ( '(' ref_column_name ')' | ) ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')' diff --git a/docs/generated/sql/bnf/foreign_key_table_level.bnf b/docs/generated/sql/bnf/foreign_key_table_level.bnf index 7d11c4e45bb5..fd714bf17eb2 100644 --- a/docs/generated/sql/bnf/foreign_key_table_level.bnf +++ b/docs/generated/sql/bnf/foreign_key_table_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' constraint_name | ) 'FOREIGN KEY' '(' ( fk_column_name ( ',' fk_column_name )* ) ')' 'REFERENCES' parent_table ( '(' ( ref_column_name ( ',' ref_column_name )* ) ')' | ) ( table_constraints | ) ')' + 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' constraint_name | ) 'FOREIGN KEY' '(' ( fk_column_name ( ',' fk_column_name )* ) ')' 'REFERENCES' parent_table ( '(' ( ref_column_name ( ',' ref_column_name )* ) ')' | ) ( table_constraints | ) ')' diff --git a/docs/generated/sql/bnf/not_null_column_level.bnf b/docs/generated/sql/bnf/not_null_column_level.bnf index e8e6da436a56..8e32276759c3 100644 --- a/docs/generated/sql/bnf/not_null_column_level.bnf +++ b/docs/generated/sql/bnf/not_null_column_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' column_name column_type 'NOT NULL' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')' + 'CREATE' 'TABLE' table_name '(' column_name column_type 'NOT NULL' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')' diff --git a/docs/generated/sql/bnf/primary_key_column_level.bnf b/docs/generated/sql/bnf/primary_key_column_level.bnf index 1a9d75fb2900..2a59ba4ed18e 100644 --- a/docs/generated/sql/bnf/primary_key_column_level.bnf +++ b/docs/generated/sql/bnf/primary_key_column_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' column_name column_type 'PRIMARY KEY' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')' + 'CREATE' 'TABLE' table_name '(' column_name column_type 'PRIMARY KEY' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')' diff --git a/docs/generated/sql/bnf/primary_key_table_level.bnf b/docs/generated/sql/bnf/primary_key_table_level.bnf index 55c0cd91874f..4b704f8d7bfa 100644 --- a/docs/generated/sql/bnf/primary_key_table_level.bnf +++ b/docs/generated/sql/bnf/primary_key_table_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' name | ) 'PRIMARY KEY' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')' + 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' name | ) 'PRIMARY KEY' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')' diff --git a/docs/generated/sql/bnf/stmt_block.bnf b/docs/generated/sql/bnf/stmt_block.bnf index c7bf3339aeee..7c83b7687003 100644 --- a/docs/generated/sql/bnf/stmt_block.bnf +++ b/docs/generated/sql/bnf/stmt_block.bnf @@ -2509,7 +2509,7 @@ table_ref ::= | 'LATERAL' select_with_parens opt_ordinality opt_alias_clause | joined_table | '(' joined_table ')' opt_ordinality alias_clause - | func_table opt_ordinality opt_alias_clause + | func_table opt_ordinality opt_func_alias_clause | 'LATERAL' func_table opt_ordinality opt_alias_clause | '[' row_source_extension_stmt ']' opt_ordinality opt_alias_clause @@ -2926,8 +2926,8 @@ bare_col_label ::= | bare_label_keywords common_table_expr ::= - table_alias_name opt_column_list 'AS' '(' preparable_stmt ')' - | table_alias_name opt_column_list 'AS' materialize_clause '(' preparable_stmt ')' + table_alias_name opt_col_def_list_no_types 'AS' '(' preparable_stmt ')' + | table_alias_name opt_col_def_list_no_types 'AS' materialize_clause '(' preparable_stmt ')' index_flags_param_list ::= ( index_flags_param ) ( ( ',' index_flags_param ) )* @@ -3005,13 +3005,17 @@ joined_table ::= | table_ref 'NATURAL' 'JOIN' table_ref alias_clause ::= - 'AS' table_alias_name opt_column_list - | table_alias_name opt_column_list + 'AS' table_alias_name opt_col_def_list_no_types + | table_alias_name opt_col_def_list_no_types func_table ::= func_expr_windowless | 'ROWS' 'FROM' '(' rowsfrom_list ')' +opt_func_alias_clause ::= + func_alias_clause + | + row_source_extension_stmt ::= delete_stmt | explain_stmt @@ -3068,10 +3072,10 @@ user_priority ::= alter_table_cmd ::= 'RENAME' opt_column column_name 'TO' column_name | 'RENAME' 'CONSTRAINT' column_name 'TO' column_name - | 'ADD' column_def - | 'ADD' 'IF' 'NOT' 'EXISTS' column_def - | 'ADD' 'COLUMN' column_def - | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_def + | 'ADD' column_table_def + | 'ADD' 'IF' 'NOT' 'EXISTS' column_table_def + | 'ADD' 'COLUMN' column_table_def + | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_table_def | 'ALTER' opt_column column_name alter_column_default | 'ALTER' opt_column column_name alter_column_on_update | 'ALTER' opt_column column_name alter_column_visible @@ -3268,7 +3272,7 @@ storage_parameter ::= storage_parameter_key '=' var_value table_elem ::= - column_def + column_table_def | index_def | family_def | table_constraint opt_validate_behavior @@ -3320,6 +3324,10 @@ bare_label_keywords ::= | 'VOLATILE' | 'SETOF' +opt_col_def_list_no_types ::= + '(' col_def_list_no_types ')' + | + materialize_clause ::= 'MATERIALIZED' | 'NOT' 'MATERIALIZED' @@ -3384,6 +3392,10 @@ join_qual ::= rowsfrom_list ::= ( rowsfrom_item ) ( ( ',' rowsfrom_item ) )* +func_alias_clause ::= + 'AS' table_alias_name opt_col_def_list + | table_alias_name opt_col_def_list + opt_varying ::= 'VARYING' | @@ -3398,7 +3410,7 @@ opt_column ::= 'COLUMN' | -column_def ::= +column_table_def ::= column_name typename col_qual_list alter_column_default ::= @@ -3604,6 +3616,9 @@ common_func_opt_item ::= | 'LEAKPROOF' | 'NOT' 'LEAKPROOF' +col_def_list_no_types ::= + ( name ) ( ( ',' name ) )* + group_by_item ::= a_expr @@ -3615,7 +3630,10 @@ join_outer ::= | rowsfrom_item ::= - func_expr_windowless + func_expr_windowless opt_func_alias_clause + +opt_col_def_list ::= + '(' col_def_list ')' char_aliases ::= 'CHAR' @@ -3691,6 +3709,9 @@ func_arg_class ::= param_name ::= type_function_name +col_def_list ::= + ( col_def ) ( ( ',' col_def ) )* + col_qualification ::= 'CONSTRAINT' constraint_name col_qualification_elem | col_qualification_elem @@ -3720,6 +3741,10 @@ opt_partition_by ::= create_as_param ::= column_name +col_def ::= + name + | name typename + col_qualification_elem ::= 'NOT' 'NULL' | 'NULL' diff --git a/docs/generated/sql/bnf/table_ref.bnf b/docs/generated/sql/bnf/table_ref.bnf index dd24ece24f8e..711d02ef6ca6 100644 --- a/docs/generated/sql/bnf/table_ref.bnf +++ b/docs/generated/sql/bnf/table_ref.bnf @@ -1,9 +1,9 @@ table_ref ::= - table_name ( '@' index_name | ) ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) - | '(' select_stmt ')' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) - | 'LATERAL' '(' select_stmt ')' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) + table_name ( '@' index_name | ) ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name opt_col_def_list_no_types | table_alias_name opt_col_def_list_no_types ) | ) + | '(' select_stmt ')' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name opt_col_def_list_no_types | table_alias_name opt_col_def_list_no_types ) | ) + | 'LATERAL' '(' select_stmt ')' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name opt_col_def_list_no_types | table_alias_name opt_col_def_list_no_types ) | ) | joined_table - | '(' joined_table ')' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) - | func_application ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) - | 'LATERAL' func_application ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) - | '[' row_source_extension_stmt ']' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) + | '(' joined_table ')' ( 'WITH' 'ORDINALITY' | ) ( 'AS' table_alias_name opt_col_def_list_no_types | table_alias_name opt_col_def_list_no_types ) + | func_application ( 'WITH' 'ORDINALITY' | ) opt_func_alias_clause + | 'LATERAL' func_application ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name opt_col_def_list_no_types | table_alias_name opt_col_def_list_no_types ) | ) + | '[' row_source_extension_stmt ']' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name opt_col_def_list_no_types | table_alias_name opt_col_def_list_no_types ) | ) diff --git a/docs/generated/sql/bnf/unique_column_level.bnf b/docs/generated/sql/bnf/unique_column_level.bnf index a1621a96ba97..784a79aa2ab8 100644 --- a/docs/generated/sql/bnf/unique_column_level.bnf +++ b/docs/generated/sql/bnf/unique_column_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' column_name column_type 'UNIQUE' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')' + 'CREATE' 'TABLE' table_name '(' column_name column_type 'UNIQUE' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')' diff --git a/docs/generated/sql/bnf/unique_table_level.bnf b/docs/generated/sql/bnf/unique_table_level.bnf index dcd0abc3e5e0..e57b084f555e 100644 --- a/docs/generated/sql/bnf/unique_table_level.bnf +++ b/docs/generated/sql/bnf/unique_table_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' name | ) 'UNIQUE' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')' + 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' name | ) 'UNIQUE' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')' diff --git a/docs/generated/sql/bnf/with_clause.bnf b/docs/generated/sql/bnf/with_clause.bnf index 05b08523f7e4..a512476d702d 100644 --- a/docs/generated/sql/bnf/with_clause.bnf +++ b/docs/generated/sql/bnf/with_clause.bnf @@ -1,3 +1,3 @@ with_clause ::= - 'WITH' ( ( ( table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' '(' preparable_stmt ')' | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) ( ( ',' ( table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' '(' preparable_stmt ')' | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) )* ) ( insert_stmt | update_stmt | delete_stmt | upsert_stmt | select_stmt ) - | 'WITH' 'RECURSIVE' ( ( ( table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' '(' preparable_stmt ')' | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) ( ( ',' ( table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' '(' preparable_stmt ')' | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) )* ) ( insert_stmt | update_stmt | delete_stmt | upsert_stmt | select_stmt ) + 'WITH' ( ( ( table_alias_name opt_col_def_list_no_types 'AS' '(' preparable_stmt ')' | table_alias_name opt_col_def_list_no_types 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) ( ( ',' ( table_alias_name opt_col_def_list_no_types 'AS' '(' preparable_stmt ')' | table_alias_name opt_col_def_list_no_types 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) )* ) ( insert_stmt | update_stmt | delete_stmt | upsert_stmt | select_stmt ) + | 'WITH' 'RECURSIVE' ( ( ( table_alias_name opt_col_def_list_no_types 'AS' '(' preparable_stmt ')' | table_alias_name opt_col_def_list_no_types 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) ( ( ',' ( table_alias_name opt_col_def_list_no_types 'AS' '(' preparable_stmt ')' | table_alias_name opt_col_def_list_no_types 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) )* ) ( insert_stmt | update_stmt | delete_stmt | upsert_stmt | select_stmt ) diff --git a/docs/generated/sql/functions.md b/docs/generated/sql/functions.md index afba41bdded1..2dc01aefc883 100644 --- a/docs/generated/sql/functions.md +++ b/docs/generated/sql/functions.md @@ -1260,6 +1260,10 @@ the locality flag on node startup. Returns an error if no region is set.

Stable json_populate_recordset(base: anyelement, from_json: jsonb) → anyelement

Expands the outermost array of objects in from_json to a set of rows whose columns match the record type defined by base

Stable +json_to_record(input: jsonb) → tuple

Builds an arbitrary record from a JSON object.

+Stable +json_to_recordset(input: jsonb) → tuple

Builds an arbitrary set of records from a JSON array of objects.

+Stable jsonb_array_elements(input: jsonb) → jsonb

Expands a JSON array to a set of JSON values.

Immutable jsonb_array_elements_text(input: jsonb) → string

Expands a JSON array to a set of text values.

@@ -1274,6 +1278,10 @@ the locality flag on node startup. Returns an error if no region is set.

Stable jsonb_populate_recordset(base: anyelement, from_json: jsonb) → anyelement

Expands the outermost array of objects in from_json to a set of rows whose columns match the record type defined by base

Stable +jsonb_to_record(input: jsonb) → tuple

Builds an arbitrary record from a JSON object.

+Stable +jsonb_to_recordset(input: jsonb) → tuple

Builds an arbitrary set of records from a JSON array of objects.

+Stable pg_get_keywords() → tuple{string AS word, string AS catcode, string AS catdesc}

Produces a virtual table containing the keywords known to the SQL parser.

Immutable pg_options_to_table(options: string[]) → tuple{string AS option_name, string AS option_value}

Converts the options array format to a table.

diff --git a/pkg/cmd/docgen/diagrams.go b/pkg/cmd/docgen/diagrams.go index 0422d3e02fd5..338d27f8a68e 100644 --- a/pkg/cmd/docgen/diagrams.go +++ b/pkg/cmd/docgen/diagrams.go @@ -343,7 +343,7 @@ var specs = []stmtSpec{ { name: "add_column", stmt: "alter_onetable_stmt", - inline: []string{"alter_table_cmds", "alter_table_cmd", "column_def", "col_qual_list"}, + inline: []string{"alter_table_cmds", "alter_table_cmd", "column_table_def", "col_qual_list"}, regreplace: map[string]string{ ` \( \( col_qualification \) \)\* .*`: `( ( col_qualification ) )*`, }, @@ -500,7 +500,7 @@ var specs = []stmtSpec{ { name: "alter_table", stmt: "alter_onetable_stmt", - inline: []string{"alter_table_cmds", "alter_table_cmd", "column_def", "opt_drop_behavior", "alter_column_default", "opt_column", "opt_set_data", "table_constraint", "opt_collate", "opt_alter_column_using"}, + inline: []string{"alter_table_cmds", "alter_table_cmd", "column_table_def", "opt_drop_behavior", "alter_column_default", "opt_column", "opt_set_data", "table_constraint", "opt_collate", "opt_alter_column_using"}, replace: map[string]string{ "'VALIDATE' 'CONSTRAINT' name": "", "opt_validate_behavior": "", @@ -594,18 +594,18 @@ var specs = []stmtSpec{ { name: "check_column_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'CHECK' '(' check_expr ')' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'CHECK' '(' check_expr ')' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')'"}, unlink: []string{"table_name", "column_name", "column_type", "check_expr", "column_constraints", "table_constraints"}, }, { name: "check_table_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' constraint_name | ) 'CHECK' '(' check_expr ')' ( table_constraints | ) ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' constraint_name | ) 'CHECK' '(' check_expr ')' ( table_constraints | ) ')'"}, unlink: []string{"table_name", "check_expr", "table_constraints"}, }, { - name: "column_def", - stmt: "column_def", + name: "column_table_def", + stmt: "column_table_def", inline: []string{"col_qual_list"}, }, { @@ -772,7 +772,7 @@ var specs = []stmtSpec{ name: "default_value_column_level", stmt: "stmt_block", replace: map[string]string{ - " stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'DEFAULT' default_value ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'", + " stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'DEFAULT' default_value ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')'", }, unlink: []string{"table_name", "column_name", "column_type", "default_value", "table_constraints"}, }, @@ -948,13 +948,13 @@ var specs = []stmtSpec{ { name: "foreign_key_column_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'REFERENCES' parent_table ( '(' ref_column_name ')' | ) ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'REFERENCES' parent_table ( '(' ref_column_name ')' | ) ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')'"}, unlink: []string{"table_name", "column_name", "column_type", "parent_table", "table_constraints"}, }, { name: "foreign_key_table_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' constraint_name | ) 'FOREIGN KEY' '(' ( fk_column_name ( ',' fk_column_name )* ) ')' 'REFERENCES' parent_table ( '(' ( ref_column_name ( ',' ref_column_name )* ) ')' | ) ( table_constraints | ) ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' constraint_name | ) 'FOREIGN KEY' '(' ( fk_column_name ( ',' fk_column_name )* ) ')' 'REFERENCES' parent_table ( '(' ( ref_column_name ( ',' ref_column_name )* ) ')' | ) ( table_constraints | ) ')'"}, unlink: []string{"table_name", "column_name", "parent_table", "table_constraints"}, }, { @@ -1034,7 +1034,7 @@ var specs = []stmtSpec{ { name: "not_null_column_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'NOT NULL' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'NOT NULL' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')'"}, unlink: []string{"table_name", "column_name", "column_type", "table_constraints"}, }, { @@ -1056,13 +1056,13 @@ var specs = []stmtSpec{ { name: "primary_key_column_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'PRIMARY KEY' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'PRIMARY KEY' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')'"}, unlink: []string{"table_name", "column_name", "column_type", "table_constraints"}, }, { name: "primary_key_table_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' name | ) 'PRIMARY KEY' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' name | ) 'PRIMARY KEY' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')'"}, unlink: []string{"table_name", "column_name", "table_constraints"}, }, { @@ -1467,13 +1467,13 @@ var specs = []stmtSpec{ { name: "unique_column_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'UNIQUE' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'UNIQUE' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')'"}, unlink: []string{"table_name", "column_name", "column_type", "table_constraints"}, }, { name: "unique_table_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' name | ) 'UNIQUE' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' name | ) 'UNIQUE' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')'"}, unlink: []string{"table_name", "check_expr", "table_constraints"}, }, { diff --git a/pkg/gen/bnf.bzl b/pkg/gen/bnf.bzl index ed9518d5e80f..a08fbc491801 100644 --- a/pkg/gen/bnf.bzl +++ b/pkg/gen/bnf.bzl @@ -68,7 +68,7 @@ BNF_SRCS = [ "//docs/generated/sql/bnf:check_table_level.bnf", "//docs/generated/sql/bnf:close_cursor_stmt.bnf", "//docs/generated/sql/bnf:col_qualification.bnf", - "//docs/generated/sql/bnf:column_def.bnf", + "//docs/generated/sql/bnf:column_table_def.bnf", "//docs/generated/sql/bnf:comment.bnf", "//docs/generated/sql/bnf:commit_transaction.bnf", "//docs/generated/sql/bnf:copy_from_stmt.bnf", diff --git a/pkg/gen/diagrams.bzl b/pkg/gen/diagrams.bzl index a67b32ed92a4..b945ebec2fbb 100644 --- a/pkg/gen/diagrams.bzl +++ b/pkg/gen/diagrams.bzl @@ -67,7 +67,7 @@ DIAGRAMS_SRCS = [ "//docs/generated/sql/bnf:check_table_level.html", "//docs/generated/sql/bnf:close_cursor.html", "//docs/generated/sql/bnf:col_qualification.html", - "//docs/generated/sql/bnf:column_def.html", + "//docs/generated/sql/bnf:column_table_def.html", "//docs/generated/sql/bnf:comment.html", "//docs/generated/sql/bnf:commit_transaction.html", "//docs/generated/sql/bnf:copy_from.html", diff --git a/pkg/gen/docs.bzl b/pkg/gen/docs.bzl index 1226b925ce7a..55fa65f2c247 100644 --- a/pkg/gen/docs.bzl +++ b/pkg/gen/docs.bzl @@ -80,7 +80,7 @@ DOCS_SRCS = [ "//docs/generated/sql/bnf:check_table_level.bnf", "//docs/generated/sql/bnf:close_cursor_stmt.bnf", "//docs/generated/sql/bnf:col_qualification.bnf", - "//docs/generated/sql/bnf:column_def.bnf", + "//docs/generated/sql/bnf:column_table_def.bnf", "//docs/generated/sql/bnf:comment.bnf", "//docs/generated/sql/bnf:commit_transaction.bnf", "//docs/generated/sql/bnf:copy_from_stmt.bnf", diff --git a/pkg/internal/sqlsmith/relational.go b/pkg/internal/sqlsmith/relational.go index 2089eeb0a98f..509c0a3ca3ee 100644 --- a/pkg/internal/sqlsmith/relational.go +++ b/pkg/internal/sqlsmith/relational.go @@ -478,11 +478,11 @@ func (s *Smither) makeWith() (*tree.With, tableRefs) { } alias := s.name("with") tblName := tree.NewUnqualifiedTableName(alias) - cols := make(tree.NameList, len(stmtRefs)) + cols := make(tree.ColumnDefList, len(stmtRefs)) defs := make([]*tree.ColumnTableDef, len(stmtRefs)) for i, r := range stmtRefs { var err error - cols[i] = r.item.ColumnName + cols[i].Name = r.item.ColumnName defs[i], err = tree.NewColumnTableDef(r.item.ColumnName, r.typ, false /* isSerial */, nil) if err != nil { panic(err) @@ -561,15 +561,15 @@ func makeSelectTable(s *Smither, refs colRefs, forJoin bool) (tree.TableExpr, co } table := s.name("tab") - names := make(tree.NameList, len(stmtRefs)) + names := make(tree.ColumnDefList, len(stmtRefs)) clauseRefs := make(colRefs, len(stmtRefs)) for i, ref := range stmtRefs { - names[i] = s.name("col") + names[i].Name = s.name("col") clauseRefs[i] = &colRef{ typ: ref.typ, item: tree.NewColumnItem( tree.NewUnqualifiedTableName(table), - names[i], + names[i].Name, ), } } @@ -1163,15 +1163,15 @@ func makeValues(s *Smither, desiredTypes []*types.T, refs colRefs) (tree.TableEx values.Rows[i] = tuple } table := s.name("tab") - names := make(tree.NameList, len(desiredTypes)) + names := make(tree.ColumnDefList, len(desiredTypes)) valuesRefs := make(colRefs, len(desiredTypes)) for i, typ := range desiredTypes { - names[i] = s.name("col") + names[i].Name = s.name("col") valuesRefs[i] = &colRef{ typ: typ, item: tree.NewColumnItem( tree.NewUnqualifiedTableName(table), - names[i], + names[i].Name, ), } } diff --git a/pkg/sql/distsql_physical_planner.go b/pkg/sql/distsql_physical_planner.go index 5c312937f2d7..a93a8bc6c51f 100644 --- a/pkg/sql/distsql_physical_planner.go +++ b/pkg/sql/distsql_physical_planner.go @@ -3391,9 +3391,10 @@ func createProjectSetSpec( planCtx *PlanningCtx, n *projectSetPlanningInfo, indexVarMap []int, ) (*execinfrapb.ProjectSetSpec, error) { spec := execinfrapb.ProjectSetSpec{ - Exprs: make([]execinfrapb.Expression, len(n.exprs)), - GeneratedColumns: make([]*types.T, len(n.columns)-n.numColsInSource), - NumColsPerGen: make([]uint32, len(n.exprs)), + Exprs: make([]execinfrapb.Expression, len(n.exprs)), + GeneratedColumns: make([]*types.T, len(n.columns)-n.numColsInSource), + GeneratedColumnLabels: make([]string, len(n.columns)-n.numColsInSource), + NumColsPerGen: make([]uint32, len(n.exprs)), } for i, expr := range n.exprs { var err error @@ -3404,6 +3405,7 @@ func createProjectSetSpec( } for i, col := range n.columns[n.numColsInSource:] { spec.GeneratedColumns[i] = col.Typ + spec.GeneratedColumnLabels[i] = col.Name } for i, n := range n.numColsPerGen { spec.NumColsPerGen[i] = uint32(n) diff --git a/pkg/sql/execinfrapb/processors_sql.proto b/pkg/sql/execinfrapb/processors_sql.proto index 684ce304fb83..d9fa0913ab70 100644 --- a/pkg/sql/execinfrapb/processors_sql.proto +++ b/pkg/sql/execinfrapb/processors_sql.proto @@ -925,6 +925,10 @@ message ProjectSetSpec { // The number of columns each expression returns. Same length as exprs. repeated uint32 num_cols_per_gen = 3; + + // Column labels for the generated values. Needed for some builtin functions + // (like record_to_json) that require the column labels to do their jobs. + repeated string generated_column_labels = 4; } // WindowerSpec is the specification of a processor that performs computations diff --git a/pkg/sql/logictest/logic.go b/pkg/sql/logictest/logic.go index 7a41b3e66753..2d4f82abf83d 100644 --- a/pkg/sql/logictest/logic.go +++ b/pkg/sql/logictest/logic.go @@ -2460,7 +2460,8 @@ func (t *logicTest) processSubtest( // Don't error if --rewrite is specified, since the expected // results are ignored in that case. if !*rewriteResultsInTestfiles && len(results) != len(query.colTypes) { - return errors.Errorf("expected results are invalid: unexpected column count") + return errors.Errorf("expected results are invalid: unexpected column count %d != %d (%s)", + len(results), len(query.colTypes), results) } query.expectedResults = append(query.expectedResults, results...) } diff --git a/pkg/sql/logictest/testdata/logic_test/json_builtins b/pkg/sql/logictest/testdata/logic_test/json_builtins index 1125c9fb8bfc..c5538c131af3 100644 --- a/pkg/sql/logictest/testdata/logic_test/json_builtins +++ b/pkg/sql/logictest/testdata/logic_test/json_builtins @@ -1361,3 +1361,113 @@ SELECT * FROM json_populate_recordset(NULL::testtab, '[{"i": 3, "ia": [1,2,3], " ---- 3 {1,2,3} foo {a,b} 2017-01-01 00:00:00 +0000 +0000 {"a": "b", "c": 3, "d": [1, false, true, null, {"1": "2"}]} NULL NULL NULL NULL NULL NULL + +query error invalid non-object argument to json_to_record +SELECT * FROM json_to_record('3') AS t(a INT) + +query error invalid non-object argument to json_to_record +SELECT * FROM json_to_record('"a"') AS t(a TEXT) + +query error invalid non-object argument to json_to_record +SELECT * FROM json_to_record('null') AS t(a INT) + +query error invalid non-object argument to json_to_record +SELECT * FROM json_to_record('true') AS t(a INT) + +query error invalid non-object argument to json_to_record +SELECT * FROM json_to_record('[1,2]') AS t(a INT) + +query error column definition list is required for functions returning \"record\" +SELECT * FROM json_to_record('{"a": "b"}') AS t(a) + +query error column definition list is required for functions returning \"record\" +SELECT * FROM json_to_record('{"a": "b"}') + +# Test that non-record generators don't permit col definition lists (with types). +query error a column definition list is only allowed for functions returning \"record\" +SELECT * FROM generate_series(1,10) g(g int) + +statement ok +CREATE TABLE j (j) AS SELECT '{ + "str": "a", + "int": 1, + "bool": true, + "nul": null, + "dec": 2.45, + "arrint": [1,2], + "arrmixed": [1,2,true], + "arrstr": ["a", "b"], + "arrbool": [true, false], + "obj": {"i": 3, "t": "blah", "z": true} + }'::JSONB + +statement ok +INSERT INTO j VALUES('{"str": "zzz"}') + +query TIBTFTTTTT +SELECT t.* FROM j, json_to_record(j.j) AS t( + str TEXT, + int INT, + bool BOOL, + nul TEXT, + dec DECIMAL, + arrint INT[], + arrmixed TEXT, + arrstr TEXT[], + arrbool BOOL[], + obj TEXT +) ORDER BY rowid +---- +a 1 true NULL 2.45 {1,2} [1, 2, true] {a,b} {t,f} {"i": 3, "t": "blah", "z": true} +zzz NULL NULL NULL NULL NULL NULL NULL NULL NULL + +# Test that mismatched types return an error +query error could not parse \"true\" as type int +SELECT t.bool FROM j, json_to_record(j.j) AS t(bool INT) + +# But types can be coerced. +query TT rowsort +SELECT t.* FROM j, json_to_record(j.j) AS t(int TEXT, bool TEXT) +---- +1 true +NULL NULL + +# Mixed type arrays +query error could not parse \"2\" as type bool +SELECT t.arrmixed FROM j, json_to_record(j.j) AS t(arrmixed BOOL[]) + +# Record with custom type +query T rowsort +SELECT t.obj FROM j, json_to_record(j.j) AS t(obj testtab) +---- +(3,,blah,,,) +NULL + +# Test json_to_recordset +query TIBTFTTTTT +SELECT t.* FROM j, json_to_recordset(j.j || '[]' || j.j) AS t( + str TEXT, + int INT, + bool BOOL, + nul TEXT, + dec DECIMAL, + arrint INT[], + arrmixed TEXT, + arrstr TEXT[], + arrbool BOOL[], + obj TEXT +) ORDER BY rowid +---- +a 1 true NULL 2.45 {1,2} [1, 2, true] {a,b} {t,f} {"i": 3, "t": "blah", "z": true} +a 1 true NULL 2.45 {1,2} [1, 2, true] {a,b} {t,f} {"i": 3, "t": "blah", "z": true} +zzz NULL NULL NULL NULL NULL NULL NULL NULL NULL +zzz NULL NULL NULL NULL NULL NULL NULL NULL NULL + +query TT rowsort +SELECT * FROM jsonb_to_recordset('[{"foo": "bar"}, {"foo": "bar2"}]') AS t(foo TEXT), + jsonb_to_recordset('[{"foo": "blah"}, {"foo": "blah2"}]') AS u(foo TEXT) +---- +bar blah +bar blah2 +bar2 blah +bar2 blah2 diff --git a/pkg/sql/opt/optbuilder/scalar.go b/pkg/sql/opt/optbuilder/scalar.go index bee9671e88ad..f8b51d7c2846 100644 --- a/pkg/sql/opt/optbuilder/scalar.go +++ b/pkg/sql/opt/optbuilder/scalar.go @@ -561,7 +561,7 @@ func (b *Builder) buildFunction( }) if f.ResolvedOverload().Class == tree.GeneratorClass { - return b.finishBuildGeneratorFunction(f, out, inScope, outScope, outCol) + return b.finishBuildGeneratorFunction(f, def, out, inScope, outScope, outCol) } // Add a dependency on sequences that are used as a string argument. diff --git a/pkg/sql/opt/optbuilder/scope.go b/pkg/sql/opt/optbuilder/scope.go index ad6d1c446447..1d2f98b2c18f 100644 --- a/pkg/sql/opt/optbuilder/scope.go +++ b/pkg/sql/opt/optbuilder/scope.go @@ -101,6 +101,10 @@ type scope struct { // scopes. ctes map[string]*cteSource + // lastAlias is set to the last data source alias we've come across, if we + // are processing a data source with an alias. + lastAlias *tree.AliasClause + // context is the current context in the SQL query (e.g., "SELECT" or // "HAVING"). It is used for error messages and to identify scoping errors // (e.g., aggregates are not allowed in the FROM clause of their own query @@ -629,6 +633,17 @@ func (s *scope) findExistingCol(expr tree.TypedExpr, allowSideEffects bool) *sco return col } +// getLastAlias gets the most recently seen AliasClause (e.g. AS t(a,b)) in this +// statement, or nil if there was none. +func (s *scope) getLastAlias() *tree.AliasClause { + for ; s != nil; s = s.parent { + if s.lastAlias != nil { + return s.lastAlias + } + } + return nil +} + // startAggFunc is called when the builder starts building an aggregate // function. It is used to disallow nested aggregates and ensure that a // grouping error is not called on the aggregate arguments. For example: diff --git a/pkg/sql/opt/optbuilder/select.go b/pkg/sql/opt/optbuilder/select.go index db1e40911298..7c5848838435 100644 --- a/pkg/sql/opt/optbuilder/select.go +++ b/pkg/sql/opt/optbuilder/select.go @@ -46,6 +46,13 @@ func (b *Builder) buildDataSource( inScope.atRoot = prevAtRoot }(inScope.atRoot) inScope.atRoot = false + /* + if b.lastAlias != nil { + // Make sure we reset the last alias state if it's been set, so we don't + // keep it past a single level of recursion. + defer func() { b.lastAlias = nil }() + } + */ // NB: The case statements are sorted lexicographically. switch source := texpr.(type) { case *tree.AliasedTableExpr: @@ -54,12 +61,16 @@ func (b *Builder) buildDataSource( telemetry.Inc(sqltelemetry.IndexHintSelectUseCounter) indexFlags = source.IndexFlags } + previousLastAlias := inScope.lastAlias if source.As.Alias != "" { + inScope.lastAlias = &source.As locking = locking.filter(source.As.Alias) } outScope = b.buildDataSource(source.Expr, indexFlags, locking, inScope) + inScope.lastAlias = previousLastAlias + if source.Ordinality { outScope = b.buildWithOrdinality(outScope) } @@ -331,7 +342,7 @@ func (b *Builder) renameSource(as tree.AliasClause, scope *scope) { // table name. noColNameSpecified := len(colAlias) == 0 if scope.isAnonymousTable() && noColNameSpecified && scope.singleSRFColumn { - colAlias = tree.NameList{as.Alias} + colAlias = tree.ColumnDefList{tree.ColumnDef{Name: as.Alias}} } // If an alias was specified, use that to qualify the column names. @@ -361,11 +372,11 @@ func (b *Builder) renameSource(as tree.AliasClause, scope *scope) { if col.visibility != visible { continue } - col.name = scopeColName(colAlias[aliasIdx]) + col.name = scopeColName(colAlias[aliasIdx].Name) if isScan { // Override column metadata alias. colMeta := b.factory.Metadata().ColumnMeta(col.id) - colMeta.Alias = string(colAlias[aliasIdx]) + colMeta.Alias = string(colAlias[aliasIdx].Name) } aliasIdx++ } diff --git a/pkg/sql/opt/optbuilder/srfs.go b/pkg/sql/opt/optbuilder/srfs.go index 35625bc82eb5..45902d3b78f6 100644 --- a/pkg/sql/opt/optbuilder/srfs.go +++ b/pkg/sql/opt/optbuilder/srfs.go @@ -15,6 +15,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util/errorutil/unimplemented" @@ -143,12 +145,45 @@ func (b *Builder) buildZip(exprs tree.Exprs, inScope *scope) (outScope *scope) { // (SRF) such as generate_series() or unnest(). It synthesizes new columns in // outScope for each of the SRF's output columns. func (b *Builder) finishBuildGeneratorFunction( - f *tree.FuncExpr, fn opt.ScalarExpr, inScope, outScope *scope, outCol *scopeColumn, + f *tree.FuncExpr, + def *tree.FunctionDefinition, + fn opt.ScalarExpr, + inScope, outScope *scope, + outCol *scopeColumn, ) (out opt.ScalarExpr) { + lastAlias := inScope.getLastAlias() + if def.ReturnsRecordType { + if lastAlias == nil { + panic(pgerror.New(pgcode.Syntax, "a column definition list is required for functions returning \"record\"")) + } + } else if lastAlias != nil { + // Non-record type return with a table alias that includes types is not + // permitted. + for _, c := range lastAlias.Cols { + if c.Type != nil { + panic(pgerror.Newf(pgcode.Syntax, "a column definition list is only allowed for functions returning \"record\"")) + } + } + } // Add scope columns. if outCol != nil { // Single-column return type. b.populateSynthesizedColumn(outCol, fn) + } else if def.ReturnsRecordType && lastAlias != nil && len(lastAlias.Cols) > 0 { + // If we're building a generator function that returns a record type, like + // json_to_record, we need to know the alias that was assigned to the + // generator function - without that, we won't know the list of columns + // to output. + for _, c := range lastAlias.Cols { + if c.Type == nil { + panic(pgerror.Newf(pgcode.Syntax, "a column definition list is required for functions returning \"record\"")) + } + typ, err := tree.ResolveType(b.ctx, c.Type, b.semaCtx.TypeResolver) + if err != nil { + panic(err) + } + b.synthesizeColumn(outScope, scopeColName(c.Name), typ, nil, fn) + } } else { // Multi-column return type. Use the tuple labels in the SRF's return type // as column aliases. diff --git a/pkg/sql/opt/optbuilder/testdata/srfs b/pkg/sql/opt/optbuilder/testdata/srfs index 612356e2614f..f1c96dfa37f8 100644 --- a/pkg/sql/opt/optbuilder/testdata/srfs +++ b/pkg/sql/opt/optbuilder/testdata/srfs @@ -1000,3 +1000,92 @@ project └── projections ├── (unnest:6).a [as=a:7] └── (unnest:6).b [as=b:8] + +# Test record-returning SRFs, which need to keep track of the most recently set +# AS clause above them. + +exec-ddl +CREATE TABLE j (j json) +---- + +build +SELECT t.a, t.z FROM j, json_to_record(j.j) AS t(a text, z int) +---- +project + ├── columns: a:5 z:6 + └── inner-join-apply + ├── columns: j:1 rowid:2!null crdb_internal_mvcc_timestamp:3 tableoid:4 a:5 z:6 + ├── scan j + │ └── columns: j:1 rowid:2!null crdb_internal_mvcc_timestamp:3 tableoid:4 + ├── project-set + │ ├── columns: a:5 z:6 + │ ├── values + │ │ └── () + │ └── zip + │ └── json_to_record(j:1) + └── filters (true) + +# Test that outer AS clauses don't get incorrectly propagated down into +# json_to_record. +build +SELECT blah.x || 'foo' FROM (SELECT t.a, t.z FROM j, json_to_record(j.j) AS t(a text, z int)) AS blah(x) +---- +project + ├── columns: "?column?":7 + ├── project + │ ├── columns: a:5 z:6 + │ └── inner-join-apply + │ ├── columns: j:1 rowid:2!null crdb_internal_mvcc_timestamp:3 tableoid:4 a:5 z:6 + │ ├── scan j + │ │ └── columns: j:1 rowid:2!null crdb_internal_mvcc_timestamp:3 tableoid:4 + │ ├── project-set + │ │ ├── columns: a:5 z:6 + │ │ ├── values + │ │ │ └── () + │ │ └── zip + │ │ └── json_to_record(j:1) + │ └── filters (true) + └── projections + └── a:5 || 'foo' [as="?column?":7] + +# Test that outer AS clauses don't get incorrectly propagated down into +# json_to_record, even if an inner one is missing. +build +SELECT * FROM (SELECT t.a, t.z FROM j, json_to_record(j.j)) AS blah(a) +---- +error (42601): a column definition list is required for functions returning "record" + +# Test that nested AS clauses and json_to_record statements work okay. +build +SELECT blah.a || 'foo', * FROM +(SELECT t.a, t.z, j.x FROM (SELECT * FROM j, json_to_record('{"foo": "bar"}') AS x(foo text)) j(j, x), + json_to_record(j.j) AS t(a text, z int)) AS blah(a, z, x) +---- +project + ├── columns: "?column?":8 a:6 z:7 x:5 + ├── project + │ ├── columns: foo:5 a:6 z:7 + │ └── inner-join-apply + │ ├── columns: j:1 foo:5 a:6 z:7 + │ ├── project + │ │ ├── columns: j:1 foo:5 + │ │ └── inner-join-apply + │ │ ├── columns: j:1 rowid:2!null crdb_internal_mvcc_timestamp:3 tableoid:4 foo:5 + │ │ ├── scan j + │ │ │ └── columns: j:1 rowid:2!null crdb_internal_mvcc_timestamp:3 tableoid:4 + │ │ ├── project-set + │ │ │ ├── columns: foo:5 + │ │ │ ├── values + │ │ │ │ └── () + │ │ │ └── zip + │ │ │ └── json_to_record('{"foo": "bar"}') + │ │ └── filters (true) + │ ├── project-set + │ │ ├── columns: a:6 z:7 + │ │ ├── values + │ │ │ └── () + │ │ └── zip + │ │ └── json_to_record(j:1) + │ └── filters (true) + └── projections + └── a:6 || 'foo' [as="?column?":8] diff --git a/pkg/sql/opt/optbuilder/with.go b/pkg/sql/opt/optbuilder/with.go index 2dee1a4bc40c..48f1ef9864bd 100644 --- a/pkg/sql/opt/optbuilder/with.go +++ b/pkg/sql/opt/optbuilder/with.go @@ -382,7 +382,7 @@ func (b *Builder) getCTECols(cteScope *scope, name tree.AliasClause) physical.Pr )) } for i := range presentation { - presentation[i].Alias = string(name.Cols[i]) + presentation[i].Alias = string(name.Cols[i].Name) } return presentation } diff --git a/pkg/sql/parser/parse_test.go b/pkg/sql/parser/parse_test.go index 2fc2b420d2cd..149779ed55b5 100644 --- a/pkg/sql/parser/parse_test.go +++ b/pkg/sql/parser/parse_test.go @@ -526,8 +526,6 @@ func TestUnimplementedSyntax(t *testing.T) { {`INSERT INTO foo(a, a.b) VALUES (1,2)`, 27792, ``, ``}, - {`SELECT * FROM ROWS FROM (a(b) AS (d))`, 0, `ROWS FROM with col_def_list`, ``}, - {`SELECT a(b) 'c'`, 0, `a(...) SCONST`, ``}, {`SELECT UNIQUE (SELECT b)`, 0, `UNIQUE predicate`, ``}, {`SELECT GROUPING (a,b,c)`, 0, `d_expr grouping`, ``}, diff --git a/pkg/sql/parser/sql.y b/pkg/sql/parser/sql.y index 03351e83b417..c0b0bd709ddd 100644 --- a/pkg/sql/parser/sql.y +++ b/pkg/sql/parser/sql.y @@ -359,9 +359,15 @@ func (u *sqlSymUnion) slct() *tree.Select { func (u *sqlSymUnion) selectStmt() tree.SelectStatement { return u.val.(tree.SelectStatement) } -func (u *sqlSymUnion) colDef() *tree.ColumnTableDef { +func (u *sqlSymUnion) colTableDef() *tree.ColumnTableDef { return u.val.(*tree.ColumnTableDef) } +func (u *sqlSymUnion) colDef() tree.ColumnDef { + return u.val.(tree.ColumnDef) +} +func (u *sqlSymUnion) colDefList() tree.ColumnDefList { + return u.val.(tree.ColumnDefList) +} func (u *sqlSymUnion) constraintDef() tree.ConstraintTableDef { return u.val.(tree.ConstraintTableDef) } @@ -1371,7 +1377,8 @@ func (u *sqlSymUnion) routineBody() *tree.RoutineBody { %type first_or_next %type insert_rest -%type opt_col_def_list +%type opt_col_def_list col_def_list opt_col_def_list_no_types col_def_list_no_types +%type col_def %type <*tree.OnConflict> on_conflict %type begin_transaction @@ -1380,7 +1387,7 @@ func (u *sqlSymUnion) routineBody() *tree.RoutineBody { %type opt_hash_sharded_bucket_count %type <*tree.ShardedIndexDef> opt_hash_sharded %type opt_storing -%type <*tree.ColumnTableDef> column_def +%type <*tree.ColumnTableDef> column_table_def %type table_elem %type where_clause opt_where_clause %type <*tree.ArraySubscript> array_subscript @@ -1402,7 +1409,7 @@ func (u *sqlSymUnion) routineBody() *tree.RoutineBody { %type <[]*tree.When> when_clause_list %type sub_type %type numeric_only -%type alias_clause opt_alias_clause +%type alias_clause opt_alias_clause func_alias_clause opt_func_alias_clause %type opt_ordinality opt_compact %type <*tree.Order> sortby %type index_elem index_elem_options create_as_param @@ -2399,24 +2406,24 @@ alter_table_cmd: $$.val = &tree.AlterTableRenameConstraint{Constraint: tree.Name($3), NewName: tree.Name($5) } } // ALTER TABLE ADD -| ADD column_def +| ADD column_table_def { - $$.val = &tree.AlterTableAddColumn{IfNotExists: false, ColumnDef: $2.colDef()} + $$.val = &tree.AlterTableAddColumn{IfNotExists: false, ColumnDef: $2.colTableDef()} } // ALTER TABLE ADD IF NOT EXISTS -| ADD IF NOT EXISTS column_def +| ADD IF NOT EXISTS column_table_def { - $$.val = &tree.AlterTableAddColumn{IfNotExists: true, ColumnDef: $5.colDef()} + $$.val = &tree.AlterTableAddColumn{IfNotExists: true, ColumnDef: $5.colTableDef()} } // ALTER TABLE ADD COLUMN -| ADD COLUMN column_def +| ADD COLUMN column_table_def { - $$.val = &tree.AlterTableAddColumn{IfNotExists: false, ColumnDef: $3.colDef()} + $$.val = &tree.AlterTableAddColumn{IfNotExists: false, ColumnDef: $3.colTableDef()} } // ALTER TABLE ADD COLUMN IF NOT EXISTS -| ADD COLUMN IF NOT EXISTS column_def +| ADD COLUMN IF NOT EXISTS column_table_def { - $$.val = &tree.AlterTableAddColumn{IfNotExists: true, ColumnDef: $6.colDef()} + $$.val = &tree.AlterTableAddColumn{IfNotExists: true, ColumnDef: $6.colTableDef()} } // ALTER TABLE ALTER [COLUMN] {SET DEFAULT |DROP DEFAULT} | ALTER opt_column column_name alter_column_default @@ -7825,9 +7832,9 @@ table_elem_list: } table_elem: - column_def + column_table_def { - $$.val = $1.colDef() + $$.val = $1.colTableDef() } | index_def | family_def @@ -8012,7 +8019,7 @@ range_partition: // Treat SERIAL pseudo-types as separate case so that types.T does not have to // support them as first-class types (e.g. they should not be supported as CAST // target types). -column_def: +column_table_def: column_name typename col_qual_list { typ := $2.typeReference() @@ -10805,20 +10812,20 @@ materialize_clause: } common_table_expr: - table_alias_name opt_column_list AS '(' preparable_stmt ')' + table_alias_name opt_col_def_list_no_types AS '(' preparable_stmt ')' { $$.val = &tree.CTE{ - Name: tree.AliasClause{Alias: tree.Name($1), Cols: $2.nameList() }, + Name: tree.AliasClause{Alias: tree.Name($1), Cols: $2.colDefList() }, Mtr: tree.MaterializeClause{ Set: false, }, Stmt: $5.stmt(), } } -| table_alias_name opt_column_list AS materialize_clause '(' preparable_stmt ')' +| table_alias_name opt_col_def_list_no_types AS materialize_clause '(' preparable_stmt ')' { $$.val = &tree.CTE{ - Name: tree.AliasClause{Alias: tree.Name($1), Cols: $2.nameList() }, + Name: tree.AliasClause{Alias: tree.Name($1), Cols: $2.colDefList() }, Mtr: tree.MaterializeClause{ Materialize: $4.bool(), Set: true, @@ -11323,7 +11330,7 @@ table_ref: { $$.val = &tree.AliasedTableExpr{Expr: &tree.ParenTableExpr{Expr: $2.tblExpr()}, Ordinality: $4.bool(), As: $5.aliasClause()} } -| func_table opt_ordinality opt_alias_clause +| func_table opt_ordinality opt_func_alias_clause { f := $1.tblExpr() $$.val = &tree.AliasedTableExpr{ @@ -11391,16 +11398,61 @@ rowsfrom_list: { $$.val = append($1.exprs(), $3.expr()) } rowsfrom_item: - func_expr_windowless opt_col_def_list + func_expr_windowless opt_func_alias_clause { $$.val = $1.expr() } +opt_col_def_list_no_types: + '(' col_def_list_no_types ')' + { + $$.val = $2.colDefList() + } +| /* EMPTY */ + { + $$.val = tree.ColumnDefList(nil) + } + +col_def_list_no_types: + name + { + $$.val = tree.ColumnDefList{tree.ColumnDef{Name: tree.Name($1)}} + } +| col_def_list_no_types ',' name + { + $$.val = append($1.colDefList(), tree.ColumnDef{Name: tree.Name($3)}) + } + + opt_col_def_list: /* EMPTY */ - { } -| AS '(' error - { return unimplemented(sqllex, "ROWS FROM with col_def_list") } + { + $$.val = tree.ColumnDefList(nil) + } +| '(' col_def_list ')' + { + $$.val = $2.colDefList() + } + +col_def_list: + col_def + { + $$.val = tree.ColumnDefList{$1.colDef()} + } +| col_def_list ',' col_def + { + $$.val = append($1.colDefList(), $3.colDef()) + } + +col_def: + name + { + $$.val = tree.ColumnDef{Name: tree.Name($1)} + } +| name typename + { + $$.val = tree.ColumnDef{Name: tree.Name($1), Type: $2.typeReference()} + } opt_tableref_col_list: /* EMPTY */ { $$.val = nil } @@ -11468,13 +11520,13 @@ joined_table: } alias_clause: - AS table_alias_name opt_column_list + AS table_alias_name opt_col_def_list_no_types { - $$.val = tree.AliasClause{Alias: tree.Name($2), Cols: $3.nameList()} + $$.val = tree.AliasClause{Alias: tree.Name($2), Cols: $3.colDefList()} } -| table_alias_name opt_column_list +| table_alias_name opt_col_def_list_no_types { - $$.val = tree.AliasClause{Alias: tree.Name($1), Cols: $2.nameList()} + $$.val = tree.AliasClause{Alias: tree.Name($1), Cols: $2.colDefList()} } opt_alias_clause: @@ -11484,6 +11536,23 @@ opt_alias_clause: $$.val = tree.AliasClause{} } +func_alias_clause: + AS table_alias_name opt_col_def_list + { + $$.val = tree.AliasClause{Alias: tree.Name($2), Cols: $3.colDefList()} + } +| table_alias_name opt_col_def_list + { + $$.val = tree.AliasClause{Alias: tree.Name($1), Cols: $2.colDefList()} + } + +opt_func_alias_clause: + func_alias_clause +| /* EMPTY */ + { + $$.val = tree.AliasClause{} + } + as_of_clause: AS_LA OF SYSTEM TIME a_expr { diff --git a/pkg/sql/parser/testdata/select_clauses b/pkg/sql/parser/testdata/select_clauses index 471144618058..116c288d1265 100644 --- a/pkg/sql/parser/testdata/select_clauses +++ b/pkg/sql/parser/testdata/select_clauses @@ -3035,3 +3035,11 @@ SELECT * FROM ((t1 NATURAL JOIN t2 WITH ORDINALITY AS o1)) WITH ORDINALITY AS o2 SELECT (*) FROM ((t1 NATURAL JOIN t2 WITH ORDINALITY AS o1)) WITH ORDINALITY AS o2 -- fully parenthesized SELECT * FROM ((t1 NATURAL JOIN t2 WITH ORDINALITY AS o1)) WITH ORDINALITY AS o2 -- literals removed SELECT * FROM ((_ NATURAL JOIN _ WITH ORDINALITY AS _)) WITH ORDINALITY AS _ -- identifiers removed + +parse +SELECT * FROM json_to_record('') AS t(a INT, b TEXT, c foo) +---- +SELECT * FROM ROWS FROM (json_to_record('')) AS t (a INT8, b STRING, c foo) -- normalized! +SELECT (*) FROM ROWS FROM ((json_to_record(('')))) AS t (a INT8, b STRING, c foo) -- fully parenthesized +SELECT * FROM ROWS FROM (json_to_record('_')) AS t (a INT8, b STRING, c foo) -- literals removed +SELECT * FROM ROWS FROM (json_to_record('')) AS _ (_ INT8, _ STRING, _ foo) -- identifiers removed diff --git a/pkg/sql/rowexec/project_set.go b/pkg/sql/rowexec/project_set.go index 852340a0ecd0..ea8e1959345f 100644 --- a/pkg/sql/rowexec/project_set.go +++ b/pkg/sql/rowexec/project_set.go @@ -168,6 +168,11 @@ func (ps *projectSetProcessor) nextInputRow() ( if gen == nil { gen = builtins.EmptyGenerator() } + if aliasSetter, ok := gen.(eval.AliasAwareValueGenerator); ok { + if err := aliasSetter.SetAlias(ps.spec.GeneratedColumns, ps.spec.GeneratedColumnLabels); err != nil { + return nil, nil, err + } + } if err := gen.Start(ps.Ctx, ps.FlowCtx.Txn); err != nil { return nil, nil, err } diff --git a/pkg/sql/sem/builtins/builtins.go b/pkg/sql/sem/builtins/builtins.go index 08b23fcf4b3f..222dc2dbccec 100644 --- a/pkg/sql/sem/builtins/builtins.go +++ b/pkg/sql/sem/builtins/builtins.go @@ -3863,9 +3863,6 @@ value if you rely on the HLC for accuracy.`, // The behavior of both the JSON and JSONB data types in CockroachDB is // similar to the behavior of the JSONB data type in Postgres. - "json_to_recordset": makeBuiltin(tree.FunctionProperties{UnsupportedWithIssue: 33285, Category: builtinconstants.CategoryJSON}), - "jsonb_to_recordset": makeBuiltin(tree.FunctionProperties{UnsupportedWithIssue: 33285, Category: builtinconstants.CategoryJSON}), - "jsonb_path_exists": makeBuiltin(tree.FunctionProperties{UnsupportedWithIssue: 22513, Category: builtinconstants.CategoryJSON}), "jsonb_path_exists_opr": makeBuiltin(tree.FunctionProperties{UnsupportedWithIssue: 22513, Category: builtinconstants.CategoryJSON}), "jsonb_path_match": makeBuiltin(tree.FunctionProperties{UnsupportedWithIssue: 22513, Category: builtinconstants.CategoryJSON}), diff --git a/pkg/sql/sem/builtins/generator_builtins.go b/pkg/sql/sem/builtins/generator_builtins.go index 38bb7e3ca833..14220584837d 100644 --- a/pkg/sql/sem/builtins/generator_builtins.go +++ b/pkg/sql/sem/builtins/generator_builtins.go @@ -72,6 +72,14 @@ func genPropsWithLabels(returnLabels []string) tree.FunctionProperties { } } +func recordGenProps() tree.FunctionProperties { + return tree.FunctionProperties{ + Class: tree.GeneratorClass, + Category: builtinconstants.CategoryGenerator, + ReturnsRecordType: true, + } +} + var aclexplodeGeneratorType = types.MakeLabeledTuple( []*types.T{types.Oid, types.Oid, types.String, types.Bool}, []string{"grantor", "grantee", "privilege_type", "is_grantable"}, @@ -378,6 +386,11 @@ var generators = map[string]builtinDefinition{ "jsonb_populate_recordset": makeBuiltin(jsonPopulateProps, makeJSONPopulateImpl(makeJSONPopulateRecordSetGenerator, "Expands the outermost array of objects in from_json to a set of rows whose columns match the record type defined by base")), + "json_to_record": makeBuiltin(recordGenProps(), jsonToRecordImpl), + "jsonb_to_record": makeBuiltin(recordGenProps(), jsonToRecordImpl), + "json_to_recordset": makeBuiltin(recordGenProps(), jsonToRecordSetImpl), + "jsonb_to_recordset": makeBuiltin(recordGenProps(), jsonToRecordSetImpl), + "crdb_internal.check_consistency": makeBuiltin( tree.FunctionProperties{ Class: tree.GeneratorClass, @@ -1381,6 +1394,26 @@ var jsonEachTextImpl = makeGeneratorOverload( volatility.Immutable, ) +var jsonToRecordImpl = makeGeneratorOverload( + tree.ArgTypes{{"input", types.Jsonb}}, + // NOTE: this type will never actually get used. It is replaced in the + // optimizer by looking at the most recent AS alias clause. + types.EmptyTuple, + makeJSONRecordGenerator, + "Builds an arbitrary record from a JSON object.", + volatility.Stable, +) + +var jsonToRecordSetImpl = makeGeneratorOverload( + tree.ArgTypes{{"input", types.Jsonb}}, + // NOTE: this type will never actually get used. It is replaced in the + // optimizer by looking at the most recent AS alias clause. + types.EmptyTuple, + makeJSONRecordSetGenerator, + "Builds an arbitrary set of records from a JSON array of objects.", + volatility.Stable, +) + var jsonEachGeneratorLabels = []string{"key", "value"} var jsonEachGeneratorType = types.MakeLabeledTuple( @@ -1664,6 +1697,134 @@ func (j *jsonPopulateRecordSetGenerator) Values() (tree.Datums, error) { return output.D, nil } +func makeJSONRecordGenerator(evalCtx *eval.Context, args tree.Datums) (eval.ValueGenerator, error) { + target := tree.MustBeDJSON(args[0]) + return &jsonRecordGenerator{ + evalCtx: evalCtx, + target: target.JSON, + }, nil +} + +type jsonRecordGenerator struct { + evalCtx *eval.Context + target json.JSON + + wasCalled bool + values tree.Datums + types []*types.T + labels []string + // labelToRowIndexMap maps the column label to its position within the row. + labelToRowIndexMap map[string]int +} + +func (j *jsonRecordGenerator) SetAlias(types []*types.T, labels []string) error { + j.types = types + j.labels = labels + j.labelToRowIndexMap = make(map[string]int) + for i := range types { + j.labelToRowIndexMap[j.labels[i]] = i + } + if len(types) != len(labels) { + return errors.AssertionFailedf("unexpected mismatched types/labels list in json record generator %v %v", types, labels) + } + return nil +} + +func (j jsonRecordGenerator) ResolvedType() *types.T { + return types.AnyTuple +} + +func (j *jsonRecordGenerator) Start(ctx context.Context, _ *kv.Txn) error { + j.values = make(tree.Datums, len(j.types)) + if j.target.Type() != json.ObjectJSONType { + return pgerror.Newf(pgcode.InvalidParameterValue, + "invalid non-object argument to json_to_record") + } + return nil +} + +func (j *jsonRecordGenerator) Next(ctx context.Context) (bool, error) { + if j.wasCalled { + return false, nil + } + for i := range j.values { + j.values[i] = tree.DNull + } + iter, err := j.target.ObjectIter() + if err != nil { + return false, err + } + for iter.Next() { + idx, ok := j.labelToRowIndexMap[iter.Key()] + if !ok { + continue + } + v := iter.Value() + datum, err := eval.PopulateDatumWithJSON(j.evalCtx, v, j.types[idx]) + if err != nil { + return false, err + } + j.values[idx] = datum + } + + j.wasCalled = true + return true, nil +} + +func (j jsonRecordGenerator) Values() (tree.Datums, error) { + return j.values, nil +} + +func (j jsonRecordGenerator) Close(ctx context.Context) {} + +type jsonRecordSetGenerator struct { + jsonRecordGenerator + + arr tree.DJSON + nextIndex int +} + +func makeJSONRecordSetGenerator( + evalCtx *eval.Context, args tree.Datums, +) (eval.ValueGenerator, error) { + arr := tree.MustBeDJSON(args[0]) + return &jsonRecordSetGenerator{ + arr: arr, + jsonRecordGenerator: jsonRecordGenerator{ + evalCtx: evalCtx, + }, + }, nil +} + +func (j *jsonRecordSetGenerator) Start(ctx context.Context, _ *kv.Txn) error { + j.values = make(tree.Datums, len(j.types)) + if j.arr.Type() != json.ArrayJSONType { + return pgerror.Newf(pgcode.InvalidParameterValue, + "argument to json_to_recordset must be an array of objects") + } + j.nextIndex = -1 + return nil +} + +func (j *jsonRecordSetGenerator) Next(ctx context.Context) (bool, error) { + j.nextIndex++ + next, err := j.arr.FetchValIdx(j.nextIndex) + if err != nil || next == nil { + return false, err + } + if next.Type() != json.ObjectJSONType { + return false, pgerror.Newf(pgcode.InvalidParameterValue, + "argument to json_to_recordset must be an array of objects") + } + j.target = next + j.wasCalled = false + _, err = j.jsonRecordGenerator.Next(ctx) + if err != nil { + return false, err + } + return true, nil +} + type checkConsistencyGenerator struct { consistencyChecker eval.ConsistencyCheckRunner from, to roachpb.Key diff --git a/pkg/sql/sem/eval/generators.go b/pkg/sql/sem/eval/generators.go index 754b773c3133..5ac37a9472a0 100644 --- a/pkg/sql/sem/eval/generators.go +++ b/pkg/sql/sem/eval/generators.go @@ -83,6 +83,12 @@ type ValueGenerator interface { Close(ctx context.Context) } +// AliasAwareValueGenerator is a value generator that can inspect the alias with +// which it was invoked. SetAlias will always be run before Start. +type AliasAwareValueGenerator interface { + SetAlias(types []*types.T, labels []string) error +} + // streamingValueGenerator is a marker-type indicating that the wrapped // generator is of "streaming" nature, thus, projectSet processor must be // streaming too. diff --git a/pkg/sql/sem/tree/function_definition.go b/pkg/sql/sem/tree/function_definition.go index 420e3dc6a6fb..3f64096d1b8f 100644 --- a/pkg/sql/sem/tree/function_definition.go +++ b/pkg/sql/sem/tree/function_definition.go @@ -98,6 +98,16 @@ type FunctionProperties struct { // // See memo.CanBeCompositeSensitive. CompositeInsensitive bool + + // ReturnsRecordType indicates that this function is a record-returning + // function, which implies that it's unusable without a corresponding type + // alias. + // + // For example, consider the case of json_to_record('{"a":"b", "c":"d"}'). + // This function returns an error unless it as an `AS t(a,b,c)` declaration, + // since its definition is to pick out the JSON attributes within the input + // that match, by name, to the columns in the aliased record type. + ReturnsRecordType bool } // ShouldDocument returns whether the built-in function should be included in diff --git a/pkg/sql/sem/tree/select.go b/pkg/sql/sem/tree/select.go index a6fe48f23592..f0255adac955 100644 --- a/pkg/sql/sem/tree/select.go +++ b/pkg/sql/sem/tree/select.go @@ -187,24 +187,55 @@ func (node *SelectExpr) Format(ctx *FmtCtx) { } } -// AliasClause represents an alias, optionally with a column list: -// "AS name" or "AS name(col1, col2)". +// AliasClause represents an alias, optionally with a column def list: +// "AS name", "AS name(col1, col2)", or "AS name(col1 INT, col2 STRING)". +// Note that the last form is only valid in the context of record-returning +// functions, which also require the last form to define their output types. type AliasClause struct { Alias Name - Cols NameList + Cols ColumnDefList } // Format implements the NodeFormatter interface. -func (a *AliasClause) Format(ctx *FmtCtx) { - ctx.FormatNode(&a.Alias) - if len(a.Cols) != 0 { +func (f *AliasClause) Format(ctx *FmtCtx) { + ctx.FormatNode(&f.Alias) + if len(f.Cols) != 0 { // Format as "alias (col1, col2, ...)". ctx.WriteString(" (") - ctx.FormatNode(&a.Cols) + ctx.FormatNode(&f.Cols) ctx.WriteByte(')') } } +// ColumnDef represents a column definition in the context of a record type +// alias, like in select * from json_to_record(...) AS foo(a INT, b INT). +type ColumnDef struct { + Name Name + Type ResolvableTypeReference +} + +// Format implements the NodeFormatter interface. +func (c *ColumnDef) Format(ctx *FmtCtx) { + ctx.FormatNode(&c.Name) + if c.Type != nil { + ctx.WriteByte(' ') + ctx.WriteString(c.Type.SQLString()) + } +} + +// ColumnDefList represents a list of ColumnDefs. +type ColumnDefList []ColumnDef + +// Format implements the NodeFormatter interface. +func (c *ColumnDefList) Format(ctx *FmtCtx) { + for i := range *c { + if i > 0 { + ctx.WriteString(", ") + } + ctx.FormatNode(&(*c)[i]) + } +} + // AsOfClause represents an as of time. type AsOfClause struct { Expr Expr