From 5995ec11bae973c8812ed195ad1d6301c2569fca Mon Sep 17 00:00:00 2001 From: Jordan Lewis Date: Fri, 10 Sep 2021 19:13:52 -0300 Subject: [PATCH] sql: add json{,b}_to_record{,set}; recordtype srfs This commit adds support for the family of four json{,b}_to_record{,set} functions, which deconstruct JSON input (either an object or an array) and return a row or set of rows with well-typed elements from the JSON, governed by the *table alias* that the function is invoked with. For example, the query below deconstructs the input JSON, returning each of the keys "requested" by the table alias definition. Missing keys are replaced with `NULL`. ``` SELECT * FROM json_to_record('{"a": "b", "c": true}') AS t(a TEXT, z INT) ---- b NULL ``` The logic that governs the type casting from JSON to SQL mimics the logic in the similar `json_populate_record` family of functions, and should be identical to Postgres's such logic. It's permissible to use the virtual table type to deconstruct a sub-object into a record field within the top level JSON, like this: ``` CREATE TABLE mytable (a INT, b TEXT) SELECT * FROM json_to_record('{"foo": {"a": 3, "b": "bar"}}') AS t(foo mytable) ---- (3,bar) ``` Functions like these ones that return record types in PostgreSQL must be aliased to a named and typed tuple (like `AS t(a INT, b INT)`) to be usable. As a result, this commit: 1. adds parser support for this form of alias, with types 2. pushes the alias information for an aliased expression down the optimizer's call stack so that the aliased expression can access the alias information. Previously, the aliased expression was blind to any aliases that were applied to it. 3. teaches the generator function factories to get the alias information plumbed in as well. Release note (sql change): add the json{,b}_to_record{,set} builtin functions, which transform JSON into structured SQL records. --- docs/generated/sql/bnf/BUILD.bazel | 2 +- .../sql/bnf/alter_table_partition_by.bnf | 4 +- docs/generated/sql/bnf/check_column_level.bnf | 2 +- docs/generated/sql/bnf/check_table_level.bnf | 2 +- .../{column_def.bnf => column_table_def.bnf} | 2 +- docs/generated/sql/bnf/create_table_stmt.bnf | 4 +- .../sql/bnf/default_value_column_level.bnf | 2 +- .../sql/bnf/foreign_key_column_level.bnf | 2 +- .../sql/bnf/foreign_key_table_level.bnf | 2 +- .../sql/bnf/not_null_column_level.bnf | 2 +- .../sql/bnf/primary_key_column_level.bnf | 2 +- .../sql/bnf/primary_key_table_level.bnf | 2 +- docs/generated/sql/bnf/stmt_block.bnf | 49 ++++-- docs/generated/sql/bnf/table_ref.bnf | 14 +- .../generated/sql/bnf/unique_column_level.bnf | 2 +- docs/generated/sql/bnf/unique_table_level.bnf | 2 +- docs/generated/sql/bnf/with_clause.bnf | 4 +- docs/generated/sql/functions.md | 8 + pkg/cmd/docgen/diagrams.go | 28 +-- pkg/gen/bnf.bzl | 2 +- pkg/gen/diagrams.bzl | 2 +- pkg/gen/docs.bzl | 2 +- pkg/internal/sqlsmith/relational.go | 16 +- pkg/sql/distsql_physical_planner.go | 8 +- pkg/sql/execinfrapb/processors_sql.proto | 4 + pkg/sql/logictest/logic.go | 3 +- .../testdata/logic_test/json_builtins | 110 ++++++++++++ pkg/sql/opt/optbuilder/builder.go | 4 + pkg/sql/opt/optbuilder/scalar.go | 2 +- pkg/sql/opt/optbuilder/select.go | 12 +- pkg/sql/opt/optbuilder/srfs.go | 36 +++- pkg/sql/opt/optbuilder/with.go | 2 +- pkg/sql/opt/optgen/cmd/optgen/metadata.go | 1 + pkg/sql/parser/parse_test.go | 2 - pkg/sql/parser/sql.y | 125 +++++++++++--- pkg/sql/parser/testdata/select_clauses | 8 + pkg/sql/rowexec/project_set.go | 5 + pkg/sql/sem/builtins/builtins.go | 3 - pkg/sql/sem/builtins/generator_builtins.go | 161 ++++++++++++++++++ pkg/sql/sem/eval/generators.go | 6 + pkg/sql/sem/tree/function_definition.go | 10 ++ pkg/sql/sem/tree/select.go | 45 ++++- 42 files changed, 592 insertions(+), 112 deletions(-) rename docs/generated/sql/bnf/{column_def.bnf => column_table_def.bnf} (70%) diff --git a/docs/generated/sql/bnf/BUILD.bazel b/docs/generated/sql/bnf/BUILD.bazel index 7f6aae1a976d..b6e87cb60a3d 100644 --- a/docs/generated/sql/bnf/BUILD.bazel +++ b/docs/generated/sql/bnf/BUILD.bazel @@ -66,7 +66,7 @@ FILES = [ "check_table_level", "close_cursor_stmt", "col_qualification", - "column_def", + "column_table_def", "comment", "commit_transaction", "copy_from_stmt", diff --git a/docs/generated/sql/bnf/alter_table_partition_by.bnf b/docs/generated/sql/bnf/alter_table_partition_by.bnf index b11ee35dc480..a828c0ce3a27 100644 --- a/docs/generated/sql/bnf/alter_table_partition_by.bnf +++ b/docs/generated/sql/bnf/alter_table_partition_by.bnf @@ -1,3 +1,3 @@ alter_onetable_stmt ::= - 'ALTER' 'TABLE' table_name 'PARTITION' 'ALL' 'BY' partition_by_inner ( ( ',' ( 'RENAME' opt_column column_name 'TO' column_name | 'RENAME' 'CONSTRAINT' column_name 'TO' column_name | 'ADD' column_def | 'ADD' 'IF' 'NOT' 'EXISTS' column_def | 'ADD' 'COLUMN' column_def | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_def | 'ALTER' opt_column column_name alter_column_default | 'ALTER' opt_column column_name alter_column_on_update | 'ALTER' opt_column column_name alter_column_visible | 'ALTER' opt_column column_name 'DROP' 'NOT' 'NULL' | 'ALTER' opt_column column_name 'DROP' 'STORED' | 'ALTER' opt_column column_name 'SET' 'NOT' 'NULL' | 'DROP' opt_column 'IF' 'EXISTS' column_name opt_drop_behavior | 'DROP' opt_column column_name opt_drop_behavior | 'ALTER' opt_column column_name opt_set_data 'TYPE' typename opt_collate opt_alter_column_using | 'ADD' table_constraint opt_validate_behavior | 'ADD' 'CONSTRAINT' 'IF' 'NOT' 'EXISTS' constraint_name constraint_elem opt_validate_behavior | 'ALTER' 'PRIMARY' 'KEY' 'USING' 'COLUMNS' '(' index_params ')' opt_hash_sharded opt_with_storage_parameter_list | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name opt_drop_behavior | 'DROP' 'CONSTRAINT' constraint_name opt_drop_behavior | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | ( 'PARTITION' 'BY' partition_by_inner | 'PARTITION' 'ALL' 'BY' partition_by_inner ) | 'SET' '(' storage_parameter_list ')' | 'RESET' '(' storage_parameter_key_list ')' ) ) )* - | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'PARTITION' 'ALL' 'BY' partition_by_inner ( ( ',' ( 'RENAME' opt_column column_name 'TO' column_name | 'RENAME' 'CONSTRAINT' column_name 'TO' column_name | 'ADD' column_def | 'ADD' 'IF' 'NOT' 'EXISTS' column_def | 'ADD' 'COLUMN' column_def | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_def | 'ALTER' opt_column column_name alter_column_default | 'ALTER' opt_column column_name alter_column_on_update | 'ALTER' opt_column column_name alter_column_visible | 'ALTER' opt_column column_name 'DROP' 'NOT' 'NULL' | 'ALTER' opt_column column_name 'DROP' 'STORED' | 'ALTER' opt_column column_name 'SET' 'NOT' 'NULL' | 'DROP' opt_column 'IF' 'EXISTS' column_name opt_drop_behavior | 'DROP' opt_column column_name opt_drop_behavior | 'ALTER' opt_column column_name opt_set_data 'TYPE' typename opt_collate opt_alter_column_using | 'ADD' table_constraint opt_validate_behavior | 'ADD' 'CONSTRAINT' 'IF' 'NOT' 'EXISTS' constraint_name constraint_elem opt_validate_behavior | 'ALTER' 'PRIMARY' 'KEY' 'USING' 'COLUMNS' '(' index_params ')' opt_hash_sharded opt_with_storage_parameter_list | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name opt_drop_behavior | 'DROP' 'CONSTRAINT' constraint_name opt_drop_behavior | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | ( 'PARTITION' 'BY' partition_by_inner | 'PARTITION' 'ALL' 'BY' partition_by_inner ) | 'SET' '(' storage_parameter_list ')' | 'RESET' '(' storage_parameter_key_list ')' ) ) )* + 'ALTER' 'TABLE' table_name 'PARTITION' 'ALL' 'BY' partition_by_inner ( ( ',' ( 'RENAME' opt_column column_name 'TO' column_name | 'RENAME' 'CONSTRAINT' column_name 'TO' column_name | 'ADD' column_table_def | 'ADD' 'IF' 'NOT' 'EXISTS' column_table_def | 'ADD' 'COLUMN' column_table_def | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_table_def | 'ALTER' opt_column column_name alter_column_default | 'ALTER' opt_column column_name alter_column_on_update | 'ALTER' opt_column column_name alter_column_visible | 'ALTER' opt_column column_name 'DROP' 'NOT' 'NULL' | 'ALTER' opt_column column_name 'DROP' 'STORED' | 'ALTER' opt_column column_name 'SET' 'NOT' 'NULL' | 'DROP' opt_column 'IF' 'EXISTS' column_name opt_drop_behavior | 'DROP' opt_column column_name opt_drop_behavior | 'ALTER' opt_column column_name opt_set_data 'TYPE' typename opt_collate opt_alter_column_using | 'ADD' table_constraint opt_validate_behavior | 'ADD' 'CONSTRAINT' 'IF' 'NOT' 'EXISTS' constraint_name constraint_elem opt_validate_behavior | 'ALTER' 'PRIMARY' 'KEY' 'USING' 'COLUMNS' '(' index_params ')' opt_hash_sharded opt_with_storage_parameter_list | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name opt_drop_behavior | 'DROP' 'CONSTRAINT' constraint_name opt_drop_behavior | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | ( 'PARTITION' 'BY' partition_by_inner | 'PARTITION' 'ALL' 'BY' partition_by_inner ) | 'SET' '(' storage_parameter_list ')' | 'RESET' '(' storage_parameter_key_list ')' ) ) )* + | 'ALTER' 'TABLE' 'IF' 'EXISTS' table_name 'PARTITION' 'ALL' 'BY' partition_by_inner ( ( ',' ( 'RENAME' opt_column column_name 'TO' column_name | 'RENAME' 'CONSTRAINT' column_name 'TO' column_name | 'ADD' column_table_def | 'ADD' 'IF' 'NOT' 'EXISTS' column_table_def | 'ADD' 'COLUMN' column_table_def | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_table_def | 'ALTER' opt_column column_name alter_column_default | 'ALTER' opt_column column_name alter_column_on_update | 'ALTER' opt_column column_name alter_column_visible | 'ALTER' opt_column column_name 'DROP' 'NOT' 'NULL' | 'ALTER' opt_column column_name 'DROP' 'STORED' | 'ALTER' opt_column column_name 'SET' 'NOT' 'NULL' | 'DROP' opt_column 'IF' 'EXISTS' column_name opt_drop_behavior | 'DROP' opt_column column_name opt_drop_behavior | 'ALTER' opt_column column_name opt_set_data 'TYPE' typename opt_collate opt_alter_column_using | 'ADD' table_constraint opt_validate_behavior | 'ADD' 'CONSTRAINT' 'IF' 'NOT' 'EXISTS' constraint_name constraint_elem opt_validate_behavior | 'ALTER' 'PRIMARY' 'KEY' 'USING' 'COLUMNS' '(' index_params ')' opt_hash_sharded opt_with_storage_parameter_list | 'VALIDATE' 'CONSTRAINT' constraint_name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' constraint_name opt_drop_behavior | 'DROP' 'CONSTRAINT' constraint_name opt_drop_behavior | 'EXPERIMENTAL_AUDIT' 'SET' audit_mode | ( 'PARTITION' 'BY' partition_by_inner | 'PARTITION' 'ALL' 'BY' partition_by_inner ) | 'SET' '(' storage_parameter_list ')' | 'RESET' '(' storage_parameter_key_list ')' ) ) )* diff --git a/docs/generated/sql/bnf/check_column_level.bnf b/docs/generated/sql/bnf/check_column_level.bnf index 68e1da18ce2f..9cf320eccee9 100644 --- a/docs/generated/sql/bnf/check_column_level.bnf +++ b/docs/generated/sql/bnf/check_column_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' column_name column_type 'CHECK' '(' check_expr ')' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')' + 'CREATE' 'TABLE' table_name '(' column_name column_type 'CHECK' '(' check_expr ')' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')' diff --git a/docs/generated/sql/bnf/check_table_level.bnf b/docs/generated/sql/bnf/check_table_level.bnf index 799741a33b87..4ba8e0c0c48f 100644 --- a/docs/generated/sql/bnf/check_table_level.bnf +++ b/docs/generated/sql/bnf/check_table_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' constraint_name | ) 'CHECK' '(' check_expr ')' ( table_constraints | ) ')' + 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' constraint_name | ) 'CHECK' '(' check_expr ')' ( table_constraints | ) ')' diff --git a/docs/generated/sql/bnf/column_def.bnf b/docs/generated/sql/bnf/column_table_def.bnf similarity index 70% rename from docs/generated/sql/bnf/column_def.bnf rename to docs/generated/sql/bnf/column_table_def.bnf index 8da88cb31903..24c5babebeee 100644 --- a/docs/generated/sql/bnf/column_def.bnf +++ b/docs/generated/sql/bnf/column_table_def.bnf @@ -1,2 +1,2 @@ -column_def ::= +column_table_def ::= column_name typename ( ( col_qualification ) )* diff --git a/docs/generated/sql/bnf/create_table_stmt.bnf b/docs/generated/sql/bnf/create_table_stmt.bnf index 46a04cc2cd88..ee40d76b05d7 100644 --- a/docs/generated/sql/bnf/create_table_stmt.bnf +++ b/docs/generated/sql/bnf/create_table_stmt.bnf @@ -1,3 +1,3 @@ create_table_stmt ::= - 'CREATE' opt_persistence_temp_table 'TABLE' table_name '(' ( ( ( ( column_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) ( ( ',' ( column_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) )* ) | ) ')' opt_partition_by_table ( opt_with_storage_parameter_list ) ( 'ON' 'COMMIT' 'PRESERVE' 'ROWS' ) opt_locality - | 'CREATE' opt_persistence_temp_table 'TABLE' 'IF' 'NOT' 'EXISTS' table_name '(' ( ( ( ( column_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) ( ( ',' ( column_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) )* ) | ) ')' opt_partition_by_table ( opt_with_storage_parameter_list ) ( 'ON' 'COMMIT' 'PRESERVE' 'ROWS' ) opt_locality + 'CREATE' opt_persistence_temp_table 'TABLE' table_name '(' ( ( ( ( column_table_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) ( ( ',' ( column_table_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) )* ) | ) ')' opt_partition_by_table ( opt_with_storage_parameter_list ) ( 'ON' 'COMMIT' 'PRESERVE' 'ROWS' ) opt_locality + | 'CREATE' opt_persistence_temp_table 'TABLE' 'IF' 'NOT' 'EXISTS' table_name '(' ( ( ( ( column_table_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) ( ( ',' ( column_table_def | index_def | family_def | table_constraint opt_validate_behavior | 'LIKE' table_name like_table_option_list ) ) )* ) | ) ')' opt_partition_by_table ( opt_with_storage_parameter_list ) ( 'ON' 'COMMIT' 'PRESERVE' 'ROWS' ) opt_locality diff --git a/docs/generated/sql/bnf/default_value_column_level.bnf b/docs/generated/sql/bnf/default_value_column_level.bnf index 84eee1641203..61b218b3a308 100644 --- a/docs/generated/sql/bnf/default_value_column_level.bnf +++ b/docs/generated/sql/bnf/default_value_column_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' column_name column_type 'DEFAULT' default_value ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')' + 'CREATE' 'TABLE' table_name '(' column_name column_type 'DEFAULT' default_value ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')' diff --git a/docs/generated/sql/bnf/foreign_key_column_level.bnf b/docs/generated/sql/bnf/foreign_key_column_level.bnf index a13da4f43917..d0aa363d3b5d 100644 --- a/docs/generated/sql/bnf/foreign_key_column_level.bnf +++ b/docs/generated/sql/bnf/foreign_key_column_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' column_name column_type 'REFERENCES' parent_table ( '(' ref_column_name ')' | ) ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')' + 'CREATE' 'TABLE' table_name '(' column_name column_type 'REFERENCES' parent_table ( '(' ref_column_name ')' | ) ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')' diff --git a/docs/generated/sql/bnf/foreign_key_table_level.bnf b/docs/generated/sql/bnf/foreign_key_table_level.bnf index 7d11c4e45bb5..fd714bf17eb2 100644 --- a/docs/generated/sql/bnf/foreign_key_table_level.bnf +++ b/docs/generated/sql/bnf/foreign_key_table_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' constraint_name | ) 'FOREIGN KEY' '(' ( fk_column_name ( ',' fk_column_name )* ) ')' 'REFERENCES' parent_table ( '(' ( ref_column_name ( ',' ref_column_name )* ) ')' | ) ( table_constraints | ) ')' + 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' constraint_name | ) 'FOREIGN KEY' '(' ( fk_column_name ( ',' fk_column_name )* ) ')' 'REFERENCES' parent_table ( '(' ( ref_column_name ( ',' ref_column_name )* ) ')' | ) ( table_constraints | ) ')' diff --git a/docs/generated/sql/bnf/not_null_column_level.bnf b/docs/generated/sql/bnf/not_null_column_level.bnf index e8e6da436a56..8e32276759c3 100644 --- a/docs/generated/sql/bnf/not_null_column_level.bnf +++ b/docs/generated/sql/bnf/not_null_column_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' column_name column_type 'NOT NULL' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')' + 'CREATE' 'TABLE' table_name '(' column_name column_type 'NOT NULL' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')' diff --git a/docs/generated/sql/bnf/primary_key_column_level.bnf b/docs/generated/sql/bnf/primary_key_column_level.bnf index 1a9d75fb2900..2a59ba4ed18e 100644 --- a/docs/generated/sql/bnf/primary_key_column_level.bnf +++ b/docs/generated/sql/bnf/primary_key_column_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' column_name column_type 'PRIMARY KEY' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')' + 'CREATE' 'TABLE' table_name '(' column_name column_type 'PRIMARY KEY' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')' diff --git a/docs/generated/sql/bnf/primary_key_table_level.bnf b/docs/generated/sql/bnf/primary_key_table_level.bnf index 55c0cd91874f..4b704f8d7bfa 100644 --- a/docs/generated/sql/bnf/primary_key_table_level.bnf +++ b/docs/generated/sql/bnf/primary_key_table_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' name | ) 'PRIMARY KEY' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')' + 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' name | ) 'PRIMARY KEY' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')' diff --git a/docs/generated/sql/bnf/stmt_block.bnf b/docs/generated/sql/bnf/stmt_block.bnf index c669f6bc5661..dae6b65ce201 100644 --- a/docs/generated/sql/bnf/stmt_block.bnf +++ b/docs/generated/sql/bnf/stmt_block.bnf @@ -2487,7 +2487,7 @@ table_ref ::= | 'LATERAL' select_with_parens opt_ordinality opt_alias_clause | joined_table | '(' joined_table ')' opt_ordinality alias_clause - | func_table opt_ordinality opt_alias_clause + | func_table opt_ordinality opt_func_alias_clause | 'LATERAL' func_table opt_ordinality opt_alias_clause | '[' row_source_extension_stmt ']' opt_ordinality opt_alias_clause @@ -2901,8 +2901,8 @@ bare_col_label ::= | bare_label_keywords common_table_expr ::= - table_alias_name opt_column_list 'AS' '(' preparable_stmt ')' - | table_alias_name opt_column_list 'AS' materialize_clause '(' preparable_stmt ')' + table_alias_name opt_col_def_list_no_types 'AS' '(' preparable_stmt ')' + | table_alias_name opt_col_def_list_no_types 'AS' materialize_clause '(' preparable_stmt ')' index_flags_param_list ::= ( index_flags_param ) ( ( ',' index_flags_param ) )* @@ -2980,13 +2980,17 @@ joined_table ::= | table_ref 'NATURAL' 'JOIN' table_ref alias_clause ::= - 'AS' table_alias_name opt_column_list - | table_alias_name opt_column_list + 'AS' table_alias_name opt_col_def_list_no_types + | table_alias_name opt_col_def_list_no_types func_table ::= func_expr_windowless | 'ROWS' 'FROM' '(' rowsfrom_list ')' +opt_func_alias_clause ::= + func_alias_clause + | + row_source_extension_stmt ::= delete_stmt | explain_stmt @@ -3043,10 +3047,10 @@ user_priority ::= alter_table_cmd ::= 'RENAME' opt_column column_name 'TO' column_name | 'RENAME' 'CONSTRAINT' column_name 'TO' column_name - | 'ADD' column_def - | 'ADD' 'IF' 'NOT' 'EXISTS' column_def - | 'ADD' 'COLUMN' column_def - | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_def + | 'ADD' column_table_def + | 'ADD' 'IF' 'NOT' 'EXISTS' column_table_def + | 'ADD' 'COLUMN' column_table_def + | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_table_def | 'ALTER' opt_column column_name alter_column_default | 'ALTER' opt_column column_name alter_column_on_update | 'ALTER' opt_column column_name alter_column_visible @@ -3243,7 +3247,7 @@ storage_parameter ::= storage_parameter_key '=' var_value table_elem ::= - column_def + column_table_def | index_def | family_def | table_constraint opt_validate_behavior @@ -3295,6 +3299,10 @@ bare_label_keywords ::= | 'VOLATILE' | 'SETOF' +opt_col_def_list_no_types ::= + '(' col_def_list_no_types ')' + | + materialize_clause ::= 'MATERIALIZED' | 'NOT' 'MATERIALIZED' @@ -3359,6 +3367,10 @@ join_qual ::= rowsfrom_list ::= ( rowsfrom_item ) ( ( ',' rowsfrom_item ) )* +func_alias_clause ::= + 'AS' table_alias_name opt_col_def_list + | table_alias_name opt_col_def_list + opt_varying ::= 'VARYING' | @@ -3373,7 +3385,7 @@ opt_column ::= 'COLUMN' | -column_def ::= +column_table_def ::= column_name typename col_qual_list alter_column_default ::= @@ -3579,6 +3591,9 @@ common_func_opt_item ::= | 'LEAKPROOF' | 'NOT' 'LEAKPROOF' +col_def_list_no_types ::= + ( name ) ( ( ',' name ) )* + group_by_item ::= a_expr @@ -3590,7 +3605,10 @@ join_outer ::= | rowsfrom_item ::= - func_expr_windowless + func_expr_windowless opt_func_alias_clause + +opt_col_def_list ::= + '(' col_def_list ')' char_aliases ::= 'CHAR' @@ -3666,6 +3684,9 @@ func_arg_class ::= param_name ::= type_function_name +col_def_list ::= + ( col_def ) ( ( ',' col_def ) )* + col_qualification ::= 'CONSTRAINT' constraint_name col_qualification_elem | col_qualification_elem @@ -3695,6 +3716,10 @@ opt_partition_by ::= create_as_param ::= column_name +col_def ::= + name + | name typename + col_qualification_elem ::= 'NOT' 'NULL' | 'NULL' diff --git a/docs/generated/sql/bnf/table_ref.bnf b/docs/generated/sql/bnf/table_ref.bnf index dd24ece24f8e..711d02ef6ca6 100644 --- a/docs/generated/sql/bnf/table_ref.bnf +++ b/docs/generated/sql/bnf/table_ref.bnf @@ -1,9 +1,9 @@ table_ref ::= - table_name ( '@' index_name | ) ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) - | '(' select_stmt ')' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) - | 'LATERAL' '(' select_stmt ')' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) + table_name ( '@' index_name | ) ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name opt_col_def_list_no_types | table_alias_name opt_col_def_list_no_types ) | ) + | '(' select_stmt ')' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name opt_col_def_list_no_types | table_alias_name opt_col_def_list_no_types ) | ) + | 'LATERAL' '(' select_stmt ')' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name opt_col_def_list_no_types | table_alias_name opt_col_def_list_no_types ) | ) | joined_table - | '(' joined_table ')' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) - | func_application ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) - | 'LATERAL' func_application ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) - | '[' row_source_extension_stmt ']' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) ) | ) + | '(' joined_table ')' ( 'WITH' 'ORDINALITY' | ) ( 'AS' table_alias_name opt_col_def_list_no_types | table_alias_name opt_col_def_list_no_types ) + | func_application ( 'WITH' 'ORDINALITY' | ) opt_func_alias_clause + | 'LATERAL' func_application ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name opt_col_def_list_no_types | table_alias_name opt_col_def_list_no_types ) | ) + | '[' row_source_extension_stmt ']' ( 'WITH' 'ORDINALITY' | ) ( ( 'AS' table_alias_name opt_col_def_list_no_types | table_alias_name opt_col_def_list_no_types ) | ) diff --git a/docs/generated/sql/bnf/unique_column_level.bnf b/docs/generated/sql/bnf/unique_column_level.bnf index a1621a96ba97..784a79aa2ab8 100644 --- a/docs/generated/sql/bnf/unique_column_level.bnf +++ b/docs/generated/sql/bnf/unique_column_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' column_name column_type 'UNIQUE' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')' + 'CREATE' 'TABLE' table_name '(' column_name column_type 'UNIQUE' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')' diff --git a/docs/generated/sql/bnf/unique_table_level.bnf b/docs/generated/sql/bnf/unique_table_level.bnf index dcd0abc3e5e0..e57b084f555e 100644 --- a/docs/generated/sql/bnf/unique_table_level.bnf +++ b/docs/generated/sql/bnf/unique_table_level.bnf @@ -1,2 +1,2 @@ stmt_block ::= - 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' name | ) 'UNIQUE' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')' + 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' name | ) 'UNIQUE' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')' diff --git a/docs/generated/sql/bnf/with_clause.bnf b/docs/generated/sql/bnf/with_clause.bnf index 05b08523f7e4..a512476d702d 100644 --- a/docs/generated/sql/bnf/with_clause.bnf +++ b/docs/generated/sql/bnf/with_clause.bnf @@ -1,3 +1,3 @@ with_clause ::= - 'WITH' ( ( ( table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' '(' preparable_stmt ')' | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) ( ( ',' ( table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' '(' preparable_stmt ')' | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) )* ) ( insert_stmt | update_stmt | delete_stmt | upsert_stmt | select_stmt ) - | 'WITH' 'RECURSIVE' ( ( ( table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' '(' preparable_stmt ')' | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) ( ( ',' ( table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' '(' preparable_stmt ')' | table_alias_name ( '(' ( ( name ) ( ( ',' name ) )* ) ')' | ) 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) )* ) ( insert_stmt | update_stmt | delete_stmt | upsert_stmt | select_stmt ) + 'WITH' ( ( ( table_alias_name opt_col_def_list_no_types 'AS' '(' preparable_stmt ')' | table_alias_name opt_col_def_list_no_types 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) ( ( ',' ( table_alias_name opt_col_def_list_no_types 'AS' '(' preparable_stmt ')' | table_alias_name opt_col_def_list_no_types 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) )* ) ( insert_stmt | update_stmt | delete_stmt | upsert_stmt | select_stmt ) + | 'WITH' 'RECURSIVE' ( ( ( table_alias_name opt_col_def_list_no_types 'AS' '(' preparable_stmt ')' | table_alias_name opt_col_def_list_no_types 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) ( ( ',' ( table_alias_name opt_col_def_list_no_types 'AS' '(' preparable_stmt ')' | table_alias_name opt_col_def_list_no_types 'AS' ( 'MATERIALIZED' | 'NOT' 'MATERIALIZED' ) '(' preparable_stmt ')' ) ) )* ) ( insert_stmt | update_stmt | delete_stmt | upsert_stmt | select_stmt ) diff --git a/docs/generated/sql/functions.md b/docs/generated/sql/functions.md index afba41bdded1..2dc01aefc883 100644 --- a/docs/generated/sql/functions.md +++ b/docs/generated/sql/functions.md @@ -1260,6 +1260,10 @@ the locality flag on node startup. Returns an error if no region is set.

Stable json_populate_recordset(base: anyelement, from_json: jsonb) → anyelement

Expands the outermost array of objects in from_json to a set of rows whose columns match the record type defined by base

Stable +json_to_record(input: jsonb) → tuple

Builds an arbitrary record from a JSON object.

+Stable +json_to_recordset(input: jsonb) → tuple

Builds an arbitrary set of records from a JSON array of objects.

+Stable jsonb_array_elements(input: jsonb) → jsonb

Expands a JSON array to a set of JSON values.

Immutable jsonb_array_elements_text(input: jsonb) → string

Expands a JSON array to a set of text values.

@@ -1274,6 +1278,10 @@ the locality flag on node startup. Returns an error if no region is set.

Stable jsonb_populate_recordset(base: anyelement, from_json: jsonb) → anyelement

Expands the outermost array of objects in from_json to a set of rows whose columns match the record type defined by base

Stable +jsonb_to_record(input: jsonb) → tuple

Builds an arbitrary record from a JSON object.

+Stable +jsonb_to_recordset(input: jsonb) → tuple

Builds an arbitrary set of records from a JSON array of objects.

+Stable pg_get_keywords() → tuple{string AS word, string AS catcode, string AS catdesc}

Produces a virtual table containing the keywords known to the SQL parser.

Immutable pg_options_to_table(options: string[]) → tuple{string AS option_name, string AS option_value}

Converts the options array format to a table.

diff --git a/pkg/cmd/docgen/diagrams.go b/pkg/cmd/docgen/diagrams.go index 0422d3e02fd5..338d27f8a68e 100644 --- a/pkg/cmd/docgen/diagrams.go +++ b/pkg/cmd/docgen/diagrams.go @@ -343,7 +343,7 @@ var specs = []stmtSpec{ { name: "add_column", stmt: "alter_onetable_stmt", - inline: []string{"alter_table_cmds", "alter_table_cmd", "column_def", "col_qual_list"}, + inline: []string{"alter_table_cmds", "alter_table_cmd", "column_table_def", "col_qual_list"}, regreplace: map[string]string{ ` \( \( col_qualification \) \)\* .*`: `( ( col_qualification ) )*`, }, @@ -500,7 +500,7 @@ var specs = []stmtSpec{ { name: "alter_table", stmt: "alter_onetable_stmt", - inline: []string{"alter_table_cmds", "alter_table_cmd", "column_def", "opt_drop_behavior", "alter_column_default", "opt_column", "opt_set_data", "table_constraint", "opt_collate", "opt_alter_column_using"}, + inline: []string{"alter_table_cmds", "alter_table_cmd", "column_table_def", "opt_drop_behavior", "alter_column_default", "opt_column", "opt_set_data", "table_constraint", "opt_collate", "opt_alter_column_using"}, replace: map[string]string{ "'VALIDATE' 'CONSTRAINT' name": "", "opt_validate_behavior": "", @@ -594,18 +594,18 @@ var specs = []stmtSpec{ { name: "check_column_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'CHECK' '(' check_expr ')' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'CHECK' '(' check_expr ')' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')'"}, unlink: []string{"table_name", "column_name", "column_type", "check_expr", "column_constraints", "table_constraints"}, }, { name: "check_table_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' constraint_name | ) 'CHECK' '(' check_expr ')' ( table_constraints | ) ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' constraint_name | ) 'CHECK' '(' check_expr ')' ( table_constraints | ) ')'"}, unlink: []string{"table_name", "check_expr", "table_constraints"}, }, { - name: "column_def", - stmt: "column_def", + name: "column_table_def", + stmt: "column_table_def", inline: []string{"col_qual_list"}, }, { @@ -772,7 +772,7 @@ var specs = []stmtSpec{ name: "default_value_column_level", stmt: "stmt_block", replace: map[string]string{ - " stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'DEFAULT' default_value ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'", + " stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'DEFAULT' default_value ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')'", }, unlink: []string{"table_name", "column_name", "column_type", "default_value", "table_constraints"}, }, @@ -948,13 +948,13 @@ var specs = []stmtSpec{ { name: "foreign_key_column_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'REFERENCES' parent_table ( '(' ref_column_name ')' | ) ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'REFERENCES' parent_table ( '(' ref_column_name ')' | ) ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')'"}, unlink: []string{"table_name", "column_name", "column_type", "parent_table", "table_constraints"}, }, { name: "foreign_key_table_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' constraint_name | ) 'FOREIGN KEY' '(' ( fk_column_name ( ',' fk_column_name )* ) ')' 'REFERENCES' parent_table ( '(' ( ref_column_name ( ',' ref_column_name )* ) ')' | ) ( table_constraints | ) ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' constraint_name | ) 'FOREIGN KEY' '(' ( fk_column_name ( ',' fk_column_name )* ) ')' 'REFERENCES' parent_table ( '(' ( ref_column_name ( ',' ref_column_name )* ) ')' | ) ( table_constraints | ) ')'"}, unlink: []string{"table_name", "column_name", "parent_table", "table_constraints"}, }, { @@ -1034,7 +1034,7 @@ var specs = []stmtSpec{ { name: "not_null_column_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'NOT NULL' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'NOT NULL' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')'"}, unlink: []string{"table_name", "column_name", "column_type", "table_constraints"}, }, { @@ -1056,13 +1056,13 @@ var specs = []stmtSpec{ { name: "primary_key_column_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'PRIMARY KEY' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'PRIMARY KEY' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')'"}, unlink: []string{"table_name", "column_name", "column_type", "table_constraints"}, }, { name: "primary_key_table_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' name | ) 'PRIMARY KEY' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' name | ) 'PRIMARY KEY' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')'"}, unlink: []string{"table_name", "column_name", "table_constraints"}, }, { @@ -1467,13 +1467,13 @@ var specs = []stmtSpec{ { name: "unique_column_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'UNIQUE' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' column_name column_type 'UNIQUE' ( column_constraints | ) ( ',' ( column_table_def ( ',' column_table_def )* ) | ) ( table_constraints | ) ')' ')'"}, unlink: []string{"table_name", "column_name", "column_type", "table_constraints"}, }, { name: "unique_table_level", stmt: "stmt_block", - replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' name | ) 'UNIQUE' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')'"}, + replace: map[string]string{" stmt": " 'CREATE' 'TABLE' table_name '(' ( column_table_def ( ',' column_table_def )* ) ( 'CONSTRAINT' name | ) 'UNIQUE' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')'"}, unlink: []string{"table_name", "check_expr", "table_constraints"}, }, { diff --git a/pkg/gen/bnf.bzl b/pkg/gen/bnf.bzl index 35b78827c9b2..2f96df77460f 100644 --- a/pkg/gen/bnf.bzl +++ b/pkg/gen/bnf.bzl @@ -66,7 +66,7 @@ BNF_SRCS = [ "//docs/generated/sql/bnf:check_table_level.bnf", "//docs/generated/sql/bnf:close_cursor_stmt.bnf", "//docs/generated/sql/bnf:col_qualification.bnf", - "//docs/generated/sql/bnf:column_def.bnf", + "//docs/generated/sql/bnf:column_table_def.bnf", "//docs/generated/sql/bnf:comment.bnf", "//docs/generated/sql/bnf:commit_transaction.bnf", "//docs/generated/sql/bnf:copy_from_stmt.bnf", diff --git a/pkg/gen/diagrams.bzl b/pkg/gen/diagrams.bzl index 5a73ea2e81a3..748796c6c3ed 100644 --- a/pkg/gen/diagrams.bzl +++ b/pkg/gen/diagrams.bzl @@ -65,7 +65,7 @@ DIAGRAMS_SRCS = [ "//docs/generated/sql/bnf:check_table_level.html", "//docs/generated/sql/bnf:close_cursor.html", "//docs/generated/sql/bnf:col_qualification.html", - "//docs/generated/sql/bnf:column_def.html", + "//docs/generated/sql/bnf:column_table_def.html", "//docs/generated/sql/bnf:comment.html", "//docs/generated/sql/bnf:commit_transaction.html", "//docs/generated/sql/bnf:copy_from.html", diff --git a/pkg/gen/docs.bzl b/pkg/gen/docs.bzl index df9df7c3a931..448a43521deb 100644 --- a/pkg/gen/docs.bzl +++ b/pkg/gen/docs.bzl @@ -78,7 +78,7 @@ DOCS_SRCS = [ "//docs/generated/sql/bnf:check_table_level.bnf", "//docs/generated/sql/bnf:close_cursor_stmt.bnf", "//docs/generated/sql/bnf:col_qualification.bnf", - "//docs/generated/sql/bnf:column_def.bnf", + "//docs/generated/sql/bnf:column_table_def.bnf", "//docs/generated/sql/bnf:comment.bnf", "//docs/generated/sql/bnf:commit_transaction.bnf", "//docs/generated/sql/bnf:copy_from_stmt.bnf", diff --git a/pkg/internal/sqlsmith/relational.go b/pkg/internal/sqlsmith/relational.go index 15820266ea89..34c6fcfdc802 100644 --- a/pkg/internal/sqlsmith/relational.go +++ b/pkg/internal/sqlsmith/relational.go @@ -478,11 +478,11 @@ func (s *Smither) makeWith() (*tree.With, tableRefs) { } alias := s.name("with") tblName := tree.NewUnqualifiedTableName(alias) - cols := make(tree.NameList, len(stmtRefs)) + cols := make(tree.ColumnDefList, len(stmtRefs)) defs := make([]*tree.ColumnTableDef, len(stmtRefs)) for i, r := range stmtRefs { var err error - cols[i] = r.item.ColumnName + cols[i].Name = r.item.ColumnName defs[i], err = tree.NewColumnTableDef(r.item.ColumnName, r.typ, false /* isSerial */, nil) if err != nil { panic(err) @@ -561,15 +561,15 @@ func makeSelectTable(s *Smither, refs colRefs, forJoin bool) (tree.TableExpr, co } table := s.name("tab") - names := make(tree.NameList, len(stmtRefs)) + names := make(tree.ColumnDefList, len(stmtRefs)) clauseRefs := make(colRefs, len(stmtRefs)) for i, ref := range stmtRefs { - names[i] = s.name("col") + names[i].Name = s.name("col") clauseRefs[i] = &colRef{ typ: ref.typ, item: tree.NewColumnItem( tree.NewUnqualifiedTableName(table), - names[i], + names[i].Name, ), } } @@ -1163,15 +1163,15 @@ func makeValues(s *Smither, desiredTypes []*types.T, refs colRefs) (tree.TableEx values.Rows[i] = tuple } table := s.name("tab") - names := make(tree.NameList, len(desiredTypes)) + names := make(tree.ColumnDefList, len(desiredTypes)) valuesRefs := make(colRefs, len(desiredTypes)) for i, typ := range desiredTypes { - names[i] = s.name("col") + names[i].Name = s.name("col") valuesRefs[i] = &colRef{ typ: typ, item: tree.NewColumnItem( tree.NewUnqualifiedTableName(table), - names[i], + names[i].Name, ), } } diff --git a/pkg/sql/distsql_physical_planner.go b/pkg/sql/distsql_physical_planner.go index ea67bd53059e..d6540892f9d6 100644 --- a/pkg/sql/distsql_physical_planner.go +++ b/pkg/sql/distsql_physical_planner.go @@ -3397,9 +3397,10 @@ func createProjectSetSpec( planCtx *PlanningCtx, n *projectSetPlanningInfo, indexVarMap []int, ) (*execinfrapb.ProjectSetSpec, error) { spec := execinfrapb.ProjectSetSpec{ - Exprs: make([]execinfrapb.Expression, len(n.exprs)), - GeneratedColumns: make([]*types.T, len(n.columns)-n.numColsInSource), - NumColsPerGen: make([]uint32, len(n.exprs)), + Exprs: make([]execinfrapb.Expression, len(n.exprs)), + GeneratedColumns: make([]*types.T, len(n.columns)-n.numColsInSource), + GeneratedColumnLabels: make([]string, len(n.columns)-n.numColsInSource), + NumColsPerGen: make([]uint32, len(n.exprs)), } for i, expr := range n.exprs { var err error @@ -3410,6 +3411,7 @@ func createProjectSetSpec( } for i, col := range n.columns[n.numColsInSource:] { spec.GeneratedColumns[i] = col.Typ + spec.GeneratedColumnLabels[i] = col.Name } for i, n := range n.numColsPerGen { spec.NumColsPerGen[i] = uint32(n) diff --git a/pkg/sql/execinfrapb/processors_sql.proto b/pkg/sql/execinfrapb/processors_sql.proto index 684ce304fb83..d9fa0913ab70 100644 --- a/pkg/sql/execinfrapb/processors_sql.proto +++ b/pkg/sql/execinfrapb/processors_sql.proto @@ -925,6 +925,10 @@ message ProjectSetSpec { // The number of columns each expression returns. Same length as exprs. repeated uint32 num_cols_per_gen = 3; + + // Column labels for the generated values. Needed for some builtin functions + // (like record_to_json) that require the column labels to do their jobs. + repeated string generated_column_labels = 4; } // WindowerSpec is the specification of a processor that performs computations diff --git a/pkg/sql/logictest/logic.go b/pkg/sql/logictest/logic.go index 99d872278024..76b3037b217a 100644 --- a/pkg/sql/logictest/logic.go +++ b/pkg/sql/logictest/logic.go @@ -3181,7 +3181,8 @@ func (t *logicTest) processSubtest( // Don't error if --rewrite is specified, since the expected // results are ignored in that case. if !*rewriteResultsInTestfiles && len(results) != len(query.colTypes) { - return errors.Errorf("expected results are invalid: unexpected column count") + return errors.Errorf("expected results are invalid: unexpected column count %d != %d (%s)", + len(results), len(query.colTypes), results) } query.expectedResults = append(query.expectedResults, results...) } diff --git a/pkg/sql/logictest/testdata/logic_test/json_builtins b/pkg/sql/logictest/testdata/logic_test/json_builtins index 1125c9fb8bfc..c5538c131af3 100644 --- a/pkg/sql/logictest/testdata/logic_test/json_builtins +++ b/pkg/sql/logictest/testdata/logic_test/json_builtins @@ -1361,3 +1361,113 @@ SELECT * FROM json_populate_recordset(NULL::testtab, '[{"i": 3, "ia": [1,2,3], " ---- 3 {1,2,3} foo {a,b} 2017-01-01 00:00:00 +0000 +0000 {"a": "b", "c": 3, "d": [1, false, true, null, {"1": "2"}]} NULL NULL NULL NULL NULL NULL + +query error invalid non-object argument to json_to_record +SELECT * FROM json_to_record('3') AS t(a INT) + +query error invalid non-object argument to json_to_record +SELECT * FROM json_to_record('"a"') AS t(a TEXT) + +query error invalid non-object argument to json_to_record +SELECT * FROM json_to_record('null') AS t(a INT) + +query error invalid non-object argument to json_to_record +SELECT * FROM json_to_record('true') AS t(a INT) + +query error invalid non-object argument to json_to_record +SELECT * FROM json_to_record('[1,2]') AS t(a INT) + +query error column definition list is required for functions returning \"record\" +SELECT * FROM json_to_record('{"a": "b"}') AS t(a) + +query error column definition list is required for functions returning \"record\" +SELECT * FROM json_to_record('{"a": "b"}') + +# Test that non-record generators don't permit col definition lists (with types). +query error a column definition list is only allowed for functions returning \"record\" +SELECT * FROM generate_series(1,10) g(g int) + +statement ok +CREATE TABLE j (j) AS SELECT '{ + "str": "a", + "int": 1, + "bool": true, + "nul": null, + "dec": 2.45, + "arrint": [1,2], + "arrmixed": [1,2,true], + "arrstr": ["a", "b"], + "arrbool": [true, false], + "obj": {"i": 3, "t": "blah", "z": true} + }'::JSONB + +statement ok +INSERT INTO j VALUES('{"str": "zzz"}') + +query TIBTFTTTTT +SELECT t.* FROM j, json_to_record(j.j) AS t( + str TEXT, + int INT, + bool BOOL, + nul TEXT, + dec DECIMAL, + arrint INT[], + arrmixed TEXT, + arrstr TEXT[], + arrbool BOOL[], + obj TEXT +) ORDER BY rowid +---- +a 1 true NULL 2.45 {1,2} [1, 2, true] {a,b} {t,f} {"i": 3, "t": "blah", "z": true} +zzz NULL NULL NULL NULL NULL NULL NULL NULL NULL + +# Test that mismatched types return an error +query error could not parse \"true\" as type int +SELECT t.bool FROM j, json_to_record(j.j) AS t(bool INT) + +# But types can be coerced. +query TT rowsort +SELECT t.* FROM j, json_to_record(j.j) AS t(int TEXT, bool TEXT) +---- +1 true +NULL NULL + +# Mixed type arrays +query error could not parse \"2\" as type bool +SELECT t.arrmixed FROM j, json_to_record(j.j) AS t(arrmixed BOOL[]) + +# Record with custom type +query T rowsort +SELECT t.obj FROM j, json_to_record(j.j) AS t(obj testtab) +---- +(3,,blah,,,) +NULL + +# Test json_to_recordset +query TIBTFTTTTT +SELECT t.* FROM j, json_to_recordset(j.j || '[]' || j.j) AS t( + str TEXT, + int INT, + bool BOOL, + nul TEXT, + dec DECIMAL, + arrint INT[], + arrmixed TEXT, + arrstr TEXT[], + arrbool BOOL[], + obj TEXT +) ORDER BY rowid +---- +a 1 true NULL 2.45 {1,2} [1, 2, true] {a,b} {t,f} {"i": 3, "t": "blah", "z": true} +a 1 true NULL 2.45 {1,2} [1, 2, true] {a,b} {t,f} {"i": 3, "t": "blah", "z": true} +zzz NULL NULL NULL NULL NULL NULL NULL NULL NULL +zzz NULL NULL NULL NULL NULL NULL NULL NULL NULL + +query TT rowsort +SELECT * FROM jsonb_to_recordset('[{"foo": "bar"}, {"foo": "bar2"}]') AS t(foo TEXT), + jsonb_to_recordset('[{"foo": "blah"}, {"foo": "blah2"}]') AS u(foo TEXT) +---- +bar blah +bar blah2 +bar2 blah +bar2 blah2 diff --git a/pkg/sql/opt/optbuilder/builder.go b/pkg/sql/opt/optbuilder/builder.go index 069485ab7798..f6567ae5c93a 100644 --- a/pkg/sql/opt/optbuilder/builder.go +++ b/pkg/sql/opt/optbuilder/builder.go @@ -145,6 +145,10 @@ type Builder struct { // (without ON CONFLICT) or false otherwise. All mutated tables will have an // entry in the map. areAllTableMutationsSimpleInserts map[cat.StableID]bool + + // lastAlias is set to the last data source alias we've come across, if we + // are processing a data source with an alias. + lastAlias *tree.AliasClause } // New creates a new Builder structure initialized with the given diff --git a/pkg/sql/opt/optbuilder/scalar.go b/pkg/sql/opt/optbuilder/scalar.go index c6b50b81571f..e7196370edd4 100644 --- a/pkg/sql/opt/optbuilder/scalar.go +++ b/pkg/sql/opt/optbuilder/scalar.go @@ -551,7 +551,7 @@ func (b *Builder) buildFunction( }) if isGenerator(def) { - return b.finishBuildGeneratorFunction(f, out, inScope, outScope, outCol) + return b.finishBuildGeneratorFunction(f, def, out, inScope, outScope, outCol) } // Add a dependency on sequences that are used as a string argument. diff --git a/pkg/sql/opt/optbuilder/select.go b/pkg/sql/opt/optbuilder/select.go index 5b1580da7607..40b02a1f997e 100644 --- a/pkg/sql/opt/optbuilder/select.go +++ b/pkg/sql/opt/optbuilder/select.go @@ -46,6 +46,11 @@ func (b *Builder) buildDataSource( inScope.atRoot = prevAtRoot }(inScope.atRoot) inScope.atRoot = false + if b.lastAlias != nil { + // Make sure we reset the last alias state if it's been set, so we don't + // keep it past a single level of recursion. + defer func() { b.lastAlias = nil }() + } // NB: The case statements are sorted lexicographically. switch source := texpr.(type) { case *tree.AliasedTableExpr: @@ -55,6 +60,7 @@ func (b *Builder) buildDataSource( indexFlags = source.IndexFlags } if source.As.Alias != "" { + b.lastAlias = &source.As locking = locking.filter(source.As.Alias) } @@ -331,7 +337,7 @@ func (b *Builder) renameSource(as tree.AliasClause, scope *scope) { // table name. noColNameSpecified := len(colAlias) == 0 if scope.isAnonymousTable() && noColNameSpecified && scope.singleSRFColumn { - colAlias = tree.NameList{as.Alias} + colAlias = tree.ColumnDefList{tree.ColumnDef{Name: as.Alias}} } // If an alias was specified, use that to qualify the column names. @@ -361,11 +367,11 @@ func (b *Builder) renameSource(as tree.AliasClause, scope *scope) { if col.visibility != visible { continue } - col.name = scopeColName(colAlias[aliasIdx]) + col.name = scopeColName(colAlias[aliasIdx].Name) if isScan { // Override column metadata alias. colMeta := b.factory.Metadata().ColumnMeta(col.id) - colMeta.Alias = string(colAlias[aliasIdx]) + colMeta.Alias = string(colAlias[aliasIdx].Name) } aliasIdx++ } diff --git a/pkg/sql/opt/optbuilder/srfs.go b/pkg/sql/opt/optbuilder/srfs.go index 1cd06fba4651..94a673f75911 100644 --- a/pkg/sql/opt/optbuilder/srfs.go +++ b/pkg/sql/opt/optbuilder/srfs.go @@ -15,6 +15,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util/errorutil/unimplemented" @@ -142,12 +144,44 @@ func (b *Builder) buildZip(exprs tree.Exprs, inScope *scope) (outScope *scope) { // (SRF) such as generate_series() or unnest(). It synthesizes new columns in // outScope for each of the SRF's output columns. func (b *Builder) finishBuildGeneratorFunction( - f *tree.FuncExpr, fn opt.ScalarExpr, inScope, outScope *scope, outCol *scopeColumn, + f *tree.FuncExpr, + def *tree.FunctionDefinition, + fn opt.ScalarExpr, + inScope, outScope *scope, + outCol *scopeColumn, ) (out opt.ScalarExpr) { + if def.ReturnsRecordType { + if b.lastAlias == nil { + panic(pgerror.New(pgcode.Syntax, "a column definition list is required for functions returning \"record\"")) + } + } else if b.lastAlias != nil { + // Non-record type return with a table alias that includes types is not + // permitted. + for _, c := range b.lastAlias.Cols { + if c.Type != nil { + panic(pgerror.Newf(pgcode.Syntax, "a column definition list is only allowed for functions returning \"record\"")) + } + } + } // Add scope columns. if outCol != nil { // Single-column return type. b.populateSynthesizedColumn(outCol, fn) + } else if def.ReturnsRecordType && b.lastAlias != nil && len(b.lastAlias.Cols) > 0 { + // If we're building a generator function that returns a record type, like + // json_to_record, we need to know the alias that was assigned to the + // generator function - without that, we won't know the list of columns + // to output. + for _, c := range b.lastAlias.Cols { + if c.Type == nil { + panic(pgerror.Newf(pgcode.Syntax, "a column definition list is required for functions returning \"record\"")) + } + typ, err := tree.ResolveType(b.ctx, c.Type, b.semaCtx.TypeResolver) + if err != nil { + panic(err) + } + b.synthesizeColumn(outScope, scopeColName(c.Name), typ, nil, fn) + } } else { // Multi-column return type. Use the tuple labels in the SRF's return type // as column aliases. diff --git a/pkg/sql/opt/optbuilder/with.go b/pkg/sql/opt/optbuilder/with.go index 2dee1a4bc40c..48f1ef9864bd 100644 --- a/pkg/sql/opt/optbuilder/with.go +++ b/pkg/sql/opt/optbuilder/with.go @@ -382,7 +382,7 @@ func (b *Builder) getCTECols(cteScope *scope, name tree.AliasClause) physical.Pr )) } for i := range presentation { - presentation[i].Alias = string(name.Cols[i]) + presentation[i].Alias = string(name.Cols[i].Name) } return presentation } diff --git a/pkg/sql/opt/optgen/cmd/optgen/metadata.go b/pkg/sql/opt/optgen/cmd/optgen/metadata.go index a304e19cad9f..a4388507ab23 100644 --- a/pkg/sql/opt/optgen/cmd/optgen/metadata.go +++ b/pkg/sql/opt/optgen/cmd/optgen/metadata.go @@ -207,6 +207,7 @@ func newMetadata(compiled *lang.CompiledExpr, pkg string) *metadata { "JoinFlags": {fullName: "memo.JoinFlags", passByVal: true}, "WindowFrame": {fullName: "memo.WindowFrame", passByVal: true}, "FKCascades": {fullName: "memo.FKCascades", passByVal: true}, + "ColumnDefList": {fullName: "tree.ColumnDefList", passByVal: true}, "ExplainOptions": {fullName: "tree.ExplainOptions", passByVal: true}, "StatementReturnType": {fullName: "tree.StatementReturnType", passByVal: true}, "StatementType": {fullName: "tree.StatementType", passByVal: true}, diff --git a/pkg/sql/parser/parse_test.go b/pkg/sql/parser/parse_test.go index 2fc2b420d2cd..149779ed55b5 100644 --- a/pkg/sql/parser/parse_test.go +++ b/pkg/sql/parser/parse_test.go @@ -526,8 +526,6 @@ func TestUnimplementedSyntax(t *testing.T) { {`INSERT INTO foo(a, a.b) VALUES (1,2)`, 27792, ``, ``}, - {`SELECT * FROM ROWS FROM (a(b) AS (d))`, 0, `ROWS FROM with col_def_list`, ``}, - {`SELECT a(b) 'c'`, 0, `a(...) SCONST`, ``}, {`SELECT UNIQUE (SELECT b)`, 0, `UNIQUE predicate`, ``}, {`SELECT GROUPING (a,b,c)`, 0, `d_expr grouping`, ``}, diff --git a/pkg/sql/parser/sql.y b/pkg/sql/parser/sql.y index 7585717c2597..7bd51ad93bb5 100644 --- a/pkg/sql/parser/sql.y +++ b/pkg/sql/parser/sql.y @@ -359,9 +359,15 @@ func (u *sqlSymUnion) slct() *tree.Select { func (u *sqlSymUnion) selectStmt() tree.SelectStatement { return u.val.(tree.SelectStatement) } -func (u *sqlSymUnion) colDef() *tree.ColumnTableDef { +func (u *sqlSymUnion) colTableDef() *tree.ColumnTableDef { return u.val.(*tree.ColumnTableDef) } +func (u *sqlSymUnion) colDef() tree.ColumnDef { + return u.val.(tree.ColumnDef) +} +func (u *sqlSymUnion) colDefList() tree.ColumnDefList { + return u.val.(tree.ColumnDefList) +} func (u *sqlSymUnion) constraintDef() tree.ConstraintTableDef { return u.val.(tree.ConstraintTableDef) } @@ -1368,7 +1374,8 @@ func (u *sqlSymUnion) routineBody() *tree.RoutineBody { %type first_or_next %type insert_rest -%type opt_col_def_list +%type opt_col_def_list col_def_list opt_col_def_list_no_types col_def_list_no_types +%type col_def %type <*tree.OnConflict> on_conflict %type begin_transaction @@ -1377,7 +1384,7 @@ func (u *sqlSymUnion) routineBody() *tree.RoutineBody { %type opt_hash_sharded_bucket_count %type <*tree.ShardedIndexDef> opt_hash_sharded %type opt_storing -%type <*tree.ColumnTableDef> column_def +%type <*tree.ColumnTableDef> column_table_def %type table_elem %type where_clause opt_where_clause %type <*tree.ArraySubscript> array_subscript @@ -1399,7 +1406,7 @@ func (u *sqlSymUnion) routineBody() *tree.RoutineBody { %type <[]*tree.When> when_clause_list %type sub_type %type numeric_only -%type alias_clause opt_alias_clause +%type alias_clause opt_alias_clause func_alias_clause opt_func_alias_clause %type opt_ordinality opt_compact %type <*tree.Order> sortby %type index_elem index_elem_options create_as_param @@ -2375,24 +2382,24 @@ alter_table_cmd: $$.val = &tree.AlterTableRenameConstraint{Constraint: tree.Name($3), NewName: tree.Name($5) } } // ALTER TABLE ADD -| ADD column_def +| ADD column_table_def { - $$.val = &tree.AlterTableAddColumn{IfNotExists: false, ColumnDef: $2.colDef()} + $$.val = &tree.AlterTableAddColumn{IfNotExists: false, ColumnDef: $2.colTableDef()} } // ALTER TABLE ADD IF NOT EXISTS -| ADD IF NOT EXISTS column_def +| ADD IF NOT EXISTS column_table_def { - $$.val = &tree.AlterTableAddColumn{IfNotExists: true, ColumnDef: $5.colDef()} + $$.val = &tree.AlterTableAddColumn{IfNotExists: true, ColumnDef: $5.colTableDef()} } // ALTER TABLE ADD COLUMN -| ADD COLUMN column_def +| ADD COLUMN column_table_def { - $$.val = &tree.AlterTableAddColumn{IfNotExists: false, ColumnDef: $3.colDef()} + $$.val = &tree.AlterTableAddColumn{IfNotExists: false, ColumnDef: $3.colTableDef()} } // ALTER TABLE ADD COLUMN IF NOT EXISTS -| ADD COLUMN IF NOT EXISTS column_def +| ADD COLUMN IF NOT EXISTS column_table_def { - $$.val = &tree.AlterTableAddColumn{IfNotExists: true, ColumnDef: $6.colDef()} + $$.val = &tree.AlterTableAddColumn{IfNotExists: true, ColumnDef: $6.colTableDef()} } // ALTER TABLE ALTER [COLUMN] {SET DEFAULT |DROP DEFAULT} | ALTER opt_column column_name alter_column_default @@ -7778,9 +7785,9 @@ table_elem_list: } table_elem: - column_def + column_table_def { - $$.val = $1.colDef() + $$.val = $1.colTableDef() } | index_def | family_def @@ -7965,7 +7972,7 @@ range_partition: // Treat SERIAL pseudo-types as separate case so that types.T does not have to // support them as first-class types (e.g. they should not be supported as CAST // target types). -column_def: +column_table_def: column_name typename col_qual_list { typ := $2.typeReference() @@ -10723,20 +10730,20 @@ materialize_clause: } common_table_expr: - table_alias_name opt_column_list AS '(' preparable_stmt ')' + table_alias_name opt_col_def_list_no_types AS '(' preparable_stmt ')' { $$.val = &tree.CTE{ - Name: tree.AliasClause{Alias: tree.Name($1), Cols: $2.nameList() }, + Name: tree.AliasClause{Alias: tree.Name($1), Cols: $2.colDefList() }, Mtr: tree.MaterializeClause{ Set: false, }, Stmt: $5.stmt(), } } -| table_alias_name opt_column_list AS materialize_clause '(' preparable_stmt ')' +| table_alias_name opt_col_def_list_no_types AS materialize_clause '(' preparable_stmt ')' { $$.val = &tree.CTE{ - Name: tree.AliasClause{Alias: tree.Name($1), Cols: $2.nameList() }, + Name: tree.AliasClause{Alias: tree.Name($1), Cols: $2.colDefList() }, Mtr: tree.MaterializeClause{ Materialize: $4.bool(), Set: true, @@ -11241,7 +11248,7 @@ table_ref: { $$.val = &tree.AliasedTableExpr{Expr: &tree.ParenTableExpr{Expr: $2.tblExpr()}, Ordinality: $4.bool(), As: $5.aliasClause()} } -| func_table opt_ordinality opt_alias_clause +| func_table opt_ordinality opt_func_alias_clause { f := $1.tblExpr() $$.val = &tree.AliasedTableExpr{ @@ -11309,16 +11316,61 @@ rowsfrom_list: { $$.val = append($1.exprs(), $3.expr()) } rowsfrom_item: - func_expr_windowless opt_col_def_list + func_expr_windowless opt_func_alias_clause { $$.val = $1.expr() } +opt_col_def_list_no_types: + '(' col_def_list_no_types ')' + { + $$.val = $2.colDefList() + } +| /* EMPTY */ + { + $$.val = tree.ColumnDefList(nil) + } + +col_def_list_no_types: + name + { + $$.val = tree.ColumnDefList{tree.ColumnDef{Name: tree.Name($1)}} + } +| col_def_list_no_types ',' name + { + $$.val = append($1.colDefList(), tree.ColumnDef{Name: tree.Name($3)}) + } + + opt_col_def_list: /* EMPTY */ - { } -| AS '(' error - { return unimplemented(sqllex, "ROWS FROM with col_def_list") } + { + $$.val = tree.ColumnDefList(nil) + } +| '(' col_def_list ')' + { + $$.val = $2.colDefList() + } + +col_def_list: + col_def + { + $$.val = tree.ColumnDefList{$1.colDef()} + } +| col_def_list ',' col_def + { + $$.val = append($1.colDefList(), $3.colDef()) + } + +col_def: + name + { + $$.val = tree.ColumnDef{Name: tree.Name($1)} + } +| name typename + { + $$.val = tree.ColumnDef{Name: tree.Name($1), Type: $2.typeReference()} + } opt_tableref_col_list: /* EMPTY */ { $$.val = nil } @@ -11386,13 +11438,13 @@ joined_table: } alias_clause: - AS table_alias_name opt_column_list + AS table_alias_name opt_col_def_list_no_types { - $$.val = tree.AliasClause{Alias: tree.Name($2), Cols: $3.nameList()} + $$.val = tree.AliasClause{Alias: tree.Name($2), Cols: $3.colDefList()} } -| table_alias_name opt_column_list +| table_alias_name opt_col_def_list_no_types { - $$.val = tree.AliasClause{Alias: tree.Name($1), Cols: $2.nameList()} + $$.val = tree.AliasClause{Alias: tree.Name($1), Cols: $2.colDefList()} } opt_alias_clause: @@ -11402,6 +11454,23 @@ opt_alias_clause: $$.val = tree.AliasClause{} } +func_alias_clause: + AS table_alias_name opt_col_def_list + { + $$.val = tree.AliasClause{Alias: tree.Name($2), Cols: $3.colDefList()} + } +| table_alias_name opt_col_def_list + { + $$.val = tree.AliasClause{Alias: tree.Name($1), Cols: $2.colDefList()} + } + +opt_func_alias_clause: + func_alias_clause +| /* EMPTY */ + { + $$.val = tree.AliasClause{} + } + as_of_clause: AS_LA OF SYSTEM TIME a_expr { diff --git a/pkg/sql/parser/testdata/select_clauses b/pkg/sql/parser/testdata/select_clauses index 471144618058..116c288d1265 100644 --- a/pkg/sql/parser/testdata/select_clauses +++ b/pkg/sql/parser/testdata/select_clauses @@ -3035,3 +3035,11 @@ SELECT * FROM ((t1 NATURAL JOIN t2 WITH ORDINALITY AS o1)) WITH ORDINALITY AS o2 SELECT (*) FROM ((t1 NATURAL JOIN t2 WITH ORDINALITY AS o1)) WITH ORDINALITY AS o2 -- fully parenthesized SELECT * FROM ((t1 NATURAL JOIN t2 WITH ORDINALITY AS o1)) WITH ORDINALITY AS o2 -- literals removed SELECT * FROM ((_ NATURAL JOIN _ WITH ORDINALITY AS _)) WITH ORDINALITY AS _ -- identifiers removed + +parse +SELECT * FROM json_to_record('') AS t(a INT, b TEXT, c foo) +---- +SELECT * FROM ROWS FROM (json_to_record('')) AS t (a INT8, b STRING, c foo) -- normalized! +SELECT (*) FROM ROWS FROM ((json_to_record(('')))) AS t (a INT8, b STRING, c foo) -- fully parenthesized +SELECT * FROM ROWS FROM (json_to_record('_')) AS t (a INT8, b STRING, c foo) -- literals removed +SELECT * FROM ROWS FROM (json_to_record('')) AS _ (_ INT8, _ STRING, _ foo) -- identifiers removed diff --git a/pkg/sql/rowexec/project_set.go b/pkg/sql/rowexec/project_set.go index 852340a0ecd0..ea8e1959345f 100644 --- a/pkg/sql/rowexec/project_set.go +++ b/pkg/sql/rowexec/project_set.go @@ -168,6 +168,11 @@ func (ps *projectSetProcessor) nextInputRow() ( if gen == nil { gen = builtins.EmptyGenerator() } + if aliasSetter, ok := gen.(eval.AliasAwareValueGenerator); ok { + if err := aliasSetter.SetAlias(ps.spec.GeneratedColumns, ps.spec.GeneratedColumnLabels); err != nil { + return nil, nil, err + } + } if err := gen.Start(ps.Ctx, ps.FlowCtx.Txn); err != nil { return nil, nil, err } diff --git a/pkg/sql/sem/builtins/builtins.go b/pkg/sql/sem/builtins/builtins.go index 31ee2b326d9e..fd33257dd663 100644 --- a/pkg/sql/sem/builtins/builtins.go +++ b/pkg/sql/sem/builtins/builtins.go @@ -3861,9 +3861,6 @@ value if you rely on the HLC for accuracy.`, // The behavior of both the JSON and JSONB data types in CockroachDB is // similar to the behavior of the JSONB data type in Postgres. - "json_to_recordset": makeBuiltin(tree.FunctionProperties{UnsupportedWithIssue: 33285, Category: builtinconstants.CategoryJSON}), - "jsonb_to_recordset": makeBuiltin(tree.FunctionProperties{UnsupportedWithIssue: 33285, Category: builtinconstants.CategoryJSON}), - "jsonb_path_exists": makeBuiltin(tree.FunctionProperties{UnsupportedWithIssue: 22513, Category: builtinconstants.CategoryJSON}), "jsonb_path_exists_opr": makeBuiltin(tree.FunctionProperties{UnsupportedWithIssue: 22513, Category: builtinconstants.CategoryJSON}), "jsonb_path_match": makeBuiltin(tree.FunctionProperties{UnsupportedWithIssue: 22513, Category: builtinconstants.CategoryJSON}), diff --git a/pkg/sql/sem/builtins/generator_builtins.go b/pkg/sql/sem/builtins/generator_builtins.go index a5f01585ae55..d076e551749c 100644 --- a/pkg/sql/sem/builtins/generator_builtins.go +++ b/pkg/sql/sem/builtins/generator_builtins.go @@ -72,6 +72,14 @@ func genPropsWithLabels(returnLabels []string) tree.FunctionProperties { } } +func recordGenProps() tree.FunctionProperties { + return tree.FunctionProperties{ + Class: tree.GeneratorClass, + Category: builtinconstants.CategoryGenerator, + ReturnsRecordType: true, + } +} + var aclexplodeGeneratorType = types.MakeLabeledTuple( []*types.T{types.Oid, types.Oid, types.String, types.Bool}, []string{"grantor", "grantee", "privilege_type", "is_grantable"}, @@ -378,6 +386,11 @@ var generators = map[string]builtinDefinition{ "jsonb_populate_recordset": makeBuiltin(jsonPopulateProps, makeJSONPopulateImpl(makeJSONPopulateRecordSetGenerator, "Expands the outermost array of objects in from_json to a set of rows whose columns match the record type defined by base")), + "json_to_record": makeBuiltin(recordGenProps(), jsonToRecordImpl), + "jsonb_to_record": makeBuiltin(recordGenProps(), jsonToRecordImpl), + "json_to_recordset": makeBuiltin(recordGenProps(), jsonToRecordSetImpl), + "jsonb_to_recordset": makeBuiltin(recordGenProps(), jsonToRecordSetImpl), + "crdb_internal.check_consistency": makeBuiltin( tree.FunctionProperties{ Class: tree.GeneratorClass, @@ -1381,6 +1394,26 @@ var jsonEachTextImpl = makeGeneratorOverload( volatility.Immutable, ) +var jsonToRecordImpl = makeGeneratorOverload( + tree.ArgTypes{{"input", types.Jsonb}}, + // NOTE: this type will never actually get used. It is replaced in the + // optimizer by looking at the most recent AS alias clause. + types.EmptyTuple, + makeJSONRecordGenerator, + "Builds an arbitrary record from a JSON object.", + volatility.Stable, +) + +var jsonToRecordSetImpl = makeGeneratorOverload( + tree.ArgTypes{{"input", types.Jsonb}}, + // NOTE: this type will never actually get used. It is replaced in the + // optimizer by looking at the most recent AS alias clause. + types.EmptyTuple, + makeJSONRecordSetGenerator, + "Builds an arbitrary set of records from a JSON array of objects.", + volatility.Stable, +) + var jsonEachGeneratorLabels = []string{"key", "value"} var jsonEachGeneratorType = types.MakeLabeledTuple( @@ -1666,6 +1699,134 @@ func (j *jsonPopulateRecordSetGenerator) Values() (tree.Datums, error) { return output.D, nil } +func makeJSONRecordGenerator(evalCtx *eval.Context, args tree.Datums) (eval.ValueGenerator, error) { + target := tree.MustBeDJSON(args[0]) + return &jsonRecordGenerator{ + evalCtx: evalCtx, + target: target.JSON, + }, nil +} + +type jsonRecordGenerator struct { + evalCtx *eval.Context + target json.JSON + + wasCalled bool + values tree.Datums + types []*types.T + labels []string + // labelToRowIndexMap maps the column label to its position within the row. + labelToRowIndexMap map[string]int +} + +func (j *jsonRecordGenerator) SetAlias(types []*types.T, labels []string) error { + j.types = types + j.labels = labels + j.labelToRowIndexMap = make(map[string]int) + for i := range types { + j.labelToRowIndexMap[j.labels[i]] = i + } + if len(types) != len(labels) { + return errors.AssertionFailedf("unexpected mismatched types/labels list in json record generator %v %v", types, labels) + } + return nil +} + +func (j jsonRecordGenerator) ResolvedType() *types.T { + return types.AnyTuple +} + +func (j *jsonRecordGenerator) Start(ctx context.Context, _ *kv.Txn) error { + j.values = make(tree.Datums, len(j.types)) + if j.target.Type() != json.ObjectJSONType { + return pgerror.Newf(pgcode.InvalidParameterValue, + "invalid non-object argument to json_to_record") + } + return nil +} + +func (j *jsonRecordGenerator) Next(ctx context.Context) (bool, error) { + if j.wasCalled { + return false, nil + } + for i := range j.values { + j.values[i] = tree.DNull + } + iter, err := j.target.ObjectIter() + if err != nil { + return false, err + } + for iter.Next() { + idx, ok := j.labelToRowIndexMap[iter.Key()] + if !ok { + continue + } + v := iter.Value() + datum, err := eval.PopulateDatumWithJSON(j.evalCtx, v, j.types[idx]) + if err != nil { + return false, err + } + j.values[idx] = datum + } + + j.wasCalled = true + return true, nil +} + +func (j jsonRecordGenerator) Values() (tree.Datums, error) { + return j.values, nil +} + +func (j jsonRecordGenerator) Close(ctx context.Context) {} + +type jsonRecordSetGenerator struct { + jsonRecordGenerator + + arr tree.DJSON + nextIndex int +} + +func makeJSONRecordSetGenerator( + evalCtx *eval.Context, args tree.Datums, +) (eval.ValueGenerator, error) { + arr := tree.MustBeDJSON(args[0]) + return &jsonRecordSetGenerator{ + arr: arr, + jsonRecordGenerator: jsonRecordGenerator{ + evalCtx: evalCtx, + }, + }, nil +} + +func (j *jsonRecordSetGenerator) Start(ctx context.Context, _ *kv.Txn) error { + j.values = make(tree.Datums, len(j.types)) + if j.arr.Type() != json.ArrayJSONType { + return pgerror.Newf(pgcode.InvalidParameterValue, + "argument to json_to_recordset must be an array of objects") + } + j.nextIndex = -1 + return nil +} + +func (j *jsonRecordSetGenerator) Next(ctx context.Context) (bool, error) { + j.nextIndex++ + next, err := j.arr.FetchValIdx(j.nextIndex) + if err != nil || next == nil { + return false, err + } + if next.Type() != json.ObjectJSONType { + return false, pgerror.Newf(pgcode.InvalidParameterValue, + "argument to json_to_recordset must be an array of objects") + } + j.target = next + j.wasCalled = false + _, err = j.jsonRecordGenerator.Next(ctx) + if err != nil { + return false, err + } + return true, nil +} + type checkConsistencyGenerator struct { consistencyChecker eval.ConsistencyCheckRunner from, to roachpb.Key diff --git a/pkg/sql/sem/eval/generators.go b/pkg/sql/sem/eval/generators.go index 754b773c3133..5ac37a9472a0 100644 --- a/pkg/sql/sem/eval/generators.go +++ b/pkg/sql/sem/eval/generators.go @@ -83,6 +83,12 @@ type ValueGenerator interface { Close(ctx context.Context) } +// AliasAwareValueGenerator is a value generator that can inspect the alias with +// which it was invoked. SetAlias will always be run before Start. +type AliasAwareValueGenerator interface { + SetAlias(types []*types.T, labels []string) error +} + // streamingValueGenerator is a marker-type indicating that the wrapped // generator is of "streaming" nature, thus, projectSet processor must be // streaming too. diff --git a/pkg/sql/sem/tree/function_definition.go b/pkg/sql/sem/tree/function_definition.go index cb83d94a8ced..c5b7e0792a96 100644 --- a/pkg/sql/sem/tree/function_definition.go +++ b/pkg/sql/sem/tree/function_definition.go @@ -94,6 +94,16 @@ type FunctionProperties struct { // // See memo.CanBeCompositeSensitive. CompositeInsensitive bool + + // ReturnsRecordType indicates that this function is a record-returning + // function, which implies that it's unusable without a corresponding type + // alias. + // + // For example, consider the case of json_to_record('{"a":"b", "c":"d"}'). + // This function returns an error unless it as an `AS t(a,b,c)` declaration, + // since its definition is to pick out the JSON attributes within the input + // that match, by name, to the columns in the aliased record type. + ReturnsRecordType bool } // ShouldDocument returns whether the built-in function should be included in diff --git a/pkg/sql/sem/tree/select.go b/pkg/sql/sem/tree/select.go index a6fe48f23592..f0255adac955 100644 --- a/pkg/sql/sem/tree/select.go +++ b/pkg/sql/sem/tree/select.go @@ -187,24 +187,55 @@ func (node *SelectExpr) Format(ctx *FmtCtx) { } } -// AliasClause represents an alias, optionally with a column list: -// "AS name" or "AS name(col1, col2)". +// AliasClause represents an alias, optionally with a column def list: +// "AS name", "AS name(col1, col2)", or "AS name(col1 INT, col2 STRING)". +// Note that the last form is only valid in the context of record-returning +// functions, which also require the last form to define their output types. type AliasClause struct { Alias Name - Cols NameList + Cols ColumnDefList } // Format implements the NodeFormatter interface. -func (a *AliasClause) Format(ctx *FmtCtx) { - ctx.FormatNode(&a.Alias) - if len(a.Cols) != 0 { +func (f *AliasClause) Format(ctx *FmtCtx) { + ctx.FormatNode(&f.Alias) + if len(f.Cols) != 0 { // Format as "alias (col1, col2, ...)". ctx.WriteString(" (") - ctx.FormatNode(&a.Cols) + ctx.FormatNode(&f.Cols) ctx.WriteByte(')') } } +// ColumnDef represents a column definition in the context of a record type +// alias, like in select * from json_to_record(...) AS foo(a INT, b INT). +type ColumnDef struct { + Name Name + Type ResolvableTypeReference +} + +// Format implements the NodeFormatter interface. +func (c *ColumnDef) Format(ctx *FmtCtx) { + ctx.FormatNode(&c.Name) + if c.Type != nil { + ctx.WriteByte(' ') + ctx.WriteString(c.Type.SQLString()) + } +} + +// ColumnDefList represents a list of ColumnDefs. +type ColumnDefList []ColumnDef + +// Format implements the NodeFormatter interface. +func (c *ColumnDefList) Format(ctx *FmtCtx) { + for i := range *c { + if i > 0 { + ctx.WriteString(", ") + } + ctx.FormatNode(&(*c)[i]) + } +} + // AsOfClause represents an as of time. type AsOfClause struct { Expr Expr