sql/parser: support GENERATED {ALWAYS | BY DEFAULT} AS IDENTITY syntax

This commit is to allow the parser to support
the `GENERATED {ALWAYS | BY DEFAULT} AS IDENTITY` syntax
under `CREATE TABLE` statement.

Release note (sql change): Added support for `GENERATED {ALWAYS | BY DEFAULT} AS IDENTITY`
syntax in a column definition. This will automatically create
a sequence for the given column.
This matches PostgreSQL syntax and functionality.
See https://www.postgresql.org/docs/current/sql-createtable.html

docs/generated/sql/bnf/col_qualification.bnf

@@ -8,10 +8,10 @@ col_qualification ::=
 	| 'CONSTRAINT' constraint_name 'CHECK' '(' a_expr ')'
 	| 'CONSTRAINT' constraint_name 'DEFAULT' b_expr
 	| 'CONSTRAINT' constraint_name 'REFERENCES' table_name opt_name_parens key_match reference_actions
-	| 'CONSTRAINT' constraint_name 'AS' '(' a_expr ')' 'STORED'
-	| 'CONSTRAINT' constraint_name 'GENERATED_ALWAYS' 'ALWAYS' 'AS' '(' a_expr ')' 'STORED'
-	| 'CONSTRAINT' constraint_name 'AS' '(' a_expr ')' 'VIRTUAL'
-	| 'CONSTRAINT' constraint_name 'GENERATED_ALWAYS' 'ALWAYS' 'AS' '(' a_expr ')' 'VIRTUAL'
+	| 'CONSTRAINT' constraint_name generated_as '(' a_expr ')' 'STORED'
+	| 'CONSTRAINT' constraint_name generated_as '(' a_expr ')' 'VIRTUAL'
+	| 'CONSTRAINT' constraint_name 'GENERATED_ALWAYS' 'ALWAYS' 'AS' 'IDENTITY'
+	| 'CONSTRAINT' constraint_name 'GENERATED_BY_DEFAULT' 'BY' 'DEFAULT' 'AS' 'IDENTITY'
 	| 'NOT' 'NULL'
 	| 'NULL'
 	| 'NOT' 'VISIBLE'
@@ -21,10 +21,10 @@ col_qualification ::=
 	| 'CHECK' '(' a_expr ')'
 	| 'DEFAULT' b_expr
 	| 'REFERENCES' table_name opt_name_parens key_match reference_actions
-	| 'AS' '(' a_expr ')' 'STORED'
-	| 'GENERATED_ALWAYS' 'ALWAYS' 'AS' '(' a_expr ')' 'STORED'
-	| 'AS' '(' a_expr ')' 'VIRTUAL'
-	| 'GENERATED_ALWAYS' 'ALWAYS' 'AS' '(' a_expr ')' 'VIRTUAL'
+	| generated_as '(' a_expr ')' 'STORED'
+	| generated_as '(' a_expr ')' 'VIRTUAL'
+	| 'GENERATED_ALWAYS' 'ALWAYS' 'AS' 'IDENTITY'
+	| 'GENERATED_BY_DEFAULT' 'BY' 'DEFAULT' 'AS' 'IDENTITY'
 	| 'COLLATE' collation_name
 	| 'FAMILY' family_name
 	| 'CREATE' 'FAMILY' family_name

docs/generated/sql/bnf/stmt_block.bnf

@@ -3036,6 +3036,8 @@ col_qualification_elem ::=
 	| 'REFERENCES' table_name opt_name_parens key_match reference_actions
 	| generated_as '(' a_expr ')' 'STORED'
 	| generated_as '(' a_expr ')' 'VIRTUAL'
+	| generated_always_as 'IDENTITY'
+	| generated_by_default_as 'IDENTITY'
 
 family_name ::=
 	name
@@ -3175,7 +3177,13 @@ opt_name_parens ::=
 
 generated_as ::=
 	'AS'
-	| 'GENERATED_ALWAYS' 'ALWAYS' 'AS'
+	| generated_always_as
+
+generated_always_as ::=
+	'GENERATED_ALWAYS' 'ALWAYS' 'AS'
+
+generated_by_default_as ::=
+	'GENERATED_BY_DEFAULT' 'BY' 'DEFAULT' 'AS'
 
 reference_action ::=
 	'NO' 'ACTION'

pkg/cmd/docgen/diagrams.go

@@ -535,7 +535,7 @@ var specs = []stmtSpec{
 	{
 		name:   "col_qualification",
 		stmt:   "col_qualification",
-		inline: []string{"col_qualification_elem", "opt_hash_sharded", "generated_as"},
+		inline: []string{"col_qualification_elem", "opt_hash_sharded", "generated_always_as", "generated_by_default_as"},
 		replace: map[string]string{
 			"'=' a_expr": "'=' n_buckets",
 		},

pkg/sql/parser/lexer.go

@@ -102,6 +102,8 @@ func (l *lexer) Lex(lval *sqlSymType) int {
 			switch nextID {
 			case ALWAYS:
 				lval.id = GENERATED_ALWAYS
+			case BY:
+				lval.id = GENERATED_BY_DEFAULT
 			}
 
 		case WITH:

pkg/sql/parser/sql.y

@@ -852,6 +852,7 @@ func (u *sqlSymUnion) setVar() *tree.SetVar {
 // The grammar thinks these are keywords, but they are not in any category
 // and so can never be entered directly. The filter in scan.go creates these
 // tokens when required (based on looking one token ahead).
+// Reference: pkg/sql/parser/lexer.go
 //
 // NOT_LA exists so that productions such as NOT LIKE can be given the same
 // precedence as LIKE; otherwise they'd effectively have the same precedence as
@@ -861,7 +862,7 @@ func (u *sqlSymUnion) setVar() *tree.SetVar {
 // extensions (CREATE FAMILY/CREATE FAMILY family_name). RESET_ALL is used
 // to differentiate `RESET var` from `RESET ALL`. ROLE_ALL and USER_ALL are
 // used in ALTER ROLE statements that affect all roles.
-%token NOT_LA NULLS_LA WITH_LA AS_LA GENERATED_ALWAYS RESET_ALL ROLE_ALL USER_ALL
+%token NOT_LA NULLS_LA WITH_LA AS_LA GENERATED_ALWAYS GENERATED_BY_DEFAULT RESET_ALL ROLE_ALL USER_ALL
 
 %union {
   id    int32
@@ -6168,11 +6169,11 @@ alter_schema_stmt:
 // Table constraints:
 //    PRIMARY KEY ( <colnames...> ) [USING HASH WITH BUCKET_COUNT = <shard_buckets>]
 //    FOREIGN KEY ( <colnames...> ) REFERENCES <tablename> [( <colnames...> )] [ON DELETE {NO ACTION | RESTRICT}] [ON UPDATE {NO ACTION | RESTRICT}]
-//    UNIQUE ( <colnames... ) [{STORING | INCLUDE | COVERING} ( <colnames...> )] [<interleave>]
+//    UNIQUE ( <colnames...> ) [{STORING | INCLUDE | COVERING} ( <colnames...> )] [<interleave>]
 //    CHECK ( <expr> )
 //
 // Column qualifiers:
-//   [CONSTRAINT <constraintname>] {NULL | NOT NULL | NOT VISIBLE | UNIQUE | PRIMARY KEY | CHECK (<expr>) | DEFAULT <expr>}
+//   [CONSTRAINT <constraintname>] {NULL | NOT NULL | NOT VISIBLE | UNIQUE | PRIMARY KEY | CHECK (<expr>) | DEFAULT <expr> | GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [( <opt_sequence_option_list> )]}
 //   FAMILY <familyname>, CREATE [IF NOT EXISTS] FAMILY [<familyname>]
 //   REFERENCES <tablename> [( <colnames...> )] [ON DELETE {NO ACTION | RESTRICT}] [ON UPDATE {NO ACTION | RESTRICT}]
 //   COLLATE <collationname>
@@ -6703,28 +6704,36 @@ col_qualification_elem:
     $$.val = &tree.ColumnDefault{Expr: $2.expr()}
   }
 | REFERENCES table_name opt_name_parens key_match reference_actions
- {
+  {
     name := $2.unresolvedObjectName().ToTableName()
     $$.val = &tree.ColumnFKConstraint{
       Table: name,
       Col: tree.Name($3),
       Actions: $5.referenceActions(),
       Match: $4.compositeKeyMatchMethod(),
     }
- }
+  }
 | generated_as '(' a_expr ')' STORED
- {
+  {
     $$.val = &tree.ColumnComputedDef{Expr: $3.expr(), Virtual: false}
- }
+  }
 | generated_as '(' a_expr ')' VIRTUAL
- {
+  {
     $$.val = &tree.ColumnComputedDef{Expr: $3.expr(), Virtual: true}
- }
+  }
 | generated_as error
- {
+  {
     sqllex.Error("use AS ( <expr> ) STORED or AS ( <expr> ) VIRTUAL")
     return 1
- }
+  }
+| generated_always_as IDENTITY
+  {
+    $$.val = &tree.GeneratedAlwaysAsIdentity{}
+  }
+| generated_by_default_as IDENTITY
+  {
+    $$.val = &tree.GeneratedByDefAsIdentity{}
+  }
 
 opt_without_index:
   WITHOUT INDEX
@@ -6737,11 +6746,15 @@ opt_without_index:
     $$.val = false
   }
 
-// GENERATED ALWAYS is a noise word for compatibility with Postgres.
 generated_as:
   AS {}
-| GENERATED_ALWAYS ALWAYS AS {}
+| generated_always_as
+
+generated_always_as:
+  GENERATED_ALWAYS ALWAYS AS {}
 
+generated_by_default_as:
+  GENERATED_BY_DEFAULT BY DEFAULT AS {}
 
 index_def:
   INDEX opt_index_name '(' index_params ')' opt_hash_sharded opt_storing opt_interleave opt_partition_by_index opt_with_storage_parameter_list opt_where_clause

pkg/sql/parser/testdata/alter_table

@@ -289,6 +289,30 @@ ALTER TABLE a ADD COLUMN b INT8 CREATE IF NOT EXISTS FAMILY fam_b -- fully paren
 ALTER TABLE a ADD COLUMN b INT8 CREATE IF NOT EXISTS FAMILY fam_b -- literals removed
 ALTER TABLE _ ADD COLUMN _ INT8 CREATE IF NOT EXISTS FAMILY _ -- identifiers removed
 
+parse
+ALTER TABLE a ADD COLUMN b INT GENERATED ALWAYS AS IDENTITY
+----
+ALTER TABLE a ADD COLUMN b INT8 NOT NULL GENERATED ALWAYS AS IDENTITY -- normalized!
+ALTER TABLE a ADD COLUMN b INT8 NOT NULL GENERATED ALWAYS AS IDENTITY -- fully parenthesized
+ALTER TABLE a ADD COLUMN b INT8 NOT NULL GENERATED ALWAYS AS IDENTITY -- literals removed
+ALTER TABLE _ ADD COLUMN _ INT8 NOT NULL GENERATED ALWAYS AS IDENTITY -- identifiers removed
+
+parse
+ALTER TABLE a ADD COLUMN b INT GENERATED BY DEFAULT AS IDENTITY
+----
+ALTER TABLE a ADD COLUMN b INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY -- normalized!
+ALTER TABLE a ADD COLUMN b INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY -- fully parenthesized
+ALTER TABLE a ADD COLUMN b INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY -- literals removed
+ALTER TABLE _ ADD COLUMN _ INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY -- identifiers removed
+
+parse
+ALTER TABLE a ADD COLUMN b INT NOT NULL GENERATED BY DEFAULT AS IDENTITY
+----
+ALTER TABLE a ADD COLUMN b INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY -- normalized!
+ALTER TABLE a ADD COLUMN b INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY -- fully parenthesized
+ALTER TABLE a ADD COLUMN b INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY -- literals removed
+ALTER TABLE _ ADD COLUMN _ INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY -- identifiers removed
+
 parse
 ALTER TABLE t ALTER PRIMARY KEY USING COLUMNS (a, b, c)
 ----
@@ -1075,3 +1099,43 @@ DETAIL: source SQL:
 ALTER PARTITION p OF TABLE tbl@* CONFIGURE ZONE USING num_replicas = 1
                                  ^
 HINT: try ALTER PARTITION <partition> OF INDEX <tablename>@*
+
+error
+ALTER TABLE t ADD COLUMN b INT NULL GENERATED ALWAYS AS IDENTITY
+----
+at or near "EOF": syntax error: conflicting NULL/NOT NULL declarations for column "b"
+DETAIL: source SQL:
+ALTER TABLE t ADD COLUMN b INT NULL GENERATED ALWAYS AS IDENTITY
+                                                                ^
+
+error
+ALTER TABLE a ADD COLUMN b INT GENERATED BY DEFAULT AS IDENTITY GENERATED ALWAYS AS IDENTITY
+----
+at or near "EOF": syntax error: multiple identity specifications for column "b"
+DETAIL: source SQL:
+ALTER TABLE a ADD COLUMN b INT GENERATED BY DEFAULT AS IDENTITY GENERATED ALWAYS AS IDENTITY
+                                                                                            ^
+
+error
+ALTER TABLE a ADD COLUMN b INT AS (a + 10) STORED GENERATED ALWAYS AS IDENTITY
+----
+at or near "EOF": syntax error: both identity and generation expression specified for column "b"
+DETAIL: source SQL:
+ALTER TABLE a ADD COLUMN b INT AS (a + 10) STORED GENERATED ALWAYS AS IDENTITY
+                                                                              ^
+
+error
+ALTER TABLE a ADD COLUMN b INT GENERATED ALWAYS AS (1 + 1) STORED GENERATED ALWAYS AS IDENTITY
+----
+at or near "EOF": syntax error: both identity and generation expression specified for column "b"
+DETAIL: source SQL:
+ALTER TABLE a ADD COLUMN b INT GENERATED ALWAYS AS (1 + 1) STORED GENERATED ALWAYS AS IDENTITY
+                                                                                              ^
+
+error
+ALTER TABLE a ADD COLUMN b INT GENERATED ALWAYS AS (1+1) IDENTITY
+----
+at or near "identity": syntax error: use AS ( <expr> ) STORED or AS ( <expr> ) VIRTUAL
+DETAIL: source SQL:
+ALTER TABLE a ADD COLUMN b INT GENERATED ALWAYS AS (1+1) IDENTITY
+                                                         ^

pkg/sql/parser/testdata/create_table

@@ -2260,3 +2260,96 @@ CREATE TABLE operator_tbl (a INT8 DEFAULT 1 OPERATOR(+) 2, b INT8 DEFAULT 1 OPER
 CREATE TABLE operator_tbl (a INT8 DEFAULT ((1) OPERATOR(+) (2)), b INT8 DEFAULT ((1) OPERATOR(+) (2))) -- fully parenthesized
 CREATE TABLE operator_tbl (a INT8 DEFAULT _ OPERATOR(+) _, b INT8 DEFAULT _ OPERATOR(+) _) -- literals removed
 CREATE TABLE _ (_ INT8 DEFAULT 1 OPERATOR(+) 2, _ INT8 DEFAULT 1 OPERATOR(+) 2) -- identifiers removed
+
+parse
+CREATE TABLE generated_always_as_identity_tb (
+  a INT UNIQUE,
+  b INT GENERATED ALWAYS AS IDENTITY,
+  c INT GENERATED BY DEFAULT AS IDENTITY
+)
+----
+CREATE TABLE generated_always_as_identity_tb (a INT8 UNIQUE, b INT8 NOT NULL GENERATED ALWAYS AS IDENTITY, c INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY) -- normalized!
+CREATE TABLE generated_always_as_identity_tb (a INT8 UNIQUE, b INT8 NOT NULL GENERATED ALWAYS AS IDENTITY, c INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY) -- fully parenthesized
+CREATE TABLE generated_always_as_identity_tb (a INT8 UNIQUE, b INT8 NOT NULL GENERATED ALWAYS AS IDENTITY, c INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY) -- literals removed
+CREATE TABLE _ (_ INT8 UNIQUE, _ INT8 NOT NULL GENERATED ALWAYS AS IDENTITY, _ INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY) -- identifiers removed
+
+parse
+CREATE TABLE generated_not_null (
+  a INT,
+  b INT NOT NULL GENERATED ALWAYS AS IDENTITY,
+  c INT NOT NULL GENERATED BY DEFAULT AS IDENTITY
+)
+----
+CREATE TABLE generated_not_null (a INT8, b INT8 NOT NULL GENERATED ALWAYS AS IDENTITY, c INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY) -- normalized!
+CREATE TABLE generated_not_null (a INT8, b INT8 NOT NULL GENERATED ALWAYS AS IDENTITY, c INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY) -- fully parenthesized
+CREATE TABLE generated_not_null (a INT8, b INT8 NOT NULL GENERATED ALWAYS AS IDENTITY, c INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY) -- literals removed
+CREATE TABLE _ (_ INT8, _ INT8 NOT NULL GENERATED ALWAYS AS IDENTITY, _ INT8 NOT NULL GENERATED BY DEFAULT AS IDENTITY) -- identifiers removed
+
+error
+CREATE TABLE generated_error (
+  a INT UNIQUE,
+  b INT GENERATED BY DEFAULT AS IDENTITY GENERATED ALWAYS AS IDENTITY
+)
+----
+at or near ")": syntax error: multiple identity specifications for column "b"
+DETAIL: source SQL:
+CREATE TABLE generated_error (
+  a INT UNIQUE,
+  b INT GENERATED BY DEFAULT AS IDENTITY GENERATED ALWAYS AS IDENTITY
+)
+^
+
+error
+CREATE TABLE generated_error (
+  a INT UNIQUE,
+  b INT NULL GENERATED BY DEFAULT AS IDENTITY
+)
+----
+at or near ")": syntax error: conflicting NULL/NOT NULL declarations for column "b"
+DETAIL: source SQL:
+CREATE TABLE generated_error (
+  a INT UNIQUE,
+  b INT NULL GENERATED BY DEFAULT AS IDENTITY
+)
+^
+
+error
+CREATE TABLE generated_error (
+  a INT,
+  b INT AS (a + 10) STORED GENERATED ALWAYS AS IDENTITY
+)
+----
+at or near ")": syntax error: both identity and generation expression specified for column "b"
+DETAIL: source SQL:
+CREATE TABLE generated_error (
+  a INT,
+  b INT AS (a + 10) STORED GENERATED ALWAYS AS IDENTITY
+)
+^
+
+error
+CREATE TABLE generated_error (
+  a INT,
+  b INT GENERATED ALWAYS AS (a + 10) STORED GENERATED ALWAYS AS IDENTITY
+)
+----
+at or near ")": syntax error: both identity and generation expression specified for column "b"
+DETAIL: source SQL:
+CREATE TABLE generated_error (
+  a INT,
+  b INT GENERATED ALWAYS AS (a + 10) STORED GENERATED ALWAYS AS IDENTITY
+)
+^
+
+error
+CREATE TABLE generated_error (
+  a INT,
+  b INT GENERATED ALWAYS AS (a + 10) IDENTITY
+)
+----
+at or near "identity": syntax error: use AS ( <expr> ) STORED or AS ( <expr> ) VIRTUAL
+DETAIL: source SQL:
+CREATE TABLE generated_error (
+  a INT,
+  b INT GENERATED ALWAYS AS (a + 10) IDENTITY
+                                     ^