opt: parser, test catalog, query support for virtual columns

docs/generated/sql/bnf/col_qualification.bnf

@@ -9,6 +9,8 @@ col_qualification ::=
 	| 'CONSTRAINT' constraint_name 'REFERENCES' table_name opt_name_parens key_match reference_actions
 	| 'CONSTRAINT' constraint_name 'AS' '(' a_expr ')' 'STORED'
 	| 'CONSTRAINT' constraint_name 'GENERATED_ALWAYS' 'ALWAYS' 'AS' '(' a_expr ')' 'STORED'
+	| 'CONSTRAINT' constraint_name 'AS' '(' a_expr ')' 'VIRTUAL'
+	| 'CONSTRAINT' constraint_name 'GENERATED_ALWAYS' 'ALWAYS' 'AS' '(' a_expr ')' 'VIRTUAL'
 	| 'NOT' 'NULL'
 	| 'NULL'
 	| 'UNIQUE' opt_without_index
@@ -19,6 +21,8 @@ col_qualification ::=
 	| 'REFERENCES' table_name opt_name_parens key_match reference_actions
 	| 'AS' '(' a_expr ')' 'STORED'
 	| 'GENERATED_ALWAYS' 'ALWAYS' 'AS' '(' a_expr ')' 'STORED'
+	| 'AS' '(' a_expr ')' 'VIRTUAL'
+	| 'GENERATED_ALWAYS' 'ALWAYS' 'AS' '(' a_expr ')' 'VIRTUAL'
 	| 'COLLATE' collation_name
 	| 'FAMILY' family_name
 	| 'CREATE' 'FAMILY' family_name

docs/generated/sql/bnf/stmt_block.bnf

@@ -2872,6 +2872,7 @@ col_qualification_elem ::=
 	| 'DEFAULT' b_expr
 	| 'REFERENCES' table_name opt_name_parens key_match reference_actions
 	| generated_as '(' a_expr ')' 'STORED'
+	| generated_as '(' a_expr ')' 'VIRTUAL'
 
 family_name ::=
 	name

pkg/sql/add_column.go

@@ -18,6 +18,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlerrors"
+	"github.com/cockroachdb/cockroach/pkg/util/errorutil/unimplemented"
 	"github.com/cockroachdb/errors"
 )
 
@@ -142,6 +143,9 @@ func (p *planner) addColumnImpl(
 	}
 
 	if d.IsComputed() {
+		if d.IsVirtual() {
+			return unimplemented.NewWithIssue(57608, "virtual computed columns")
+		}
 		computedColValidator := schemaexpr.MakeComputedColumnValidator(
 			params.ctx,
 			n.tableDesc,

pkg/sql/create_table.go

@@ -1297,6 +1297,10 @@ func NewTableDesc(
 				n.Defs = append(n.Defs, checkConstraint)
 				columnDefaultExprs = append(columnDefaultExprs, nil)
 			}
+			if d.IsVirtual() {
+				return nil, unimplemented.NewWithIssue(57608, "virtual computed columns")
+			}
+
 			col, idx, expr, err := tabledesc.MakeColumnDefDescs(ctx, d, semaCtx, evalCtx)
 			if err != nil {
 				return nil, err

pkg/sql/logictest/testdata/logic_test/computed

@@ -271,11 +271,20 @@ CREATE TABLE y (
   a INT AS (3)
 )
 
-statement error at or near "virtual": syntax error: unimplemented
+statement error unimplemented: virtual computed columns
 CREATE TABLE y (
   a INT AS (3) VIRTUAL
 )
 
+statement ok
+CREATE TABLE tmp (x INT)
+
+statement error unimplemented: virtual computed columns
+ALTER TABLE tmp ADD COLUMN y INT AS (x+1) VIRTUAL
+
+statement ok
+DROP TABLE tmp
+
 statement error expected computed column expression to have type int, but .* has type string
 CREATE TABLE y (
   a INT AS ('not an integer!'::STRING) STORED

pkg/sql/opt/cat/column.go

@@ -161,8 +161,7 @@ const (
 	// VirtualInverted columns are implicit columns that are used by inverted
 	// indexes.
 	VirtualInverted
-	// VirtualComputed columns are non-stored computed columns that are used by
-	// expression-based indexes.
+	// VirtualComputed columns are non-stored computed columns.
 	VirtualComputed
 )
 
@@ -228,15 +227,21 @@ func (c *Column) InitVirtualInverted(
 // InitVirtualComputed is used by catalog implementations to populate a
 // VirtualComputed Column. It should not be used anywhere else.
 func (c *Column) InitVirtualComputed(
-	ordinal int, name tree.Name, datumType *types.T, nullable bool, computedExpr string,
+	ordinal int,
+	stableID StableID,
+	name tree.Name,
+	datumType *types.T,
+	nullable bool,
+	hidden bool,
+	computedExpr string,
 ) {
 	c.ordinal = ordinal
-	c.stableID = 0
+	c.stableID = stableID
 	c.name = name
 	c.kind = VirtualComputed
 	c.datumType = datumType
 	c.nullable = nullable
-	c.hidden = true
+	c.hidden = hidden
 	c.defaultExpr = ""
 	c.computedExpr = computedExpr
 	c.invertedSourceColumnOrdinal = -1

pkg/sql/opt/cat/utils.go

@@ -306,7 +306,11 @@ func formatColumn(col *Column, buf *bytes.Buffer) {
 		fmt.Fprintf(buf, " not null")
 	}
 	if col.IsComputed() {
-		fmt.Fprintf(buf, " as (%s) stored", col.ComputedExprStr())
+		if col.Kind() == VirtualComputed {
+			fmt.Fprintf(buf, " as (%s) virtual", col.ComputedExprStr())
+		} else {
+			fmt.Fprintf(buf, " as (%s) stored", col.ComputedExprStr())
+		}
 	}
 	if col.HasDefault() {
 		fmt.Fprintf(buf, " default (%s)", col.DefaultExprStr())
@@ -322,7 +326,7 @@ func formatColumn(col *Column, buf *bytes.Buffer) {
 	case VirtualInverted:
 		fmt.Fprintf(buf, " [virtual-inverted]")
 	case VirtualComputed:
-		fmt.Fprintf(buf, " [virtual-computed]")
+		// No need to show anything more (it already shows up as virtual).
 	}
 }
 

pkg/sql/opt/optbuilder/select.go

@@ -117,7 +117,7 @@ func (b *Builder) buildDataSource(
 					includeMutations:       false,
 					includeSystem:          true,
 					includeVirtualInverted: false,
-					includeVirtualComputed: false,
+					includeVirtualComputed: true,
 				}),
 				indexFlags, locking, inScope,
 			)
@@ -401,7 +401,7 @@ func (b *Builder) buildScanFromTableRef(
 			includeMutations:       false,
 			includeSystem:          true,
 			includeVirtualInverted: false,
-			includeVirtualComputed: false,
+			includeVirtualComputed: true,
 		})
 	}
 
@@ -419,9 +419,14 @@ func (b *Builder) addTable(tab cat.Table, alias *tree.TableName) *opt.TableMeta
 	return md.TableMeta(tabID)
 }
 
-// buildScan builds a memo group for a ScanOp expression on the given table.
+// buildScan builds a memo group for a ScanOp expression on the given table. If
+// the ordinals list contains any VirtualComputed columns, a ProjectOp is built
+// on top.
 //
-// The scan projects the given table ordinals.
+// The resulting scope and expression output the given table ordinals. If an
+// ordinal is for a VirtualComputed column, the ordinals it depends on must also
+// be in the list (in practice, this coincides with all "ordinary" table columns
+// being in the list).
 //
 // If scanMutationCols is true, then include columns being added or dropped from
 // the table. These are currently required by the execution engine as "fetch
@@ -453,20 +458,26 @@ func (b *Builder) buildScan(
 
 	outScope = inScope.push()
 
-	var tabColIDs opt.ColSet
+	// We collect VirtualComputed columns separately; these cannot be scanned,
+	// they can only be projected afterward.
+	var tabColIDs, virtualColIDs opt.ColSet
 	outScope.cols = make([]scopeColumn, len(ordinals))
 	for i, ord := range ordinals {
 		col := tab.Column(ord)
 		colID := tabID.ColumnID(ord)
-		tabColIDs.Add(colID)
 		name := col.ColName()
 		kind := col.Kind()
+		if kind != cat.VirtualComputed {
+			tabColIDs.Add(colID)
+		} else {
+			virtualColIDs.Add(colID)
+		}
 		outScope.cols[i] = scopeColumn{
 			id:           colID,
 			name:         name,
 			table:        tabMeta.Alias,
 			typ:          col.DatumType(),
-			hidden:       col.IsHidden() || kind != cat.Ordinary,
+			hidden:       col.IsHidden() || (kind != cat.Ordinary && kind != cat.VirtualComputed),
 			kind:         kind,
 			mutation:     kind == cat.WriteOnly || kind == cat.DeleteOnly,
 			tableOrdinal: ord,
@@ -485,79 +496,96 @@ func (b *Builder) buildScan(
 		private := memo.ScanPrivate{Table: tabID, Cols: tabColIDs}
 		outScope.expr = b.factory.ConstructScan(&private)
 
-		// Virtual tables should not be collected as view dependencies.
-	} else {
-		private := memo.ScanPrivate{Table: tabID, Cols: tabColIDs}
-		if indexFlags != nil {
-			private.Flags.NoIndexJoin = indexFlags.NoIndexJoin
-			if indexFlags.Index != "" || indexFlags.IndexID != 0 {
-				idx := -1
-				for i := 0; i < tab.IndexCount(); i++ {
-					if tab.Index(i).Name() == tree.Name(indexFlags.Index) ||
-						tab.Index(i).ID() == cat.StableID(indexFlags.IndexID) {
-						idx = i
-						break
-					}
+		// Note: virtual tables should not be collected as view dependencies.
+		return outScope
+	}
+
+	private := memo.ScanPrivate{Table: tabID, Cols: tabColIDs}
+	if indexFlags != nil {
+		private.Flags.NoIndexJoin = indexFlags.NoIndexJoin
+		if indexFlags.Index != "" || indexFlags.IndexID != 0 {
+			idx := -1
+			for i := 0; i < tab.IndexCount(); i++ {
+				if tab.Index(i).Name() == tree.Name(indexFlags.Index) ||
+					tab.Index(i).ID() == cat.StableID(indexFlags.IndexID) {
+					idx = i
+					break
 				}
-				if idx == -1 {
-					var err error
-					if indexFlags.Index != "" {
-						err = errors.Errorf("index %q not found", tree.ErrString(&indexFlags.Index))
-					} else {
-						err = errors.Errorf("index [%d] not found", indexFlags.IndexID)
-					}
-					panic(err)
+			}
+			if idx == -1 {
+				var err error
+				if indexFlags.Index != "" {
+					err = errors.Errorf("index %q not found", tree.ErrString(&indexFlags.Index))
+				} else {
+					err = errors.Errorf("index [%d] not found", indexFlags.IndexID)
 				}
-				private.Flags.ForceIndex = true
-				private.Flags.Index = idx
-				private.Flags.Direction = indexFlags.Direction
+				panic(err)
 			}
+			private.Flags.ForceIndex = true
+			private.Flags.Index = idx
+			private.Flags.Direction = indexFlags.Direction
 		}
-		if locking.isSet() {
-			private.Locking = locking.get()
-		}
-
-		b.addCheckConstraintsForTable(tabMeta)
-		b.addComputedColsForTable(tabMeta)
+	}
+	if locking.isSet() {
+		private.Locking = locking.get()
+	}
 
-		outScope.expr = b.factory.ConstructScan(&private)
+	b.addCheckConstraintsForTable(tabMeta)
+	b.addComputedColsForTable(tabMeta)
+
+	outScope.expr = b.factory.ConstructScan(&private)
+
+	if !virtualColIDs.Empty() {
+		// Project the expressions for the virtual columns (and pass through all
+		// scanned columns).
+		// TODO(radu): we don't currently support virtual columns depending on other
+		// virtual columns.
+		proj := make(memo.ProjectionsExpr, 0, virtualColIDs.Len())
+		virtualColIDs.ForEach(func(col opt.ColumnID) {
+			item := b.factory.ConstructProjectionsItem(tabMeta.ComputedCols[col], col)
+			if !item.ScalarProps().OuterCols.SubsetOf(tabColIDs) {
+				panic(errors.AssertionFailedf("scanned virtual column depends on non-scanned column"))
+			}
+			proj = append(proj, item)
+		})
+		outScope.expr = b.factory.ConstructProject(outScope.expr, proj, tabColIDs)
+	}
 
-		// Add the partial indexes after constructing the scan so we can use the
-		// logical properties of the scan to fully normalize the index
-		// predicates. Partial index predicates are only added if the outScope
-		// contains all the table's ordinary columns. If it does not, partial
-		// index predicates cannot be built because they may reference columns
-		// not in outScope. In the most common case, the outScope has the same
-		// number of columns as the table and we can skip checking that each
-		// ordinary column exists in outScope.
-		containsAllOrdinaryTableColumns := true
-		if len(outScope.cols) != tab.ColumnCount() {
-			for i := 0; i < tab.ColumnCount(); i++ {
-				col := tab.Column(i)
-				if col.Kind() == cat.Ordinary && !outScope.colSet().Contains(tabID.ColumnID(col.Ordinal())) {
-					containsAllOrdinaryTableColumns = false
-					break
-				}
+	// Add the partial indexes after constructing the scan so we can use the
+	// logical properties of the scan to fully normalize the index
+	// predicates. Partial index predicates are only added if the outScope
+	// contains all the table's ordinary columns. If it does not, partial
+	// index predicates cannot be built because they may reference columns
+	// not in outScope. In the most common case, the outScope has the same
+	// number of columns as the table and we can skip checking that each
+	// ordinary column exists in outScope.
+	containsAllOrdinaryTableColumns := true
+	if len(outScope.cols) != tab.ColumnCount() {
+		for i := 0; i < tab.ColumnCount(); i++ {
+			col := tab.Column(i)
+			if col.Kind() == cat.Ordinary && !outScope.colSet().Contains(tabID.ColumnID(col.Ordinal())) {
+				containsAllOrdinaryTableColumns = false
+				break
 			}
 		}
-		if containsAllOrdinaryTableColumns {
-			b.addPartialIndexPredicatesForTable(tabMeta, outScope)
-		}
+	}
+	if containsAllOrdinaryTableColumns {
+		b.addPartialIndexPredicatesForTable(tabMeta, outScope)
+	}
 
-		if b.trackViewDeps {
-			dep := opt.ViewDep{DataSource: tab}
-			dep.ColumnIDToOrd = make(map[opt.ColumnID]int)
-			// We will track the ColumnID to Ord mapping so Ords can be added
-			// when a column is referenced.
-			for i, col := range outScope.cols {
-				dep.ColumnIDToOrd[col.id] = ordinals[i]
-			}
-			if private.Flags.ForceIndex {
-				dep.SpecificIndex = true
-				dep.Index = private.Flags.Index
-			}
-			b.viewDeps = append(b.viewDeps, dep)
+	if b.trackViewDeps {
+		dep := opt.ViewDep{DataSource: tab}
+		dep.ColumnIDToOrd = make(map[opt.ColumnID]int)
+		// We will track the ColumnID to Ord mapping so Ords can be added
+		// when a column is referenced.
+		for i, col := range outScope.cols {
+			dep.ColumnIDToOrd[col.id] = ordinals[i]
+		}
+		if private.Flags.ForceIndex {
+			dep.SpecificIndex = true
+			dep.Index = private.Flags.Index
 		}
+		b.viewDeps = append(b.viewDeps, dep)
 	}
 	return outScope
 }

pkg/sql/opt/optbuilder/testdata/virtual-columns

@@ -0,0 +1,37 @@
+exec-ddl
+CREATE TABLE t (
+  a INT PRIMARY KEY,
+  b INT,
+  c INT AS (a+b) VIRTUAL
+)
+----
+
+build
+SELECT * FROM t
+----
+project
+ ├── columns: a:1!null b:2 c:3
+ └── project
+      ├── columns: c:3 a:1!null b:2 crdb_internal_mvcc_timestamp:4
+      ├── scan t
+      │    ├── columns: a:1!null b:2 crdb_internal_mvcc_timestamp:4
+      │    └── computed column expressions
+      │         └── c:3
+      │              └── a:1 + b:2
+      └── projections
+           └── a:1 + b:2 [as=c:3]
+
+build
+SELECT c FROM t
+----
+project
+ ├── columns: c:3
+ └── project
+      ├── columns: c:3 a:1!null b:2 crdb_internal_mvcc_timestamp:4
+      ├── scan t
+      │    ├── columns: a:1!null b:2 crdb_internal_mvcc_timestamp:4
+      │    └── computed column expressions
+      │         └── c:3
+      │              └── a:1 + b:2
+      └── projections
+           └── a:1 + b:2 [as=c:3]