sql: enable enum values to be stored in tables and used in queries

This PR enables the use of ENUM types in SQL queries and tables.

Release note (sql change): This PR enables the use of Postgres
ENUM types. It allows ENUMs to be used in queries and stored
in tables.

docs/generated/sql/bnf/stmt_block.bnf

@@ -1587,6 +1587,7 @@ case_expr ::=
 
 simple_typename ::=
 	general_type_name
+	| '@' iconst32
 	| complex_type_name
 	| const_typename
 	| bit_with_length
@@ -1925,6 +1926,9 @@ case_default ::=
 general_type_name ::=
 	type_function_name_no_crdb_extra
 
+iconst32 ::=
+	'ICONST'
+
 complex_type_name ::=
 	general_type_name '.' unrestricted_name
 	| general_type_name '.' unrestricted_name '.' unrestricted_name
@@ -2163,9 +2167,6 @@ opt_interval_qualifier ::=
 	interval_qualifier
 	| 
 
-iconst32 ::=
-	'ICONST'
-
 func_application ::=
 	func_name '(' ')'
 	| func_name '(' expr_list opt_sort_clause ')'

pkg/ccl/backupccl/backup_planning.go

@@ -34,6 +34,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/types"
 	"github.com/cockroachdb/cockroach/pkg/storage/cloud"
 	"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
+	"github.com/cockroachdb/cockroach/pkg/util/errorutil/unimplemented"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/interval"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
@@ -356,6 +357,13 @@ func backupPlanHook(
 				}
 				tables = append(tables, tableDesc)
 
+				// If the table has any user defined types, error out.
+				for _, col := range tableDesc.Columns {
+					if col.Type.UserDefined() {
+						return unimplemented.NewWithIssue(48689, "user defined types in backup")
+					}
+				}
+
 				// Collect all the table stats for this table.
 				tableStatisticsAcc, err := statsCache.GetTableStats(ctx, tableDesc.GetID())
 				if err != nil {

pkg/ccl/logictestccl/testdata/logic_test/restore

@@ -3,3 +3,11 @@
 # Check that we get through parsing and license check.
 statement error pq: failed to open backup storage location: unsupported storage scheme: ""
 RESTORE foo FROM "bar"
+
+# Check that user defined types are disallowed in backups.
+statement ok
+CREATE TYPE t AS ENUM ('hello');
+CREATE TABLE tt (x t)
+
+statement error pq: unimplemented: user defined types in backup
+BACKUP TABLE tt TO ""

pkg/server/server_sql.go

@@ -412,6 +412,7 @@ func newSQLServer(ctx context.Context, cfg sqlServerArgs) (*sqlServer, error) {
 			cfg.gossip,
 			cfg.db,
 			cfg.circularInternalExecutor,
+			codec,
 		),
 
 		// Note: don't forget to add the secondary loggers as closers

pkg/sql/create_stats.go

@@ -397,6 +397,10 @@ func (r *createStatsResumer) Resume(
 
 	dsp := p.DistSQLPlanner()
 	if err := p.ExecCfg().DB.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+		// Set the transaction on the EvalContext to this txn. This allows for
+		// use of the txn during processor setup during the execution of the flow.
+		evalCtx.Txn = txn
+
 		if details.AsOf != nil {
 			p.semaCtx.AsOfTimestamp = details.AsOf
 			p.extendedEvalCtx.SetTxnTimestamp(details.AsOf.GoTime())

pkg/sql/execinfra/expr.go

@@ -158,6 +158,7 @@ func (eh *ExprHelper) Init(
 	}
 	var err error
 	semaContext := tree.MakeSemaContext()
+	semaContext.TypeResolver = evalCtx.DistSQLTypeResolver
 	eh.Expr, err = processExpression(expr, evalCtx, &semaContext, &eh.Vars)
 	if err != nil {
 		return err

pkg/sql/execinfra/flow_context.go

@@ -70,7 +70,9 @@ type FlowCtx struct {
 // them at runtime to ensure expressions are evaluated with the correct indexed
 // var context.
 func (ctx *FlowCtx) NewEvalCtx() *tree.EvalContext {
-	return ctx.EvalCtx.Copy()
+	evalCopy := ctx.EvalCtx.Copy()
+	evalCopy.DistSQLTypeResolver = &execinfrapb.DistSQLTypeResolver{EvalContext: evalCopy}
+	return evalCopy
 }
 
 // TestingKnobs returns the distsql testing knobs for this flow context.

pkg/sql/execinfra/processorsbase.go

@@ -796,6 +796,12 @@ func (pb *ProcessorBase) InitWithEvalCtx(
 	pb.MemMonitor = memMonitor
 	pb.trailingMetaCallback = opts.TrailingMetaCallback
 	pb.inputsToDrain = opts.InputsToDrain
+
+	// Hydrate all types used in the processor.
+	if err := execinfrapb.HydrateTypeSlice(evalCtx, types); err != nil {
+		return err
+	}
+
 	return pb.Out.Init(post, types, pb.EvalCtx, output)
 }
 

pkg/sql/execinfrapb/data.go

@@ -18,6 +18,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
+	"github.com/cockroachdb/cockroach/pkg/sql/types"
 	"github.com/cockroachdb/cockroach/pkg/util/encoding"
 	"github.com/cockroachdb/cockroach/pkg/util/tracing"
 	"github.com/cockroachdb/errors"
@@ -71,11 +72,73 @@ func ConvertToMappedSpecOrdering(
 	return specOrdering
 }
 
+// DistSQLTypeResolver implements tree.ResolvableTypeReference for accessing
+// type information during DistSQL query evaluation.
+type DistSQLTypeResolver struct {
+	EvalContext *tree.EvalContext
+	// TODO (rohany): This struct should locally cache id -> types.T here
+	//  so that repeated lookups do not incur additional KV operations.
+}
+
+// ResolveType implements tree.ResolvableTypeReference.
+func (tr *DistSQLTypeResolver) ResolveType(name *tree.UnresolvedObjectName) (*types.T, error) {
+	return nil, errors.AssertionFailedf("cannot resolve types in DistSQL by name")
+}
+
+// ResolveTypeByID implements tree.ResolvableTypeReference.
+func (tr *DistSQLTypeResolver) ResolveTypeByID(id uint32) (*types.T, error) {
+	// TODO (rohany): This should eventually look into the set of cached type
+	//  descriptors before attempting to access it here.
+	typDesc, err := sqlbase.GetTypeDescFromID(
+		tr.EvalContext.Context,
+		tr.EvalContext.Txn,
+		tr.EvalContext.Codec,
+		sqlbase.ID(id),
+	)
+	if err != nil {
+		return nil, err
+	}
+	var typ *types.T
+	switch t := typDesc.Kind; t {
+	case sqlbase.TypeDescriptor_ENUM:
+		typ = types.MakeEnum(id)
+	default:
+		return nil, errors.AssertionFailedf("unknown type kind %s", t)
+	}
+	if err := typDesc.HydrateTypeInfo(typ); err != nil {
+		return nil, err
+	}
+	return typ, nil
+}
+
+// HydrateTypeSlice hydrates all user defined types in an input slice of types.
+func HydrateTypeSlice(evalCtx *tree.EvalContext, typs []*types.T) error {
+	for _, t := range typs {
+		if t.UserDefined() {
+			// TODO (rohany): This should eventually look into the set of cached type
+			//  descriptors before attempting to access it here.
+			typDesc, err := sqlbase.GetTypeDescFromID(
+				evalCtx.Context,
+				evalCtx.Txn,
+				evalCtx.Codec,
+				sqlbase.ID(t.StableTypeID()),
+			)
+			if err != nil {
+				return err
+			}
+			if err := typDesc.HydrateTypeInfo(t); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 // ExprFmtCtxBase produces a FmtCtx used for serializing expressions; a proper
 // IndexedVar formatting function needs to be added on. It replaces placeholders
 // with their values.
 func ExprFmtCtxBase(evalCtx *tree.EvalContext) *tree.FmtCtx {
-	fmtCtx := tree.NewFmtCtx(tree.FmtCheckEquivalence)
+	fmtCtx := tree.NewFmtCtx(tree.FmtDistSQLSerialization)
 	fmtCtx.SetPlaceholderFormat(
 		func(fmtCtx *tree.FmtCtx, p *tree.Placeholder) {
 			d, err := p.Eval(evalCtx)
@@ -115,7 +178,7 @@ func (e *Expression) Empty() bool {
 // String implements the Stringer interface.
 func (e Expression) String() string {
 	if e.LocalExpr != nil {
-		ctx := tree.NewFmtCtx(tree.FmtCheckEquivalence)
+		ctx := tree.NewFmtCtx(tree.FmtDistSQLSerialization)
 		ctx.FormatNode(e.LocalExpr)
 		return ctx.CloseAndGetString()
 	}

pkg/sql/logictest/testdata/logic_test/enums

@@ -145,9 +145,6 @@ true true
 statement error pq: invalid input value for enum greeting: "notagreeting"
 SELECT 'hello'::greeting = 'notagreeting'
 
-statement error pq: value type greeting cannot be used for table columns
-CREATE TABLE bad (x greeting)
-
 statement error pq: unimplemented: ALTER TYPE ADD VALUE unsupported
 ALTER TYPE greeting ADD VALUE 'hola' AFTER 'hello'
 
@@ -199,3 +196,166 @@ SELECT enum_range(NULL::dbs, NULL::dbs)
 
 query error pq: enum_range\(\): mismatched types
 SELECT enum_range('cockroach'::dbs, 'hello'::greeting)
+
+# Test inserting and reading enum data from tables.
+statement ok
+CREATE TABLE greeting_table (x1 greeting, x2 greeting)
+
+statement error pq: invalid input value for enum greeting: "bye"
+INSERT INTO greeting_table VALUES ('bye', 'hi')
+
+statement ok
+INSERT INTO greeting_table VALUES ('hi', 'hello')
+
+query TT
+SELECT * FROM greeting_table
+----
+hi hello
+
+query TT
+SELECT 'hello'::greeting, x1 FROM greeting_table
+----
+hello hi
+
+query TB
+SELECT x1, x1 < 'hello' FROM greeting_table
+----
+hi false
+
+query TT
+SELECT x1, enum_first(x1) FROM greeting_table
+----
+hi hello
+
+statement ok
+CREATE TABLE t1 (x greeting, INDEX i (x));
+CREATE TABLE t2 (x greeting, INDEX i (x));
+INSERT INTO t1 VALUES ('hello');
+INSERT INTO t2 VALUES ('hello')
+
+query TT
+SELECT * FROM t1 INNER LOOKUP JOIN t2 ON t1.x = t2.x
+----
+hello hello
+
+query TT
+SELECT * FROM t1 INNER HASH JOIN t2 ON t1.x = t2.x
+----
+hello hello
+
+query TT
+SELECT * FROM t1 INNER MERGE JOIN t2 ON t1.x = t2.x
+----
+hello hello
+
+statement ok
+INSERT INTO t2 VALUES ('hello'), ('hello'), ('howdy'), ('hi')
+
+query T rowsort
+SELECT DISTINCT x FROM t2
+----
+hello
+howdy
+hi
+
+query T
+SELECT DISTINCT x FROM t2 ORDER BY x DESC
+----
+hi
+howdy
+hello
+
+# Test out some subqueries.
+query T rowsort
+SELECT x FROM t2 WHERE x > (SELECT x FROM t1 ORDER BY x LIMIT 1)
+----
+hi
+howdy
+
+# Test ordinality.
+query TI
+SELECT * FROM t2 WITH ORDINALITY ORDER BY x
+----
+hello  1
+hello  2
+hello  3
+howdy  4
+hi     5
+
+# Test ordering with and without limits.
+statement ok
+INSERT INTO t1 VALUES ('hi'), ('hello'), ('howdy'), ('howdy'), ('howdy'), ('hello')
+
+query T
+SELECT x FROM t1 ORDER BY x DESC
+----
+hi
+howdy
+howdy
+howdy
+hello
+hello
+hello
+
+query T
+SELECT x FROM t1 ORDER BY x ASC
+----
+hello
+hello
+hello
+howdy
+howdy
+howdy
+hi
+
+query T
+SELECT x FROM t1 ORDER BY x ASC LIMIT 3
+----
+hello
+hello
+hello
+
+query T
+SELECT x FROM t1 ORDER BY x DESC LIMIT 3
+----
+hi
+howdy
+howdy
+
+# Test we can group on enums.
+query T rowsort
+(SELECT * FROM t1) UNION (SELECT * FROM t2)
+----
+hello
+howdy
+hi
+
+statement ok
+CREATE TABLE enum_agg (x greeting, y INT);
+INSERT INTO enum_agg VALUES
+  ('hello', 1),
+  ('hello', 3),
+  ('howdy', 5),
+  ('howdy', 0),
+  ('howdy', 1),
+  ('hi', 10)
+
+query TIRI rowsort
+SELECT x, max(y), sum(y), min(y) FROM enum_agg GROUP BY x
+----
+hello 3 4 1
+howdy 5 6 0
+hi 10 10 10
+
+
+# Regression to ensure that statistics jobs can be run on tables
+# with user defined types.
+statement ok
+CREATE TABLE greeting_stats (x greeting PRIMARY KEY);
+INSERT INTO greeting_stats VALUES ('hi');
+CREATE STATISTICS s FROM greeting_stats
+
+query T
+SELECT x FROM greeting_stats
+----
+hi