Merge #38904 #38913 #38972

38904: parser: Add user defined column families to CTAS. r=adityamaru27 a=adityamaru27

This change introduces grammar to support user defined
column families in a CREATE TABLE ... AS query.

38913: deps: bump go.etcd.io/etcd r=tbg a=danhhz

To pick up etcd-io/etcd#10864.

Release note: None

38972: exec: fix calculation of selectivity stat when num batches is zero r=yuzefovich a=yuzefovich

Release note: None

Co-authored-by: Aditya Maru <[email protected]>
Co-authored-by: Tobias Schottdorf <[email protected]>
Co-authored-by: Daniel Harrison <[email protected]>
Co-authored-by: Yahor Yuzefovich <[email protected]>

docs/generated/sql/bnf/stmt_block.bnf

@@ -1694,7 +1694,7 @@ partition_by ::=
 	| 'PARTITION' 'BY' 'NOTHING'
 
 create_as_table_defs ::=
-	( column_name create_as_col_qual_list ) ( ( ',' create_as_constraint_def | ',' column_name create_as_col_qual_list ) )*
+	( column_name create_as_col_qual_list ) ( ( ',' column_name create_as_col_qual_list | ',' family_def | ',' create_as_constraint_def ) )*
 
 common_table_expr ::=
 	table_alias_name opt_column_list 'AS' '(' preparable_stmt ')'
@@ -2136,6 +2136,7 @@ range_partition ::=
 
 create_as_col_qualification ::=
 	create_as_col_qualification_elem
+	| 'FAMILY' family_name
 
 create_as_constraint_elem ::=
 	'PRIMARY' 'KEY' '(' create_as_params ')'

pkg/roachpb/data_test.go

@@ -28,10 +28,13 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/duration"
 	"github.com/cockroachdb/cockroach/pkg/util/encoding"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
+	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
 	"github.com/cockroachdb/cockroach/pkg/util/randutil"
 	"github.com/cockroachdb/cockroach/pkg/util/uuid"
+	"github.com/gogo/protobuf/proto"
 	"github.com/kr/pretty"
+	"github.com/stretchr/testify/require"
 )
 
 func makeTS(walltime int64, logical int32) hlc.Timestamp {
@@ -1559,3 +1562,31 @@ func TestUpdateObservedTimestamps(t *testing.T) {
 		})
 	}
 }
+
+func TestChangeReplicasTrigger_String(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+
+	l := ReplicaType_LEARNER
+	v := ReplicaType_VOTER
+	repl := ReplicaDescriptor{NodeID: 1, StoreID: 2, ReplicaID: 3, Type: &l}
+	crt := ChangeReplicasTrigger{
+		ChangeType: ADD_REPLICA,
+		Replica:    repl,
+		Desc: &RangeDescriptor{
+			RangeID:  1,
+			StartKey: RKey("a"),
+			EndKey:   RKey("b"),
+			InternalReplicas: []ReplicaDescriptor{
+				repl,
+				{NodeID: 4, StoreID: 5, ReplicaID: 6, Type: &v},
+				{NodeID: 7, StoreID: 8, ReplicaID: 9, Type: &l},
+			},
+			NextReplicaID:        10,
+			Generation:           proto.Int64(5),
+			GenerationComparable: proto.Bool(true),
+		},
+	}
+	act := crt.String()
+	exp := `ADD_REPLICA((n1,s2):3LEARNER): updated=(n4,s5):6,(n1,s2):3LEARNER,(n7,s8):9LEARNER next=10`
+	require.Equal(t, exp, act)
+}

pkg/roachpb/metadata_replicas.go

@@ -10,7 +10,11 @@
 
 package roachpb
 
-import "sort"
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
 
 // ReplicaDescriptors is a set of replicas, usually the nodes/stores on which
 // replicas of a range are stored.
@@ -29,6 +33,17 @@ func MakeReplicaDescriptors(replicas []ReplicaDescriptor) ReplicaDescriptors {
 	return ReplicaDescriptors{wrapped: replicas}
 }
 
+func (d ReplicaDescriptors) String() string {
+	var buf strings.Builder
+	for i, desc := range d.wrapped {
+		if i > 0 {
+			buf.WriteByte(',')
+		}
+		fmt.Fprint(&buf, desc)
+	}
+	return buf.String()
+}
+
 // Unwrap returns every replica in the set. It is a placeholder for code that
 // used to work on a slice of replicas until learner replicas are added. At that
 // point, all uses of Unwrap will be migrated to All/Voters/Learners.

pkg/server/status.go

@@ -1205,7 +1205,7 @@ func (s *statusServer) Ranges(
 			state.Progress[id] = serverpb.RaftState_Progress{
 				Match:           progress.Match,
 				Next:            progress.Next,
-				Paused:          progress.Paused,
+				Paused:          progress.IsPaused(),
 				PendingSnapshot: progress.PendingSnapshot,
 				State:           progress.State.String(),
 			}

pkg/sql/execpb/stats.go

@@ -38,10 +38,14 @@ func (vs *VectorizedStats) Stats() map[string]string {
 	} else {
 		timeSuffix = executionTimeTagSuffix
 	}
+	selectivity := float64(0)
+	if vs.NumBatches > 0 {
+		selectivity = float64(vs.NumTuples) / float64(coldata.BatchSize*vs.NumBatches)
+	}
 	return map[string]string{
 		batchesOutputTagSuffix: fmt.Sprintf("%d", vs.NumBatches),
 		tuplesOutputTagSuffix:  fmt.Sprintf("%d", vs.NumTuples),
-		selectivityTagSuffix:   fmt.Sprintf("%.2f", float64(vs.NumTuples)/float64(coldata.BatchSize*vs.NumBatches)),
+		selectivityTagSuffix:   fmt.Sprintf("%.2f", selectivity),
 		timeSuffix:             fmt.Sprintf("%v", vs.Time.Round(time.Microsecond)),
 	}
 }
@@ -62,10 +66,14 @@ func (vs *VectorizedStats) StatsForQueryPlan() []string {
 	} else {
 		timeSuffix = executionTimeQueryPlanSuffix
 	}
+	selectivity := float64(0)
+	if vs.NumBatches > 0 {
+		selectivity = float64(vs.NumTuples) / float64(coldata.BatchSize*vs.NumBatches)
+	}
 	return []string{
 		fmt.Sprintf("%s: %d", batchesOutputQueryPlanSuffix, vs.NumBatches),
 		fmt.Sprintf("%s: %d", tuplesOutputQueryPlanSuffix, vs.NumTuples),
-		fmt.Sprintf("%s: %.2f", selectivityQueryPlanSuffix, float64(vs.NumTuples)/float64(coldata.BatchSize*vs.NumBatches)),
+		fmt.Sprintf("%s: %.2f", selectivityQueryPlanSuffix, selectivity),
 		fmt.Sprintf("%s: %v", timeSuffix, vs.Time.Round(time.Microsecond)),
 	}
 }

pkg/sql/logictest/testdata/logic_test/create_as

@@ -236,3 +236,65 @@ foo11  CREATE TABLE foo11 (
 statement error pq: multiple primary keys for table "foo12" are not allowed
 CREATE TABLE foo12 (x PRIMARY KEY, y, PRIMARY KEY(y)) AS VALUES (1, 2), (3, 4);
 
+# Check that CREATE TABLE AS allows users to specify column families.
+statement ok
+CREATE TABLE abcd(
+  a INT PRIMARY KEY,
+  b INT,
+  c INT,
+  d INT
+);
+
+# Test column qualifiers to define column families.
+statement ok
+CREATE TABLE foo12 (a PRIMARY KEY FAMILY f1, b, c FAMILY fam_1_c, d) AS SELECT * FROM abcd;
+
+query TT
+SHOW CREATE TABLE foo12
+----
+foo12  CREATE TABLE foo12 (
+      a INT8 NOT NULL,
+      b INT8 NULL,
+      c INT8 NULL,
+      d INT8 NULL,
+      CONSTRAINT "primary" PRIMARY KEY (a ASC),
+      FAMILY f1 (a, b, d),
+      FAMILY fam_1_c (c)
+)
+
+# Test constraint style definition of column families.
+statement ok
+CREATE TABLE foo13 (a, b, c, d, PRIMARY KEY(a, b), FAMILY pk (a, b), FAMILY (c, d)) AS SELECT * FROM abcd;
+
+query TT
+SHOW CREATE TABLE foo13
+----
+foo13  CREATE TABLE foo13 (
+      a INT8 NOT NULL,
+      b INT8 NOT NULL,
+      c INT8 NULL,
+      d INT8 NULL,
+      CONSTRAINT "primary" PRIMARY KEY (a ASC, b ASC),
+      FAMILY pk (a, b),
+      FAMILY fam_1_c_d (c, d)
+)
+
+# Test renaming columns still preserves the column families.
+statement ok
+ALTER TABLE foo13 RENAME d TO z
+
+statement ok
+ALTER TABLE foo13 RENAME c TO e
+
+query TT
+SHOW CREATE TABLE foo13
+----
+foo13  CREATE TABLE foo13 (
+            a INT8 NOT NULL,
+            b INT8 NOT NULL,
+            e INT8 NULL,
+            z INT8 NULL,
+            CONSTRAINT "primary" PRIMARY KEY (a ASC, b ASC),
+            FAMILY pk (a, b),
+            FAMILY fam_1_c_d (e, z)
+)

pkg/sql/parser/parse_test.go

@@ -257,6 +257,11 @@ func TestParse(t *testing.T) {
 		{`CREATE TABLE IF NOT EXISTS a (z PRIMARY KEY) AS SELECT * FROM b`},
 		{`CREATE TABLE a (x, y, z, PRIMARY KEY (x, y, z)) AS SELECT * FROM b`},
 		{`CREATE TABLE IF NOT EXISTS a (x, y, z, PRIMARY KEY (x, y, z)) AS SELECT * FROM b`},
+		{`CREATE TABLE a (x, FAMILY (x)) AS SELECT * FROM b`},
+		{`CREATE TABLE IF NOT EXISTS a (x, FAMILY (x)) AS SELECT * FROM b`},
+		{`CREATE TABLE a (x, y FAMILY f1) AS SELECT * FROM b`},
+		{`CREATE TABLE IF NOT EXISTS a (x, y FAMILY f1) AS SELECT * FROM b`},
+
 		{`CREATE TABLE a (b STRING COLLATE de)`},
 		{`CREATE TABLE a (b STRING(3) COLLATE de)`},
 		{`CREATE TABLE a (b STRING[] COLLATE de)`},

pkg/sql/parser/sql.y

@@ -4443,11 +4443,6 @@ create_as_table_defs:
     var colToTableDef tree.TableDef = tableDef
     $$.val = tree.TableDefs{colToTableDef}
   }
-| create_as_table_defs ',' create_as_constraint_def
-  {
-    var constraintToTableDef tree.TableDef = $3.constraintDef()
-    $$.val = append($1.tblDefs(), constraintToTableDef)
-  }
 | create_as_table_defs ',' column_name create_as_col_qual_list
   {
     tableDef, err := tree.NewColumnTableDef(tree.Name($3), nil, false, $4.colQuals())
@@ -4459,6 +4454,15 @@ create_as_table_defs:
 
     $$.val = append($1.tblDefs(), colToTableDef)
   }
+| create_as_table_defs ',' family_def
+  {
+    $$.val = append($1.tblDefs(), $3.tblDef())
+  }
+| create_as_table_defs ',' create_as_constraint_def
+{
+  var constraintToTableDef tree.TableDef = $3.constraintDef()
+  $$.val = append($1.tblDefs(), constraintToTableDef)
+}
 
 create_as_constraint_def:
   create_as_constraint_elem
@@ -4508,6 +4512,10 @@ create_as_col_qualification:
   {
     $$.val = tree.NamedColumnQualification{Qualification: $1.colQualElem()}
   }
+| FAMILY family_name
+  {
+    $$.val = tree.NamedColumnQualification{Qualification: &tree.ColumnFamilyConstraint{Family: tree.Name($2)}}
+  }
 
 create_as_col_qualification_elem:
   PRIMARY KEY

pkg/storage/allocator.go

@@ -27,6 +27,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
 	"github.com/pkg/errors"
 	"go.etcd.io/etcd/raft"
+	"go.etcd.io/etcd/raft/tracker"
 )
 
 const (
@@ -1196,7 +1197,7 @@ func replicaIsBehind(raftStatus *raft.Status, replicaID roachpb.ReplicaID) bool
 	// behind the actual commit index of the range.
 	if progress, ok := raftStatus.Progress[uint64(replicaID)]; ok {
 		if uint64(replicaID) == raftStatus.Lead ||
-			(progress.State == raft.ProgressStateReplicate &&
+			(progress.State == tracker.StateReplicate &&
 				progress.Match >= raftStatus.Commit) {
 			return false
 		}
@@ -1214,8 +1215,8 @@ func simulateFilterUnremovableReplicas(
 	brandNewReplicaID roachpb.ReplicaID,
 ) []roachpb.ReplicaDescriptor {
 	status := *raftStatus
-	status.Progress[uint64(brandNewReplicaID)] = raft.Progress{
-		State: raft.ProgressStateReplicate,
+	status.Progress[uint64(brandNewReplicaID)] = tracker.Progress{
+		State: tracker.StateReplicate,
 		Match: status.Commit,
 	}
 	return filterUnremovableReplicas(&status, replicas, brandNewReplicaID)