sql: update EXPERIMENTAL_RELOCATE to support relocation of non-voters

docs/generated/sql/bnf/stmt_block.bnf

@@ -968,6 +968,7 @@ unreserved_keyword ::=
 	| 'NOCONTROLJOB'
 	| 'NOLOGIN'
 	| 'NOMODIFYCLUSTERSETTING'
+	| 'NON_VOTERS'
 	| 'NOVIEWACTIVITY'
 	| 'NOWAIT'
 	| 'NULLS'
@@ -1120,6 +1121,7 @@ unreserved_keyword ::=
 	| 'VARYING'
 	| 'VIEW'
 	| 'VIEWACTIVITY'
+	| 'VOTERS'
 	| 'WITHIN'
 	| 'WITHOUT'
 	| 'WRITE'

pkg/ccl/kvccl/kvfollowerreadsccl/followerreads_test.go

@@ -540,7 +540,8 @@ func TestFollowerReadsWithStaleDescriptor(t *testing.T) {
 	n1 := sqlutils.MakeSQLRunner(tc.Conns[0])
 	n1.Exec(t, `CREATE DATABASE t`)
 	n1.Exec(t, `CREATE TABLE test (k INT PRIMARY KEY)`)
-	n1.Exec(t, `ALTER TABLE test EXPERIMENTAL_RELOCATE VALUES (ARRAY[1,2], 1)`)
+	n1.Exec(t, `ALTER TABLE test EXPERIMENTAL_RELOCATE VOTERS VALUES (ARRAY[1], 1)`)
+	n1.Exec(t, `ALTER TABLE test EXPERIMENTAL_RELOCATE NON_VOTERS VALUES (ARRAY[2], 1)`)
 	// Speed up closing of timestamps, as we'll in order to be able to use
 	// follower_read_timestamp().
 	// Every 0.2s we'll close the timestamp from 0.4s ago. We'll attempt follower reads
@@ -570,11 +571,12 @@ func TestFollowerReadsWithStaleDescriptor(t *testing.T) {
 	require.Equal(t, roachpb.StoreID(1), entry.Lease().Replica.StoreID)
 	require.Equal(t, []roachpb.ReplicaDescriptor{
 		{NodeID: 1, StoreID: 1, ReplicaID: 1},
-		{NodeID: 2, StoreID: 2, ReplicaID: 2},
+		{NodeID: 2, StoreID: 2, ReplicaID: 2, Type: roachpb.ReplicaTypeNonVoter()},
 	}, entry.Desc().Replicas().Descriptors())
 
 	// Relocate the follower. n2 will no longer have a replica.
-	n1.Exec(t, `ALTER TABLE test EXPERIMENTAL_RELOCATE VALUES (ARRAY[1,3], 1)`)
+	n1.Exec(t, `ALTER TABLE test EXPERIMENTAL_RELOCATE VOTERS VALUES (ARRAY[1,3], 1)`)
+	n1.Exec(t, `ALTER TABLE test EXPERIMENTAL_RELOCATE NON_VOTERS VALUES (ARRAY[], 1)`)
 
 	// Execute the query again and assert the cache is updated. This query will
 	// not be executed as a follower read since it attempts to use n2 which

pkg/sql/distsql_spec_exec_factory.go

@@ -952,7 +952,7 @@ func (e *distSQLSpecExecFactory) ConstructAlterTableUnsplitAll(index cat.Index)
 }
 
 func (e *distSQLSpecExecFactory) ConstructAlterTableRelocate(
-	index cat.Index, input exec.Node, relocateLease bool,
+	index cat.Index, input exec.Node, relocateLease bool, relocateNonVoters bool,
 ) (exec.Node, error) {
 	return nil, unimplemented.NewWithIssue(47473, "experimental opt-driven distsql planning: alter table relocate")
 }

pkg/sql/opt/exec/execbuilder/statement.go

@@ -201,6 +201,7 @@ func (b *Builder) buildAlterTableRelocate(relocate *memo.AlterTableRelocateExpr)
 		table.Index(relocate.Index),
 		input.root,
 		relocate.RelocateLease,
+		relocate.RelocateNonVoters,
 	)
 	if err != nil {
 		return execPlan{}, err

pkg/sql/opt/exec/factory.opt

@@ -609,6 +609,7 @@ define AlterTableRelocate {
     Index cat.Index
     input exec.Node
     relocateLease bool
+    relocateNonVoters bool
 }
 
 # Buffer passes through the input rows but also saves them in a buffer, which

pkg/sql/opt/memo/expr_format.go

@@ -1427,6 +1427,10 @@ func FormatPrivate(f *ExprFmtCtx, private interface{}, physProps *physical.Requi
 		FormatPrivate(f, &t.AlterTableSplitPrivate, nil)
 		if t.RelocateLease {
 			f.Buffer.WriteString(" [lease]")
+		} else if t.RelocateNonVoters {
+			f.Buffer.WriteString(" [non-voters]")
+		} else {
+			f.Buffer.WriteString(" [voters]")
 		}
 
 	case *ControlJobsPrivate:

pkg/sql/opt/ops/statement.opt

@@ -192,6 +192,7 @@ define AlterTableRelocate {
 [Private]
 define AlterTableRelocatePrivate {
     RelocateLease bool
+    RelocateNonVoters bool
     _ AlterTableSplitPrivate
 }
 

pkg/sql/opt/optbuilder/alter_table.go

@@ -61,7 +61,13 @@ func (b *Builder) buildAlterTableSplit(split *tree.Split, inScope *scope) (outSc
 		b.semaCtx.Properties.Require(emptyScope.context.String(), tree.RejectSpecial)
 
 		texpr := emptyScope.resolveType(split.ExpireExpr, types.String)
-		expiration = b.buildScalar(texpr, emptyScope, nil /* outScope */, nil /* outCol */, nil /* colRefs */)
+		expiration = b.buildScalar(
+			texpr,
+			emptyScope,
+			nil, /* outScope */
+			nil, /* outCol */
+			nil, /* colRefs */
+		)
 	} else {
 		expiration = b.factory.ConstructNull(types.String)
 	}
@@ -175,7 +181,8 @@ func (b *Builder) buildAlterTableRelocate(
 	outScope.expr = b.factory.ConstructAlterTableRelocate(
 		inputScope.expr.(memo.RelExpr),
 		&memo.AlterTableRelocatePrivate{
-			RelocateLease: relocate.RelocateLease,
+			RelocateLease:     relocate.RelocateLease,
+			RelocateNonVoters: relocate.RelocateNonVoters,
 			AlterTableSplitPrivate: memo.AlterTableSplitPrivate{
 				Table:   b.factory.Metadata().AddTable(table, &tn),
 				Index:   index.Ordinal(),

pkg/sql/opt/optbuilder/testdata/alter_table

@@ -129,7 +129,17 @@ alter-table-unsplit abc@bc
 build
 ALTER TABLE abc EXPERIMENTAL_RELOCATE VALUES (ARRAY[1,2,3], 1), (ARRAY[4], 2)
 ----
-alter-table-relocate abc
+alter-table-relocate abc [voters]
+ ├── columns: key:1 pretty:2
+ └── values
+      ├── columns: column1:3 column2:4!null
+      ├── (ARRAY[1,2,3], 1)
+      └── (ARRAY[4], 2)
+
+build
+ALTER TABLE abc EXPERIMENTAL_RELOCATE NON_VOTERS VALUES (ARRAY[1,2,3], 1), (ARRAY[4], 2)
+----
+alter-table-relocate abc [non-voters]
  ├── columns: key:1 pretty:2
  └── values
       ├── columns: column1:3 column2:4!null
@@ -149,7 +159,17 @@ error (42601): too many columns in EXPERIMENTAL_RELOCATE LEASE data
 build
 ALTER INDEX abc@bc EXPERIMENTAL_RELOCATE VALUES (ARRAY[5], 1, 'foo'), (ARRAY[6,7,8], 2, 'bar')
 ----
-alter-table-relocate abc@bc
+alter-table-relocate abc@bc [voters]
+ ├── columns: key:1 pretty:2
+ └── values
+      ├── columns: column1:3 column2:4!null column3:5!null
+      ├── (ARRAY[5], 1, 'foo')
+      └── (ARRAY[6,7,8], 2, 'bar')
+
+build
+ALTER INDEX abc@bc EXPERIMENTAL_RELOCATE NON_VOTERS VALUES (ARRAY[5], 1, 'foo'), (ARRAY[6,7,8], 2, 'bar')
+----
+alter-table-relocate abc@bc [non-voters]
  ├── columns: key:1 pretty:2
  └── values
       ├── columns: column1:3 column2:4!null column3:5!null

pkg/sql/opt_exec_factory.go

@@ -1825,17 +1825,18 @@ func (ef *execFactory) ConstructAlterTableUnsplitAll(index cat.Index) (exec.Node
 
 // ConstructAlterTableRelocate is part of the exec.Factory interface.
 func (ef *execFactory) ConstructAlterTableRelocate(
-	index cat.Index, input exec.Node, relocateLease bool,
+	index cat.Index, input exec.Node, relocateLease bool, relocateNonVoters bool,
 ) (exec.Node, error) {
 	if !ef.planner.ExecCfg().Codec.ForSystemTenant() {
 		return nil, errorutil.UnsupportedWithMultiTenancy(54250)
 	}
 
 	return &relocateNode{
-		relocateLease: relocateLease,
-		tableDesc:     index.Table().(*optTable).desc,
-		index:         index.(*optIndex).desc,
-		rows:          input.(planNode),
+		relocateLease:     relocateLease,
+		relocateNonVoters: relocateNonVoters,
+		tableDesc:         index.Table().(*optTable).desc,
+		index:             index.(*optIndex).desc,
+		rows:              input.(planNode),
 	}, nil
 }
 

pkg/sql/parser/parse_test.go

@@ -1518,11 +1518,17 @@ func TestParse(t *testing.T) {
 		{`ALTER INDEX "primary" UNSPLIT AT VALUES (2)`},
 		{`ALTER INDEX public.public."primary" UNSPLIT AT VALUES (2)`},
 
-		{`ALTER TABLE a EXPERIMENTAL_RELOCATE VALUES (ARRAY[1], 1)`},
-		{`EXPLAIN ALTER TABLE a EXPERIMENTAL_RELOCATE TABLE b`},
-		{`ALTER TABLE a EXPERIMENTAL_RELOCATE SELECT * FROM t`},
-		{`ALTER TABLE d.a EXPERIMENTAL_RELOCATE VALUES (ARRAY[1, 2, 3], 'b', 2)`},
-		{`ALTER INDEX d.i EXPERIMENTAL_RELOCATE VALUES (ARRAY[1], 2)`},
+		{`ALTER TABLE a EXPERIMENTAL_RELOCATE VOTERS VALUES (ARRAY[1], 1)`},
+		{`EXPLAIN ALTER TABLE a EXPERIMENTAL_RELOCATE VOTERS TABLE b`},
+		{`ALTER TABLE a EXPERIMENTAL_RELOCATE VOTERS SELECT * FROM t`},
+		{`ALTER TABLE d.a EXPERIMENTAL_RELOCATE VOTERS VALUES (ARRAY[1, 2, 3], 'b', 2)`},
+		{`ALTER INDEX d.i EXPERIMENTAL_RELOCATE VOTERS VALUES (ARRAY[1], 2)`},
+
+		{`ALTER TABLE a EXPERIMENTAL_RELOCATE NON_VOTERS VALUES (ARRAY[1], 1)`},
+		{`EXPLAIN ALTER TABLE a EXPERIMENTAL_RELOCATE NON_VOTERS TABLE b`},
+		{`ALTER TABLE a EXPERIMENTAL_RELOCATE NON_VOTERS SELECT * FROM t`},
+		{`ALTER TABLE d.a EXPERIMENTAL_RELOCATE NON_VOTERS VALUES (ARRAY[1, 2, 3], 'b', 2)`},
+		{`ALTER INDEX d.i EXPERIMENTAL_RELOCATE NON_VOTERS VALUES (ARRAY[1], 2)`},
 
 		{`ALTER TABLE a EXPERIMENTAL_RELOCATE LEASE VALUES (1, 1)`},
 		{`ALTER TABLE a EXPERIMENTAL_RELOCATE LEASE SELECT * FROM t`},
@@ -1911,6 +1917,27 @@ func TestParse2(t *testing.T) {
 
 		{`ANALYSE t`, `ANALYZE t`},
 
+		{
+			`ALTER TABLE a EXPERIMENTAL_RELOCATE VALUES (ARRAY[1], 1)`,
+			`ALTER TABLE a EXPERIMENTAL_RELOCATE VOTERS VALUES (ARRAY[1], 1)`,
+		},
+		{
+			`EXPLAIN ALTER TABLE a EXPERIMENTAL_RELOCATE TABLE b`,
+			`EXPLAIN ALTER TABLE a EXPERIMENTAL_RELOCATE VOTERS TABLE b`,
+		},
+		{
+			`ALTER TABLE a EXPERIMENTAL_RELOCATE SELECT * FROM t`,
+			`ALTER TABLE a EXPERIMENTAL_RELOCATE VOTERS SELECT * FROM t`,
+		},
+		{
+			`ALTER TABLE d.a EXPERIMENTAL_RELOCATE VALUES (ARRAY[1, 2, 3], 'b', 2)`,
+			`ALTER TABLE d.a EXPERIMENTAL_RELOCATE VOTERS VALUES (ARRAY[1, 2, 3], 'b', 2)`,
+		},
+		{
+			`ALTER INDEX d.i EXPERIMENTAL_RELOCATE VALUES (ARRAY[1], 2)`,
+			`ALTER INDEX d.i EXPERIMENTAL_RELOCATE VOTERS VALUES (ARRAY[1], 2)`,
+		},
+
 		{`SELECT TIMESTAMP WITHOUT TIME ZONE 'foo'`, `SELECT TIMESTAMP 'foo'`},
 		{`SELECT CAST('foo' AS TIMESTAMP WITHOUT TIME ZONE)`, `SELECT CAST('foo' AS TIMESTAMP)`},
 		{`SELECT CAST(1 AS "timestamp")`, `SELECT CAST(1 AS TIMESTAMP)`},

pkg/sql/parser/sql.y

@@ -666,7 +666,7 @@ func (u *sqlSymUnion) objectNamePrefixList() tree.ObjectNamePrefixList {
 
 %token <str> NAN NAME NAMES NATURAL NEVER NEXT NO NOCANCELQUERY NOCONTROLCHANGEFEED NOCONTROLJOB
 %token <str> NOCREATEDB NOCREATELOGIN NOCREATEROLE NOLOGIN NOMODIFYCLUSTERSETTING NO_INDEX_JOIN
-%token <str> NONE NORMAL NOT NOTHING NOTNULL NOVIEWACTIVITY NOWAIT NULL NULLIF NULLS NUMERIC
+%token <str> NONE NON_VOTERS NORMAL NOT NOTHING NOTNULL NOVIEWACTIVITY NOWAIT NULL NULLIF NULLS NUMERIC
 
 %token <str> OF OFF OFFSET OID OIDS OIDVECTOR ON ONLY OPT OPTION OPTIONS OR
 %token <str> ORDER ORDINALITY OTHERS OUT OUTER OVER OVERLAPS OVERLAY OWNED OWNER OPERATOR
@@ -701,7 +701,7 @@ func (u *sqlSymUnion) objectNamePrefixList() tree.ObjectNamePrefixList {
 %token <str> UNBOUNDED UNCOMMITTED UNION UNIQUE UNKNOWN UNLOGGED UNSPLIT
 %token <str> UPDATE UPSERT UNTIL USE USER USERS USING UUID
 
-%token <str> VALID VALIDATE VALUE VALUES VARBIT VARCHAR VARIADIC VIEW VARYING VIEWACTIVITY VIRTUAL VISIBLE
+%token <str> VALID VALIDATE VALUE VALUES VARBIT VARCHAR VARIADIC VIEW VARYING VIEWACTIVITY VIRTUAL VISIBLE VOTERS
 
 %token <str> WHEN WHERE WINDOW WITH WITHIN WITHOUT WORK WRITE
 
@@ -1684,22 +1684,41 @@ relocate_kw:
   TESTING_RELOCATE
 | EXPERIMENTAL_RELOCATE
 
+voters_kw:
+  VOTERS {}
+| /* EMPTY */ {}
+
 alter_relocate_stmt:
-  ALTER TABLE table_name relocate_kw select_stmt
+  ALTER TABLE table_name relocate_kw voters_kw select_stmt
+  {
+    /* SKIP DOC */
+    name := $3.unresolvedObjectName().ToTableName()
+    $$.val = &tree.Relocate{
+      TableOrIndex: tree.TableIndexName{Table: name},
+      Rows: $6.slct(),
+    }
+  }
+| ALTER TABLE table_name relocate_kw NON_VOTERS select_stmt
   {
     /* SKIP DOC */
     name := $3.unresolvedObjectName().ToTableName()
     $$.val = &tree.Relocate{
       TableOrIndex: tree.TableIndexName{Table: name},
-      Rows: $5.slct(),
+      Rows: $6.slct(),
+      RelocateNonVoters: true,
     }
   }
 
 alter_relocate_index_stmt:
-  ALTER INDEX table_index_name relocate_kw select_stmt
+  ALTER INDEX table_index_name relocate_kw voters_kw select_stmt
+  {
+    /* SKIP DOC */
+    $$.val = &tree.Relocate{TableOrIndex: $3.tableIndexName(), Rows: $6.slct()}
+  }
+| ALTER INDEX table_index_name relocate_kw NON_VOTERS select_stmt
   {
     /* SKIP DOC */
-    $$.val = &tree.Relocate{TableOrIndex: $3.tableIndexName(), Rows: $5.slct()}
+    $$.val = &tree.Relocate{TableOrIndex: $3.tableIndexName(), Rows: $6.slct(), RelocateNonVoters: true}
   }
 
 alter_relocate_lease_stmt:
@@ -12453,6 +12472,7 @@ unreserved_keyword:
 | NOCONTROLJOB
 | NOLOGIN
 | NOMODIFYCLUSTERSETTING
+| NON_VOTERS
 | NOVIEWACTIVITY
 | NOWAIT
 | NULLS
@@ -12605,6 +12625,7 @@ unreserved_keyword:
 | VARYING
 | VIEW
 | VIEWACTIVITY
+| VOTERS
 | WITHIN
 | WITHOUT
 | WRITE

pkg/sql/relocate.go

@@ -29,10 +29,11 @@ import (
 type relocateNode struct {
 	optColumnsSlot
 
-	relocateLease bool
-	tableDesc     catalog.TableDescriptor
-	index         *descpb.IndexDescriptor
-	rows          planNode
+	relocateLease     bool
+	relocateNonVoters bool
+	tableDesc         catalog.TableDescriptor
+	index             *descpb.IndexDescriptor
+	rows              planNode
 
 	run relocateRun
 }
@@ -52,9 +53,6 @@ func (n *relocateNode) startExec(runParams) error {
 	return nil
 }
 
-// TODO(aayush): Extend EXPERIMENTAL_RELOCATE syntax to support relocating
-// non-voting replicas.
-
 func (n *relocateNode) Next(params runParams) (bool, error) {
 	// Each Next call relocates one range (corresponding to one row from n.rows).
 	// TODO(radu): perform multiple relocations in parallel.
@@ -82,7 +80,8 @@ func (n *relocateNode) Next(params runParams) (bool, error) {
 			)
 		}
 		relocation := data[0].(*tree.DArray)
-		if len(relocation.Array) == 0 {
+		if !n.relocateNonVoters && len(relocation.Array) == 0 {
+			// We cannot remove all voters.
 			return false, errors.Errorf("empty relocation array for EXPERIMENTAL_RELOCATE")
 		}
 
@@ -129,13 +128,21 @@ func (n *relocateNode) Next(params runParams) (bool, error) {
 	}
 	n.run.lastRangeStartKey = rangeDesc.StartKey.AsRawKey()
 
+	existingVoters := rangeDesc.Replicas().Voters().ReplicationTargets()
+	existingNonVoters := rangeDesc.Replicas().NonVoters().ReplicationTargets()
 	if n.relocateLease {
 		if err := params.p.ExecCfg().DB.AdminTransferLease(params.ctx, rowKey, leaseStoreID); err != nil {
 			return false, err
 		}
+	} else if n.relocateNonVoters {
+		if err := params.p.ExecCfg().DB.AdminRelocateRange(
+			params.ctx, rowKey, existingVoters, relocationTargets,
+		); err != nil {
+			return false, err
+		}
 	} else {
 		if err := params.p.ExecCfg().DB.AdminRelocateRange(
-			params.ctx, rowKey, relocationTargets, nil, /* nonVoterTargets */
+			params.ctx, rowKey, relocationTargets, existingNonVoters,
 		); err != nil {
 			return false, err
 		}

pkg/sql/sem/tree/split.go

@@ -73,8 +73,9 @@ type Relocate struct {
 	// Each row contains an array with store ids and values for the columns in the
 	// PK or index (or a prefix of the columns).
 	// See docs/RFCS/sql_split_syntax.md.
-	Rows          *Select
-	RelocateLease bool
+	Rows              *Select
+	RelocateLease     bool
+	RelocateNonVoters bool
 }
 
 // Format implements the NodeFormatter interface.
@@ -89,6 +90,10 @@ func (node *Relocate) Format(ctx *FmtCtx) {
 	ctx.WriteString(" EXPERIMENTAL_RELOCATE ")
 	if node.RelocateLease {
 		ctx.WriteString("LEASE ")
+	} else if node.RelocateNonVoters {
+		ctx.WriteString("NON_VOTERS ")
+	} else {
+		ctx.WriteString("VOTERS ")
 	}
 	ctx.FormatNode(node.Rows)
 }

pkg/sql/sem/tree/stmt.go

@@ -673,8 +673,10 @@ func (*Relocate) StatementType() StatementType { return Rows }
 func (n *Relocate) StatementTag() string {
 	if n.RelocateLease {
 		return "EXPERIMENTAL_RELOCATE LEASE"
+	} else if n.RelocateNonVoters {
+		return "EXPERIMENTAL_RELOCATE NON_VOTERS"
 	}
-	return "EXPERIMENTAL_RELOCATE"
+	return "EXPERIMENTAL_RELOCATE VOTERS"
 }
 
 // StatementType implements the Statement interface.