pkg/sql/schemachanger/scbuild: add support for DROP COLUMN

Release note: None
cockroachdb · Jul 18, 2022 · 9f59c14 · 9f59c14
1 parent e0ff623
commit 9f59c14
Show file tree

Hide file tree

Showing 74 changed files with 6,955 additions and 398 deletions.
diff --git a/pkg/bench/rttanalysis/testdata/benchmark_expectations b/pkg/bench/rttanalysis/testdata/benchmark_expectations
@@ -15,9 +15,9 @@ exp,benchmark
 19,AlterTableConfigureZone/alter_table_configure_zone_5_replicas
 19,AlterTableConfigureZone/alter_table_configure_zone_7_replicas_
 19,AlterTableConfigureZone/alter_table_configure_zone_ranges
-14,AlterTableDropColumn/alter_table_drop_1_column
-15,AlterTableDropColumn/alter_table_drop_2_columns
-16,AlterTableDropColumn/alter_table_drop_3_columns
+18,AlterTableDropColumn/alter_table_drop_1_column
+18,AlterTableDropColumn/alter_table_drop_2_columns
+18,AlterTableDropColumn/alter_table_drop_3_columns
 13,AlterTableDropConstraint/alter_table_drop_1_check_constraint
 13,AlterTableDropConstraint/alter_table_drop_2_check_constraints
 13,AlterTableDropConstraint/alter_table_drop_3_check_constraints
@@ -31,10 +31,10 @@ exp,benchmark
 15,CreateRole/create_role_with_2_options
 16,CreateRole/create_role_with_3_options
 14,CreateRole/create_role_with_no_options
-16,DropDatabase/drop_database_0_tables
-17,DropDatabase/drop_database_1_table
-18,DropDatabase/drop_database_2_tables
-19,DropDatabase/drop_database_3_tables
+15,DropDatabase/drop_database_0_tables
+16,DropDatabase/drop_database_1_table
+17,DropDatabase/drop_database_2_tables
+18,DropDatabase/drop_database_3_tables
 20,DropRole/drop_1_role
 27,DropRole/drop_2_roles
 34,DropRole/drop_3_roles

diff --git a/pkg/ccl/changefeedccl/changefeed_test.go b/pkg/ccl/changefeedccl/changefeed_test.go
@@ -2697,12 +2697,7 @@ func TestChangefeedStopOnSchemaChange(t *testing.T) {
 			dropColumn = feed(t, f, `CREATE CHANGEFEED FOR drop_column `+
 				`WITH schema_change_events='column_changes', schema_change_policy='stop', cursor = '`+tsStr+`'`)
 			defer closeFeed(t, dropColumn)
-			// NB: You might expect to only see the new row here but we'll see them
-			// all because we cannot distinguish between the index backfill and
-			// foreground writes. See #35738.
 			assertPayloads(t, dropColumn, []string{
-				`drop_column: [0]->{"after": {"a": 0}}`,
-				`drop_column: [1]->{"after": {"a": 1}}`,
 				`drop_column: [2]->{"after": {"a": 2}}`,
 			})
 		})

diff --git a/pkg/ccl/changefeedccl/schemafeed/tableeventtype_string.go b/pkg/ccl/changefeedccl/schemafeed/tableeventtype_string.go
diff --git a/pkg/ccl/changefeedccl/schemafeed/testdata/add_column b/pkg/ccl/changefeedccl/schemafeed/testdata/add_column
@@ -26,9 +26,10 @@ t 4->5: Unknown
 t 5->6: Unknown
 t 6->7: Unknown
 t 7->8: Unknown
-t 8->9: AddColumnWithBackfill|PrimaryKeyChange
-t 9->10: Unknown
+t 8->9: Unknown
+t 9->10: AddColumnWithBackfill|PrimaryKeyChange
 t 10->11: Unknown
+t 11->12: Unknown
 
 exec
 SET use_declarative_schema_changer=off;
@@ -37,6 +38,6 @@ ALTER TABLE t ADD COLUMN l INT NOT NULL DEFAULT 42;
 
 pop f=1
 ----
-t 11->12: Unknown
 t 12->13: Unknown
-t 13->14: AddColumnWithBackfill
+t 13->14: Unknown
+t 14->15: AddColumnWithBackfill
diff --git a/pkg/ccl/changefeedccl/schemafeed/testdata/drop_column b/pkg/ccl/changefeedccl/schemafeed/testdata/drop_column
@@ -14,7 +14,12 @@ pop f=1
 ----
 t 1->2: DropColumn
 t 2->3: Unknown
-t 3->4: AddHiddenColumn
+t 3->4: Unknown
+t 4->5: Unknown
+t 5->6: Unknown
+t 6->7: PrimaryKeyChange (no column changes)
+t 7->8: Unknown
+t 8->9: AddHiddenColumn
 
 exec
 SET use_declarative_schema_changer=off;
@@ -23,6 +28,6 @@ ALTER TABLE t DROP COLUMN k;
 
 pop f=1
 ----
-t 4->5: DropColumn
-t 5->6: Unknown
-t 6->7: AddHiddenColumn
+t 9->10: DropColumn
+t 10->11: Unknown
+t 11->12: AddHiddenColumn
diff --git a/pkg/ccl/kvccl/kvfollowerreadsccl/testdata/boundedstaleness/single_row b/pkg/ccl/kvccl/kvfollowerreadsccl/testdata/boundedstaleness/single_row
@@ -185,7 +185,7 @@ query idx=2 wait-until-follower-read
 SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('10s', true) WHERE pk = 1
 ----
 1
-events (15 found):
+events (17 found):
  * event 1: colbatchscan trace on node_idx 2: local read
  * event 2: transaction retry on node_idx: 2
  * event 3: colbatchscan trace on node_idx 2: local read
@@ -200,13 +200,15 @@ events (15 found):
  * event 12: transaction retry on node_idx: 2
  * event 13: colbatchscan trace on node_idx 2: local read
  * event 14: transaction retry on node_idx: 2
- * event 15: colbatchscan trace on node_idx 2: local follower read
+ * event 15: colbatchscan trace on node_idx 2: local read
+ * event 16: transaction retry on node_idx: 2
+ * event 17: colbatchscan trace on node_idx 2: local follower read
 
 query idx=2
 SELECT * FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '10s', true) WHERE pk = 1
 ----
 1
-events (15 found):
+events (17 found):
  * event 1: colbatchscan trace on node_idx 2: local read
  * event 2: transaction retry on node_idx: 2
  * event 3: colbatchscan trace on node_idx 2: local read
@@ -221,7 +223,9 @@ events (15 found):
  * event 12: transaction retry on node_idx: 2
  * event 13: colbatchscan trace on node_idx 2: local read
  * event 14: transaction retry on node_idx: 2
- * event 15: colbatchscan trace on node_idx 2: local follower read
+ * event 15: colbatchscan trace on node_idx 2: local read
+ * event 16: transaction retry on node_idx: 2
+ * event 17: colbatchscan trace on node_idx 2: local follower read
 
 # When creating a new table, ensure when nearest_only=True, we correctly error
 # with the schema not existing if none of the followers have caught up.

diff --git a/pkg/ccl/schemachangerccl/testdata/end_to_end/create_index b/pkg/ccl/schemachangerccl/testdata/end_to_end/create_index
@@ -146,7 +146,7 @@ notified job registry to adopt jobs: [1]
 begin transaction #2
 commit transaction #2
 begin transaction #3
-## PostCommitPhase stage 1 of 6 with 3 MutationType ops
+## PostCommitPhase stage 1 of 7 with 3 MutationType ops
 upsert descriptor #104
   ...
      formatVersion: 3
@@ -168,14 +168,14 @@ upsert descriptor #104
      unexposedParentSchemaId: 101
   -  version: "2"
   +  version: "3"
-update progress of schema change job #1: "PostCommitPhase stage 2 of 6 with 1 BackfillType op pending"
+update progress of schema change job #1: "PostCommitPhase stage 2 of 7 with 1 BackfillType op pending"
 commit transaction #3
 begin transaction #4
-## PostCommitPhase stage 2 of 6 with 1 BackfillType op
+## PostCommitPhase stage 2 of 7 with 1 BackfillType op
 backfill indexes [2] from index #1 in table #104
 commit transaction #4
 begin transaction #5
-## PostCommitPhase stage 3 of 6 with 3 MutationType ops
+## PostCommitPhase stage 3 of 7 with 3 MutationType ops
 upsert descriptor #104
   ...
      formatVersion: 3
@@ -197,10 +197,10 @@ upsert descriptor #104
      unexposedParentSchemaId: 101
   -  version: "3"
   +  version: "4"
-update progress of schema change job #1: "PostCommitPhase stage 4 of 6 with 1 MutationType op pending"
+update progress of schema change job #1: "PostCommitPhase stage 4 of 7 with 1 MutationType op pending"
 commit transaction #5
 begin transaction #6
-## PostCommitPhase stage 4 of 6 with 3 MutationType ops
+## PostCommitPhase stage 4 of 7 with 3 MutationType ops
 upsert descriptor #104
   ...
      formatVersion: 3
@@ -214,25 +214,50 @@ upsert descriptor #104
          version: 4
        mutationId: 1
   -    state: DELETE_ONLY
-  +    state: DELETE_AND_WRITE_ONLY
+  +    state: MERGING
      - direction: ADD
        index:
   ...
        time: {}
      unexposedParentSchemaId: 101
   -  version: "4"
   +  version: "5"
-update progress of schema change job #1: "PostCommitPhase stage 5 of 6 with 1 BackfillType op pending"
+update progress of schema change job #1: "PostCommitPhase stage 5 of 7 with 1 BackfillType op pending"
 commit transaction #6
 begin transaction #7
-## PostCommitPhase stage 5 of 6 with 1 BackfillType op
+## PostCommitPhase stage 5 of 7 with 1 BackfillType op
 merge temporary indexes [3] into backfilled indexes [2] in table #104
 commit transaction #7
 begin transaction #8
-## PostCommitPhase stage 6 of 6 with 1 ValidationType op
-validate forward indexes [2] in table #104
+## PostCommitPhase stage 6 of 7 with 3 MutationType ops
+upsert descriptor #104
+  ...
+     formatVersion: 3
+     id: 104
+  -  modificationTime:
+  -    wallTime: "1640995200000000006"
+  +  modificationTime: {}
+     mutations:
+     - direction: ADD
+  ...
+         version: 4
+       mutationId: 1
+  -    state: MERGING
+  +    state: DELETE_AND_WRITE_ONLY
+     - direction: ADD
+       index:
+  ...
+       time: {}
+     unexposedParentSchemaId: 101
+  -  version: "5"
+  +  version: "6"
+update progress of schema change job #1: "PostCommitPhase stage 7 of 7 with 1 ValidationType op pending"
 commit transaction #8
 begin transaction #9
+## PostCommitPhase stage 7 of 7 with 1 ValidationType op
+validate forward indexes [2] in table #104
+commit transaction #9
+begin transaction #10
 ## PostCommitNonRevertiblePhase stage 1 of 2 with 5 MutationType ops
 upsert descriptor #104
   ...
@@ -245,7 +270,7 @@ upsert descriptor #104
      formatVersion: 3
      id: 104
   -  modificationTime:
-  -    wallTime: "1640995200000000006"
+  -    wallTime: "1640995200000000008"
   +  indexes:
   +  - constraintId: 2
   +    createdExplicitly: true
@@ -326,12 +351,12 @@ upsert descriptor #104
   ...
        time: {}
      unexposedParentSchemaId: 101
-  -  version: "5"
-  +  version: "6"
+  -  version: "6"
+  +  version: "7"
 update progress of schema change job #1: "PostCommitNonRevertiblePhase stage 2 of 2 with 2 MutationType ops pending"
 set schema change job #1 to non-cancellable
-commit transaction #9
-begin transaction #10
+commit transaction #10
+begin transaction #11
 ## PostCommitNonRevertiblePhase stage 2 of 2 with 4 MutationType ops
 upsert descriptor #104
   ...
@@ -358,7 +383,7 @@ upsert descriptor #104
        - money
        version: 4
   -  modificationTime:
-  -    wallTime: "1640995200000000009"
+  -    wallTime: "1640995200000000010"
   -  mutations:
   -  - direction: DROP
   -    index:
@@ -401,12 +426,12 @@ upsert descriptor #104
   ...
        time: {}
      unexposedParentSchemaId: 101
-  -  version: "6"
-  +  version: "7"
+  -  version: "7"
+  +  version: "8"
 write *eventpb.FinishSchemaChange to event log for descriptor 104
 create job #2 (non-cancelable: true): "GC for "
   descriptor IDs: [104]
 update progress of schema change job #1: "all stages completed"
-commit transaction #10
+commit transaction #11
 notified job registry to adopt jobs: [2]
 # end PostCommitPhase
diff --git a/pkg/ccl/schemachangerccl/testdata/explain/create_index b/pkg/ccl/schemachangerccl/testdata/explain/create_index
@@ -39,40 +39,47 @@ Schema change plan for CREATE INDEX ‹id1› ON ‹defaultdb›.‹public›.
  │              ├── SetJobStateOnDescriptor {"DescriptorID":104,"Initialize":true}
  │              └── CreateSchemaChangerJob {"RunningStatus":"PostCommitPhase ..."}
  ├── PostCommitPhase
- │    ├── Stage 1 of 6 in PostCommitPhase
+ │    ├── Stage 1 of 7 in PostCommitPhase
  │    │    ├── 1 element transitioning toward TRANSIENT_ABSENT
  │    │    │    └── DELETE_ONLY → WRITE_ONLY TemporaryIndex:{DescID: 104, IndexID: 3, SourceIndexID: 1}
  │    │    └── 3 Mutation operations
  │    │         ├── MakeAddedIndexDeleteAndWriteOnly {"IndexID":3,"TableID":104}
  │    │         ├── SetJobStateOnDescriptor {"DescriptorID":104}
  │    │         └── UpdateSchemaChangerJob {"RunningStatus":"PostCommitPhase ..."}
- │    ├── Stage 2 of 6 in PostCommitPhase
+ │    ├── Stage 2 of 7 in PostCommitPhase
  │    │    ├── 1 element transitioning toward PUBLIC
  │    │    │    └── BACKFILL_ONLY → BACKFILLED SecondaryIndex:{DescID: 104, IndexID: 2, ConstraintID: 0, TemporaryIndexID: 3, SourceIndexID: 1}
  │    │    └── 1 Backfill operation
  │    │         └── BackfillIndex {"IndexID":2,"SourceIndexID":1,"TableID":104}
- │    ├── Stage 3 of 6 in PostCommitPhase
+ │    ├── Stage 3 of 7 in PostCommitPhase
  │    │    ├── 1 element transitioning toward PUBLIC
  │    │    │    └── BACKFILLED → DELETE_ONLY SecondaryIndex:{DescID: 104, IndexID: 2, ConstraintID: 0, TemporaryIndexID: 3, SourceIndexID: 1}
  │    │    └── 3 Mutation operations
  │    │         ├── MakeBackfillingIndexDeleteOnly {"IndexID":2,"TableID":104}
  │    │         ├── SetJobStateOnDescriptor {"DescriptorID":104}
  │    │         └── UpdateSchemaChangerJob {"RunningStatus":"PostCommitPhase ..."}
- │    ├── Stage 4 of 6 in PostCommitPhase
+ │    ├── Stage 4 of 7 in PostCommitPhase
  │    │    ├── 1 element transitioning toward PUBLIC
  │    │    │    └── DELETE_ONLY → MERGE_ONLY SecondaryIndex:{DescID: 104, IndexID: 2, ConstraintID: 0, TemporaryIndexID: 3, SourceIndexID: 1}
  │    │    └── 3 Mutation operations
- │    │         ├── MakeAddedIndexDeleteAndWriteOnly {"IndexID":2,"TableID":104}
+ │    │         ├── MakeBackfilledIndexMerging {"IndexID":2,"TableID":104}
  │    │         ├── SetJobStateOnDescriptor {"DescriptorID":104}
  │    │         └── UpdateSchemaChangerJob {"RunningStatus":"PostCommitPhase ..."}
- │    ├── Stage 5 of 6 in PostCommitPhase
+ │    ├── Stage 5 of 7 in PostCommitPhase
  │    │    ├── 1 element transitioning toward PUBLIC
  │    │    │    └── MERGE_ONLY → MERGED SecondaryIndex:{DescID: 104, IndexID: 2, ConstraintID: 0, TemporaryIndexID: 3, SourceIndexID: 1}
  │    │    └── 1 Backfill operation
  │    │         └── MergeIndex {"BackfilledIndexID":2,"TableID":104,"TemporaryIndexID":3}
- │    └── Stage 6 of 6 in PostCommitPhase
+ │    ├── Stage 6 of 7 in PostCommitPhase
+ │    │    ├── 1 element transitioning toward PUBLIC
+ │    │    │    └── MERGED → WRITE_ONLY SecondaryIndex:{DescID: 104, IndexID: 2, ConstraintID: 0, TemporaryIndexID: 3, SourceIndexID: 1}
+ │    │    └── 3 Mutation operations
+ │    │         ├── MakeMergedIndexWriteOnly {"IndexID":2,"TableID":104}
+ │    │         ├── SetJobStateOnDescriptor {"DescriptorID":104}
+ │    │         └── UpdateSchemaChangerJob {"RunningStatus":"PostCommitPhase ..."}
+ │    └── Stage 7 of 7 in PostCommitPhase
  │         ├── 1 element transitioning toward PUBLIC
- │         │    └── MERGED → VALIDATED SecondaryIndex:{DescID: 104, IndexID: 2, ConstraintID: 0, TemporaryIndexID: 3, SourceIndexID: 1}
+ │         │    └── WRITE_ONLY → VALIDATED SecondaryIndex:{DescID: 104, IndexID: 2, ConstraintID: 0, TemporaryIndexID: 3, SourceIndexID: 1}
  │         └── 1 Validation operation
  │              └── ValidateUniqueIndex {"IndexID":2,"TableID":104}
  └── PostCommitNonRevertiblePhase