schemachanger: generalized statement-time execution

This change makes it possible to plan and execute the following change
correctly:

    BEGIN;
    DROP TABLE foo;
    CREATE UNIQUE INDEX idx ON bar (x);
    COMMIT;

Inside the transaction, the table is seen as dropped right after the
DROP TABLE statement executes, but the table is not seen as dropped by
other transactions until the new, unrelated index has been validated, at
which point the schema change can no longer fail.

Fixes #88294.

Significant changes:
- descs.Collection has a new ResetUncommitted method.
- scstage.BuildStages is rewritten with unrolled loops and a new scheme:
  statement phase has revertible op-edges, pre-commit has two stages,
  one which resets the uncommitted state by scheduling the new
  scop.UndoAllInTxnImmediateMutationOpSideEffects op, which triggers a call
  to ResetUncommitted.
- This happens in scexec.executeMutationOps, which is now split into two
  pieces, the first one which executes ops whose side-effects can be undone
  by ResetUncommitted, and the other which handles those which can’t.
- These ops implement scop.ImmediateMutationOp and scop.DeferredMutationOp
  respectively.
- These have their own visitors with their own state and their own
  dependencies.

This all means that we can handle DROPs properly now: the TXN_DROPPED
element state is removed as are all the synthetic descriptor manipulations
(outside of index and constraint validation, that is). Furthermore, the
scgraph.PreviousTransactionPrecedence edge kind no longer serves any
purpose and has been replaced by PreviousStagePrecedence everywhere.

There’s a bunch of other more-or-less related changes, mostly as a
consequence to the above:
- scbuild.Build must produce a tighter target set: no-op targets are
  elided.
- nstree.MutableCatalog has new methods for deleting comments and zone
  configs which are useful.
- sctestdeps.TestState has better storage layers modelling
  (in-memory > stored > committed)
- Added missing “relation dropped before column|index no longer public”
  rules which fix DROP COLUMN CASCADE when the column is referenced in
  a view.
- scgraph.Graph has a new GetOpEdgeTo method which is useful.
- added debug info to assertion error messages in scstage.BuildStages to
  make debugging easier during development.
- EXPLAIN (DDL) output has a nicer root node label which puts the last
  statement first.

Release note: None

build/bazelutil/check.sh

@@ -35,7 +35,8 @@ pkg/sql/opt/optgen/lang/gen.go://go:generate langgen -out expr.og.go exprs lang.
 pkg/sql/opt/optgen/lang/gen.go://go:generate langgen -out operator.og.go ops lang.opt
 pkg/sql/schemachanger/scexec/exec_backfill_test.go://go:generate mockgen -package scexec_test -destination=mocks_generated_test.go --self_package scexec . Catalog,Dependencies,Backfiller,Merger,BackfillerTracker,IndexSpanSplitter,PeriodicProgressFlusher
 pkg/sql/schemachanger/scop/backfill.go://go:generate go run ./generate_visitor.go scop Backfill backfill.go backfill_visitor_generated.go
-pkg/sql/schemachanger/scop/mutation.go://go:generate go run ./generate_visitor.go scop Mutation mutation.go mutation_visitor_generated.go
+pkg/sql/schemachanger/scop/immediate_mutation.go://go:generate go run ./generate_visitor.go scop ImmediateMutation immediate_mutation.go immediate_mutation_visitor_generated.go
+pkg/sql/schemachanger/scop/deferred_mutation.go://go:generate go run ./generate_visitor.go scop DeferredMutation deferred_mutation.go deferred_mutation_visitor_generated.go
 pkg/sql/schemachanger/scop/validation.go://go:generate go run ./generate_visitor.go scop Validation validation.go validation_visitor_generated.go
 pkg/sql/schemachanger/scpb/state.go://go:generate go run element_generator.go --in elements.proto --out elements_generated.go
 pkg/sql/schemachanger/scpb/state.go://go:generate go run element_uml_generator.go --out uml/table.puml

pkg/ccl/schemachangerccl/testdata/end_to_end/create_index

@@ -111,7 +111,10 @@ upsert descriptor #104
   +  version: "2"
 # end StatementPhase
 # begin PreCommitPhase
-## PreCommitPhase stage 1 of 1 with 2 MutationType ops
+## PreCommitPhase stage 1 of 2 with 1 MutationType op
+undo all catalog changes within txn #1
+persist all catalog changes to storage
+## PreCommitPhase stage 2 of 2 with 13 MutationType ops
 upsert descriptor #104
   ...
      createAsOfTime:
@@ -134,11 +137,91 @@ upsert descriptor #104
   +    targets: <redacted>
      families:
      - columnIds:
+  ...
+     id: 104
+     modificationTime: {}
+  +  mutations:
+  +  - direction: ADD
+  +    index:
+  +      createdAtNanos: "1640998800000000000"
+  +      createdExplicitly: true
+  +      foreignKey: {}
+  +      geoConfig: {}
+  +      id: 2
+  +      interleave: {}
+  +      keyColumnDirections:
+  +      - ASC
+  +      - ASC
+  +      keyColumnIds:
+  +      - 1
+  +      - 2
+  +      keyColumnNames:
+  +      - id
+  +      - name
+  +      name: id1
+  +      partitioning:
+  +        list:
+  +        - name: p1
+  +          subpartitioning: {}
+  +          values:
+  +          - AwI=
+  +        numColumns: 1
+  +      sharded: {}
+  +      storeColumnIds:
+  +      - 3
+  +      storeColumnNames:
+  +      - money
+  +      version: 4
+  +    mutationId: 1
+  +    state: BACKFILLING
+  +  - direction: ADD
+  +    index:
+  +      constraintId: 1
+  +      createdExplicitly: true
+  +      foreignKey: {}
+  +      geoConfig: {}
+  +      id: 3
+  +      interleave: {}
+  +      keyColumnDirections:
+  +      - ASC
+  +      - ASC
+  +      keyColumnIds:
+  +      - 1
+  +      - 2
+  +      keyColumnNames:
+  +      - id
+  +      - name
+  +      name: crdb_internal_index_3_name_placeholder
+  +      partitioning:
+  +        list:
+  +        - name: p1
+  +          subpartitioning: {}
+  +          values:
+  +          - AwI=
+  +        numColumns: 1
+  +      sharded: {}
+  +      storeColumnIds:
+  +      - 3
+  +      storeColumnNames:
+  +      - money
+  +      useDeletePreservingEncoding: true
+  +      version: 4
+  +    mutationId: 1
+  +    state: DELETE_ONLY
+     name: t1
+     nextColumnId: 4
+     nextConstraintId: 2
+     nextFamilyId: 1
+  -  nextIndexId: 2
+  +  nextIndexId: 4
+     nextMutationId: 1
+     parentId: 100
   ...
        time: {}
      unexposedParentSchemaId: 101
   -  version: "1"
   +  version: "2"
+persist all catalog changes to storage
 create job #1 (non-cancelable: false): "CREATE INDEX id1 ON defaultdb.public.t1 (id, name) STORING (money) PARTITION BY LIST (id) (PARTITION p1 VALUES IN (1))"
   descriptor IDs: [104]
 # end PreCommitPhase
@@ -162,6 +245,7 @@ upsert descriptor #104
      unexposedParentSchemaId: 101
   -  version: "2"
   +  version: "3"
+persist all catalog changes to storage
 update progress of schema change job #1: "PostCommitPhase stage 2 of 7 with 1 BackfillType op pending"
 commit transaction #3
 begin transaction #4
@@ -183,6 +267,7 @@ upsert descriptor #104
      unexposedParentSchemaId: 101
   -  version: "3"
   +  version: "4"
+persist all catalog changes to storage
 update progress of schema change job #1: "PostCommitPhase stage 4 of 7 with 1 MutationType op pending"
 commit transaction #5
 begin transaction #6
@@ -200,6 +285,7 @@ upsert descriptor #104
      unexposedParentSchemaId: 101
   -  version: "4"
   +  version: "5"
+persist all catalog changes to storage
 update progress of schema change job #1: "PostCommitPhase stage 5 of 7 with 1 BackfillType op pending"
 commit transaction #6
 begin transaction #7
@@ -221,6 +307,7 @@ upsert descriptor #104
      unexposedParentSchemaId: 101
   -  version: "5"
   +  version: "6"
+persist all catalog changes to storage
 update progress of schema change job #1: "PostCommitPhase stage 7 of 7 with 1 ValidationType op pending"
 commit transaction #8
 begin transaction #9
@@ -321,6 +408,7 @@ upsert descriptor #104
      unexposedParentSchemaId: 101
   -  version: "6"
   +  version: "7"
+persist all catalog changes to storage
 adding table for stats refresh: 104
 update progress of schema change job #1: "PostCommitNonRevertiblePhase stage 2 of 2 with 2 MutationType ops pending"
 set schema change job #1 to non-cancellable
@@ -394,6 +482,7 @@ upsert descriptor #104
      unexposedParentSchemaId: 101
   -  version: "7"
   +  version: "8"
+persist all catalog changes to storage
 create job #2 (non-cancelable: true): "GC for CREATE INDEX id1 ON defaultdb.public.t1 (id, name) STORING (money) PARTITION BY LIST (id) (PARTITION p1 VALUES IN (1))"
   descriptor IDs: [104]
 update progress of schema change job #1: "all stages completed"

pkg/ccl/schemachangerccl/testdata/end_to_end/drop_database_multiregion_primary_region

@@ -29,79 +29,63 @@ write *eventpb.DropDatabase to event log:
     statement: DROP DATABASE ‹multi_region_test_db› CASCADE
     tag: DROP DATABASE
     user: root
-## StatementPhase stage 1 of 1 with 5 MutationType ops
-add synthetic descriptor #104:
-   database:
-     id: 104
-  -  modificationTime: {}
-  +  modificationTime:
-  +    wallTime: "1640995200000000001"
-     name: multi_region_test_db
-     privileges:
+## StatementPhase stage 1 of 1 with 12 MutationType ops
+delete database namespace entry {0 0 multi_region_test_db} -> 104
+delete schema namespace entry {104 0 public} -> 105
+delete object namespace entry {104 105 crdb_internal_region} -> 106
+delete object namespace entry {104 105 _crdb_internal_region} -> 107
+delete object namespace entry {104 105 table_regional_by_table} -> 108
+upsert descriptor #104
   ...
-       public:
-         id: 105
+       regionEnumId: 106
+       survivalGoal: REGION_FAILURE
+  -  schemas:
+  -    public:
+  -      id: 105
+  -  version: "1"
   +  state: DROP
-     version: "1"
-add synthetic descriptor #105:
-   schema:
-     id: 105
-  -  modificationTime: {}
-  +  modificationTime:
-  +    wallTime: "1640995200000000001"
-     name: public
-     parentId: 104
+  +  version: "2"
+upsert descriptor #105
   ...
          withGrantOption: "2"
        version: 2
+  -  version: "1"
   +  state: DROP
-     version: "1"
-add synthetic descriptor #106:
-  ...
-     id: 106
-     kind: MULTIREGION_ENUM
-  -  modificationTime: {}
-  +  modificationTime:
-  +    wallTime: "1640995200000000001"
-     name: crdb_internal_region
-     parentId: 104
+  +  version: "2"
+upsert descriptor #106
   ...
        primaryRegion: us-east1
        zoneConfigExtensions: {}
+  -  version: "2"
   +  state: DROP
-     version: "2"
-add synthetic descriptor #107:
-  ...
-     id: 107
-     kind: ALIAS
-  -  modificationTime: {}
-  +  modificationTime:
-  +    wallTime: "1640995200000000001"
-     name: _crdb_internal_region
-     parentId: 104
+  +  version: "3"
+upsert descriptor #107
   ...
          withGrantOption: "2"
        version: 2
+  -  version: "1"
   +  state: DROP
-     version: "1"
-add synthetic descriptor #108:
+  +  version: "2"
+upsert descriptor #108
   ...
-       regionalByTable:
-         region: us-east2
-  -  modificationTime: {}
-  +  modificationTime:
-  +    wallTime: "1640995200000000001"
-     name: table_regional_by_table
-     nextColumnId: 2
+     createAsOfTime:
+       wallTime: "1640995200000000000"
+  +  dropTime: <redacted>"
+     families:
+     - columnIds:
   ...
      replacementOf:
        time: {}
   +  state: DROP
      unexposedParentSchemaId: 105
-     version: "1"
+  -  version: "1"
+  +  version: "2"
 # end StatementPhase
 # begin PreCommitPhase
-## PreCommitPhase stage 1 of 1 with 22 MutationType ops
+## PreCommitPhase stage 1 of 2 with 1 MutationType op
+undo all catalog changes within txn #1
+persist all catalog changes to storage
+## PreCommitPhase stage 2 of 2 with 22 MutationType ops
 delete database namespace entry {0 0 multi_region_test_db} -> 104
 delete schema namespace entry {104 0 public} -> 105
 delete object namespace entry {104 105 crdb_internal_region} -> 106
@@ -227,6 +211,7 @@ upsert descriptor #108
      unexposedParentSchemaId: 105
   -  version: "1"
   +  version: "2"
+persist all catalog changes to storage
 delete role settings for database on #104
 create job #1 (non-cancelable: true): "DROP DATABASE multi_region_test_db CASCADE"
   descriptor IDs: [104 105 106 107 108]
@@ -265,6 +250,7 @@ delete descriptor #104
 delete descriptor #105
 delete descriptor #106
 delete descriptor #107
+persist all catalog changes to storage
 create job #2 (non-cancelable: true): "GC for DROP DATABASE multi_region_test_db CASCADE"
   descriptor IDs: [108 104]
 update progress of schema change job #1: "all stages completed"

pkg/ccl/schemachangerccl/testdata/end_to_end/drop_table_multiregion

@@ -25,25 +25,47 @@ write *eventpb.DropTable to event log:
     tag: DROP TABLE
     user: root
   tableName: multi_region_test_db.public.table_regional_by_row
-## StatementPhase stage 1 of 1 with 1 MutationType op
-add synthetic descriptor #108:
+## StatementPhase stage 1 of 1 with 6 MutationType ops
+delete object namespace entry {104 105 table_regional_by_row} -> 108
+upsert descriptor #106
+  ...
+         withGrantOption: "2"
+       version: 2
+  -  referencingDescriptorIds:
+  -  - 108
+     regionConfig:
+       primaryRegion: us-east1
+       zoneConfigExtensions: {}
+  -  version: "2"
+  +  version: "3"
+upsert descriptor #107
+  ...
+         withGrantOption: "2"
+       version: 2
+  -  referencingDescriptorIds:
+  -  - 108
+  -  version: "2"
+  +  version: "3"
+upsert descriptor #108
   ...
-     localityConfig:
-       regionalByRow: {}
-  -  modificationTime: {}
-  +  modificationTime:
-  +    wallTime: "1640995200000000001"
-     name: table_regional_by_row
-     nextColumnId: 3
+     createAsOfTime:
+       wallTime: "1640995200000000000"
+  +  dropTime: <redacted>"
+     families:
+     - columnIds:
   ...
      replacementOf:
        time: {}
   +  state: DROP
      unexposedParentSchemaId: 105
-     version: "1"
+  -  version: "1"
+  +  version: "2"
 # end StatementPhase
 # begin PreCommitPhase
-## PreCommitPhase stage 1 of 1 with 14 MutationType ops
+## PreCommitPhase stage 1 of 2 with 1 MutationType op
+undo all catalog changes within txn #1
+persist all catalog changes to storage
+## PreCommitPhase stage 2 of 2 with 14 MutationType ops
 delete object namespace entry {104 105 table_regional_by_row} -> 108
 upsert descriptor #106
    type:
@@ -107,6 +129,7 @@ upsert descriptor #108
      unexposedParentSchemaId: 105
   -  version: "1"
   +  version: "2"
+persist all catalog changes to storage
 create job #1 (non-cancelable: true): "DROP TABLE multi_region_test_db.public.table_regional_by_row"
   descriptor IDs: [106 107 108]
 # end PreCommitPhase
@@ -169,6 +192,7 @@ upsert descriptor #108
      unexposedParentSchemaId: 105
   -  version: "2"
   +  version: "3"
+persist all catalog changes to storage
 create job #2 (non-cancelable: true): "GC for DROP TABLE multi_region_test_db.public.table_regional_by_row"
   descriptor IDs: [108]
 update progress of schema change job #1: "all stages completed"