Merge #95586 #95782 #95786

95586: roachprod: add tenant param to cluster.Conn* methods r=smg260 a=msbutler

This patch allows the roachtest writer to easily open a sql connection with an
in-process tenant.

Release note: None

Epic: None.

95782: builtins: add non-null check to a recently merged builtin overload r=yuzefovich a=yuzefovich

Fixes: #95671.

Release note: None

95786: schemachanger: move event logging to scbuild and scrun r=postamar a=postamar

Previously, the declarative schema changer would write events to the event log by means of operations planned in a mutation stage. This commit makes it so that the following schema change events are written by the schema change job:
  - reverse_schema_change,
  - finish_schema_change,
  - finish_schema_change_rollback. 

The remaining schema change events are written at build-time, which is in line with the legacy schema changer which does this in the planNodes' startExec method.

This makes for simpler declarative schema changer operation generation logic and much simpler plans. This comes at the expense of some transactionality, but it appears that this behaviour was not expected in any case.

Fixes #95779.

Release note: None

Co-authored-by: Michael Butler <[email protected]>
Co-authored-by: Yahor Yuzefovich <[email protected]>
Co-authored-by: Marius Posta <[email protected]>

pkg/ccl/schemachangerccl/testdata/end_to_end/create_index

@@ -13,6 +13,16 @@ CREATE INDEX id1
 begin transaction #1
 # begin StatementPhase
 checking for feature: CREATE INDEX
+write *eventpb.CreateIndex to event log:
+  indexName: id1
+  mutationId: 1
+  sql:
+    descriptorId: 104
+    statement: CREATE INDEX ‹id1› ON ‹defaultdb›.‹public›.‹t1› (‹id›, ‹name›) STORING
+      (‹money›) PARTITION BY LIST (‹id›) (PARTITION ‹p1› VALUES IN (‹1›))
+    tag: CREATE INDEX
+    user: root
+  tableName: defaultdb.public.t1
 ## StatementPhase stage 1 of 1 with 11 MutationType ops
 upsert descriptor #104
   ...
@@ -218,7 +228,7 @@ begin transaction #9
 validate forward indexes [2] in table #104
 commit transaction #9
 begin transaction #10
-## PostCommitNonRevertiblePhase stage 1 of 2 with 6 MutationType ops
+## PostCommitNonRevertiblePhase stage 1 of 2 with 5 MutationType ops
 upsert descriptor #104
   ...
              BY LIST (id) (PARTITION p1 VALUES IN (1))
@@ -311,7 +321,6 @@ upsert descriptor #104
      unexposedParentSchemaId: 101
   -  version: "6"
   +  version: "7"
-write *eventpb.CreateIndex to event log: CREATE INDEX ‹id1› ON ‹defaultdb›.‹public›.‹t1› (‹id›, ‹name›) STORING (‹money›) PARTITION BY LIST (‹id›) (PARTITION ‹p1› VALUES IN (‹1›))
 adding table for stats refresh: 104
 update progress of schema change job #1: "PostCommitNonRevertiblePhase stage 2 of 2 with 2 MutationType ops pending"
 set schema change job #1 to non-cancellable
@@ -385,12 +394,14 @@ upsert descriptor #104
      unexposedParentSchemaId: 101
   -  version: "7"
   +  version: "8"
-write *eventpb.FinishSchemaChange to event log
 create job #2 (non-cancelable: true): "GC for CREATE INDEX id1 ON defaultdb.public.t1 (id, name) STORING (money) PARTITION BY LIST (id) (PARTITION p1 VALUES IN (1))"
   descriptor IDs: [104]
 update progress of schema change job #1: "all stages completed"
 set schema change job #1 to non-cancellable
 updated schema change job #1 descriptor IDs to []
+write *eventpb.FinishSchemaChange to event log:
+  sc:
+    descriptorId: 104
 commit transaction #11
 notified job registry to adopt jobs: [2]
 # end PostCommitPhase

pkg/ccl/schemachangerccl/testdata/end_to_end/drop_database_multiregion_primary_region

@@ -18,6 +18,17 @@ begin transaction #1
 # begin StatementPhase
 checking for feature: DROP DATABASE
 increment telemetry for sql.schema.drop_database
+write *eventpb.DropDatabase to event log:
+  databaseName: multi_region_test_db
+  droppedSchemaObjects:
+  - multi_region_test_db.public.crdb_internal_region
+  - multi_region_test_db.public._crdb_internal_region
+  - multi_region_test_db.public.table_regional_by_table
+  sql:
+    descriptorId: 104
+    statement: DROP DATABASE ‹multi_region_test_db› CASCADE
+    tag: DROP DATABASE
+    user: root
 ## StatementPhase stage 1 of 1 with 5 MutationType ops
 add synthetic descriptor #104:
    database:
@@ -226,7 +237,7 @@ notified job registry to adopt jobs: [1]
 begin transaction #2
 commit transaction #2
 begin transaction #3
-## PostCommitNonRevertiblePhase stage 1 of 1 with 18 MutationType ops
+## PostCommitNonRevertiblePhase stage 1 of 1 with 13 MutationType ops
 upsert descriptor #108
   ...
      createAsOfTime:
@@ -254,12 +265,14 @@ delete descriptor #104
 delete descriptor #105
 delete descriptor #106
 delete descriptor #107
-write *eventpb.DropDatabase to event log: DROP DATABASE ‹multi_region_test_db› CASCADE
 create job #2 (non-cancelable: true): "GC for DROP DATABASE multi_region_test_db CASCADE"
   descriptor IDs: [108 104]
 update progress of schema change job #1: "all stages completed"
 set schema change job #1 to non-cancellable
 updated schema change job #1 descriptor IDs to []
+write *eventpb.FinishSchemaChange to event log:
+  sc:
+    descriptorId: 104
 commit transaction #3
 notified job registry to adopt jobs: [2]
 # end PostCommitPhase

pkg/ccl/schemachangerccl/testdata/end_to_end/drop_table_multiregion

@@ -18,6 +18,13 @@ begin transaction #1
 # begin StatementPhase
 checking for feature: DROP TABLE
 increment telemetry for sql.schema.drop_table
+write *eventpb.DropTable to event log:
+  sql:
+    descriptorId: 108
+    statement: DROP TABLE ‹multi_region_test_db›.‹public›.‹table_regional_by_row›
+    tag: DROP TABLE
+    user: root
+  tableName: multi_region_test_db.public.table_regional_by_row
 ## StatementPhase stage 1 of 1 with 1 MutationType op
 add synthetic descriptor #108:
   ...
@@ -109,7 +116,7 @@ notified job registry to adopt jobs: [1]
 begin transaction #2
 commit transaction #2
 begin transaction #3
-## PostCommitNonRevertiblePhase stage 1 of 1 with 7 MutationType ops
+## PostCommitNonRevertiblePhase stage 1 of 1 with 6 MutationType ops
 upsert descriptor #106
    type:
      arrayTypeId: 107
@@ -162,12 +169,14 @@ upsert descriptor #108
      unexposedParentSchemaId: 105
   -  version: "2"
   +  version: "3"
-write *eventpb.DropTable to event log: DROP TABLE ‹multi_region_test_db›.‹public›.‹table_regional_by_row›
 create job #2 (non-cancelable: true): "GC for DROP TABLE multi_region_test_db.public.table_regional_by_row"
   descriptor IDs: [108]
 update progress of schema change job #1: "all stages completed"
 set schema change job #1 to non-cancellable
 updated schema change job #1 descriptor IDs to []
+write *eventpb.FinishSchemaChange to event log:
+  sc:
+    descriptorId: 108
 commit transaction #3
 notified job registry to adopt jobs: [2]
 # end PostCommitPhase

pkg/ccl/schemachangerccl/testdata/end_to_end/drop_table_multiregion_primary_region

@@ -18,6 +18,14 @@ begin transaction #1
 # begin StatementPhase
 checking for feature: DROP TABLE
 increment telemetry for sql.schema.drop_table
+write *eventpb.DropTable to event log:
+  sql:
+    descriptorId: 108
+    statement: DROP TABLE ‹multi_region_test_db›.‹public›.‹table_regional_by_table›
+      CASCADE
+    tag: DROP TABLE
+    user: root
+  tableName: multi_region_test_db.public.table_regional_by_table
 ## StatementPhase stage 1 of 1 with 1 MutationType op
 add synthetic descriptor #108:
   ...
@@ -94,7 +102,7 @@ notified job registry to adopt jobs: [1]
 begin transaction #2
 commit transaction #2
 begin transaction #3
-## PostCommitNonRevertiblePhase stage 1 of 1 with 6 MutationType ops
+## PostCommitNonRevertiblePhase stage 1 of 1 with 5 MutationType ops
 upsert descriptor #106
    type:
      arrayTypeId: 107
@@ -134,12 +142,14 @@ upsert descriptor #108
      unexposedParentSchemaId: 105
   -  version: "2"
   +  version: "3"
-write *eventpb.DropTable to event log: DROP TABLE ‹multi_region_test_db›.‹public›.‹table_regional_by_table› CASCADE
 create job #2 (non-cancelable: true): "GC for DROP TABLE multi_region_test_db.public.table_regional_by_table CASCADE"
   descriptor IDs: [108]
 update progress of schema change job #1: "all stages completed"
 set schema change job #1 to non-cancellable
 updated schema change job #1 descriptor IDs to []
+write *eventpb.FinishSchemaChange to event log:
+  sc:
+    descriptorId: 108
 commit transaction #3
 notified job registry to adopt jobs: [2]
 # end PostCommitPhase

pkg/ccl/schemachangerccl/testdata/explain/create_index

@@ -95,10 +95,9 @@ Schema change plan for CREATE INDEX ‹id1› ON ‹defaultdb›.‹public›.
       │    │    ├── PUBLIC     → TRANSIENT_ABSENT      IndexColumn:{DescID: 104, ColumnID: 1, IndexID: 3}
       │    │    ├── PUBLIC     → TRANSIENT_ABSENT      IndexColumn:{DescID: 104, ColumnID: 2, IndexID: 3}
       │    │    └── PUBLIC     → TRANSIENT_ABSENT      IndexColumn:{DescID: 104, ColumnID: 3, IndexID: 3}
-      │    └── 6 Mutation operations
-      │         ├── RefreshStats {"TableID":104}
-      │         ├── LogEvent {"TargetStatus":2}
+      │    └── 5 Mutation operations
       │         ├── MakeValidatedSecondaryIndexPublic {"IndexID":2,"TableID":104}
+      │         ├── RefreshStats {"TableID":104}
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":3,"TableID":104}
       │         ├── SetJobStateOnDescriptor {"DescriptorID":104}
       │         └── UpdateSchemaChangerJob {"IsNonCancelable":true,"RunningStatus":"PostCommitNonRev..."}

pkg/ccl/schemachangerccl/testdata/explain/create_index.rollback_5_of_7

@@ -23,9 +23,8 @@ Schema change plan for rolling back CREATE INDEX ‹id1› ON ‹defaultdb›.pu
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 1, IndexID: 3}
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 2, IndexID: 3}
       │    │    └── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 3, IndexID: 3}
-      │    └── 6 Mutation operations
+      │    └── 5 Mutation operations
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":3,"TableID":104}
-      │         ├── LogEvent {"TargetStatus":1}
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":2,"TableID":104}
       │         ├── SetIndexName {"IndexID":2,"Name":"crdb_internal_in...","TableID":104}
       │         ├── SetJobStateOnDescriptor {"DescriptorID":104}

pkg/ccl/schemachangerccl/testdata/explain/create_index.rollback_6_of_7

@@ -23,9 +23,8 @@ Schema change plan for rolling back CREATE INDEX ‹id1› ON ‹defaultdb›.pu
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 1, IndexID: 3}
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 2, IndexID: 3}
       │    │    └── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 3, IndexID: 3}
-      │    └── 6 Mutation operations
+      │    └── 5 Mutation operations
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":3,"TableID":104}
-      │         ├── LogEvent {"TargetStatus":1}
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":2,"TableID":104}
       │         ├── SetIndexName {"IndexID":2,"Name":"crdb_internal_in...","TableID":104}
       │         ├── SetJobStateOnDescriptor {"DescriptorID":104}

pkg/ccl/schemachangerccl/testdata/explain/create_index.rollback_7_of_7

@@ -23,8 +23,7 @@ Schema change plan for rolling back CREATE INDEX ‹id1› ON ‹defaultdb›.pu
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 1, IndexID: 3}
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 2, IndexID: 3}
       │    │    └── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 3, IndexID: 3}
-      │    └── 6 Mutation operations
-      │         ├── LogEvent {"TargetStatus":1}
+      │    └── 5 Mutation operations
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":2,"TableID":104}
       │         ├── SetIndexName {"IndexID":2,"Name":"crdb_internal_in...","TableID":104}
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":3,"TableID":104}

pkg/ccl/schemachangerccl/testdata/explain/drop_database_multiregion_primary_region

@@ -115,17 +115,12 @@ Schema change plan for DROP DATABASE ‹multi_region_test_db› CASCADE;
            │    ├── DROPPED → ABSENT Table:{DescID: 108}
            │    ├── PUBLIC  → ABSENT IndexData:{DescID: 108, IndexID: 1}
            │    └── PUBLIC  → ABSENT TableData:{DescID: 108, ReferencedDescID: 104}
-           └── 18 Mutation operations
-                ├── LogEvent {"TargetStatus":1}
+           └── 13 Mutation operations
                 ├── DeleteDescriptor {"DescriptorID":104}
                 ├── CreateGCJobForDatabase {"DatabaseID":104}
-                ├── LogEvent {"TargetStatus":1}
                 ├── DeleteDescriptor {"DescriptorID":105}
-                ├── LogEvent {"TargetStatus":1}
                 ├── DeleteDescriptor {"DescriptorID":106}
-                ├── LogEvent {"TargetStatus":1}
                 ├── DeleteDescriptor {"DescriptorID":107}
-                ├── LogEvent {"TargetStatus":1}
                 ├── CreateGCJobForTable {"DatabaseID":104,"TableID":108}
                 ├── CreateGCJobForIndex {"IndexID":1,"TableID":108}
                 ├── RemoveJobStateFromDescriptor {"DescriptorID":104}

pkg/ccl/schemachangerccl/testdata/explain/drop_table_multiregion

@@ -70,8 +70,7 @@ Schema change plan for DROP TABLE ‹multi_region_test_db›.‹public›.‹tab
            │    ├── DROPPED → ABSENT Table:{DescID: 108}
            │    ├── PUBLIC  → ABSENT IndexData:{DescID: 108, IndexID: 1}
            │    └── PUBLIC  → ABSENT TableData:{DescID: 108, ReferencedDescID: 104}
-           └── 7 Mutation operations
-                ├── LogEvent {"TargetStatus":1}
+           └── 6 Mutation operations
                 ├── CreateGCJobForTable {"DatabaseID":104,"TableID":108}
                 ├── CreateGCJobForIndex {"IndexID":1,"TableID":108}
                 ├── RemoveJobStateFromDescriptor {"DescriptorID":106}

pkg/ccl/schemachangerccl/testdata/explain/drop_table_multiregion_primary_region

@@ -57,8 +57,7 @@ Schema change plan for DROP TABLE ‹multi_region_test_db›.‹public›.‹tab
            │    ├── DROPPED → ABSENT Table:{DescID: 108}
            │    ├── PUBLIC  → ABSENT IndexData:{DescID: 108, IndexID: 1}
            │    └── PUBLIC  → ABSENT TableData:{DescID: 108, ReferencedDescID: 104}
-           └── 6 Mutation operations
-                ├── LogEvent {"TargetStatus":1}
+           └── 5 Mutation operations
                 ├── CreateGCJobForTable {"DatabaseID":104,"TableID":108}
                 ├── CreateGCJobForIndex {"IndexID":1,"TableID":108}
                 ├── RemoveJobStateFromDescriptor {"DescriptorID":106}

pkg/ccl/schemachangerccl/testdata/explain_verbose/create_index

@@ -431,33 +431,15 @@ EXPLAIN (ddl, verbose) CREATE INDEX id1
     │   │       └── • skip PUBLIC → TRANSIENT_ABSENT operations
     │   │             rule: "skip index-column removal ops on index removal"
     │   │
-    │   └── • 6 Mutation operations
-    │       │
-    │       ├── • RefreshStats
-    │       │     TableID: 104
-    │       │
-    │       ├── • LogEvent
-    │       │     Element:
-    │       │       SecondaryIndex:
-    │       │         indexId: 2
-    │       │         sourceIndexId: 1
-    │       │         tableId: 104
-    │       │         temporaryIndexId: 3
-    │       │     EventBase:
-    │       │       Authorization:
-    │       │         UserName: root
-    │       │       Statement: CREATE INDEX ‹id1› ON ‹defaultdb›.‹public›.‹t1› (‹id›, ‹name›) STORING
-    │       │         (‹money›) PARTITION BY LIST (‹id›) (PARTITION ‹p1› VALUES IN (‹1›))
-    │       │       StatementTag: CREATE INDEX
-    │       │       TargetMetadata:
-    │       │         SourceElementID: 1
-    │       │         SubWorkID: 1
-    │       │     TargetStatus: 2
+    │   └── • 5 Mutation operations
     │       │
     │       ├── • MakeValidatedSecondaryIndexPublic
     │       │     IndexID: 2
     │       │     TableID: 104
     │       │
+    │       ├── • RefreshStats
+    │       │     TableID: 104
+    │       │
     │       ├── • MakeWriteOnlyIndexDeleteOnly
     │       │     IndexID: 3
     │       │     TableID: 104

pkg/ccl/schemachangerccl/testdata/explain_verbose/create_index.rollback_5_of_7

@@ -76,30 +76,12 @@ EXPLAIN (ddl, verbose) rollback at post-commit stage 5 of 7;
     │   │       └── • skip PUBLIC → ABSENT operations
     │   │             rule: "skip index-column removal ops on index removal"
     │   │
-    │   └── • 6 Mutation operations
+    │   └── • 5 Mutation operations
     │       │
     │       ├── • MakeWriteOnlyIndexDeleteOnly
     │       │     IndexID: 3
     │       │     TableID: 104
     │       │
-    │       ├── • LogEvent
-    │       │     Element:
-    │       │       SecondaryIndex:
-    │       │         indexId: 2
-    │       │         sourceIndexId: 1
-    │       │         tableId: 104
-    │       │         temporaryIndexId: 3
-    │       │     EventBase:
-    │       │       Authorization:
-    │       │         UserName: root
-    │       │       Statement: CREATE INDEX ‹id1› ON ‹defaultdb›.public.‹t1› (‹id›, ‹name›) STORING
-    │       │         (‹money›) PARTITION BY LIST (‹id›) (PARTITION ‹p1› VALUES IN (‹1›))
-    │       │       StatementTag: CREATE INDEX
-    │       │       TargetMetadata:
-    │       │         SourceElementID: 1
-    │       │         SubWorkID: 1
-    │       │     TargetStatus: 1
-    │       │
     │       ├── • MakeWriteOnlyIndexDeleteOnly
     │       │     IndexID: 2
     │       │     TableID: 104

pkg/ccl/schemachangerccl/testdata/explain_verbose/create_index.rollback_6_of_7

@@ -76,30 +76,12 @@ EXPLAIN (ddl, verbose) rollback at post-commit stage 6 of 7;
     │   │       └── • skip PUBLIC → ABSENT operations
     │   │             rule: "skip index-column removal ops on index removal"
     │   │
-    │   └── • 6 Mutation operations
+    │   └── • 5 Mutation operations
     │       │
     │       ├── • MakeWriteOnlyIndexDeleteOnly
     │       │     IndexID: 3
     │       │     TableID: 104
     │       │
-    │       ├── • LogEvent
-    │       │     Element:
-    │       │       SecondaryIndex:
-    │       │         indexId: 2
-    │       │         sourceIndexId: 1
-    │       │         tableId: 104
-    │       │         temporaryIndexId: 3
-    │       │     EventBase:
-    │       │       Authorization:
-    │       │         UserName: root
-    │       │       Statement: CREATE INDEX ‹id1› ON ‹defaultdb›.public.‹t1› (‹id›, ‹name›) STORING
-    │       │         (‹money›) PARTITION BY LIST (‹id›) (PARTITION ‹p1› VALUES IN (‹1›))
-    │       │       StatementTag: CREATE INDEX
-    │       │       TargetMetadata:
-    │       │         SourceElementID: 1
-    │       │         SubWorkID: 1
-    │       │     TargetStatus: 1
-    │       │
     │       ├── • MakeWriteOnlyIndexDeleteOnly
     │       │     IndexID: 2
     │       │     TableID: 104

pkg/ccl/schemachangerccl/testdata/explain_verbose/create_index.rollback_7_of_7

@@ -76,25 +76,7 @@ EXPLAIN (ddl, verbose) rollback at post-commit stage 7 of 7;
     │   │       └── • skip PUBLIC → ABSENT operations
     │   │             rule: "skip index-column removal ops on index removal"
     │   │
-    │   └── • 6 Mutation operations
-    │       │
-    │       ├── • LogEvent
-    │       │     Element:
-    │       │       SecondaryIndex:
-    │       │         indexId: 2
-    │       │         sourceIndexId: 1
-    │       │         tableId: 104
-    │       │         temporaryIndexId: 3
-    │       │     EventBase:
-    │       │       Authorization:
-    │       │         UserName: root
-    │       │       Statement: CREATE INDEX ‹id1› ON ‹defaultdb›.public.‹t1› (‹id›, ‹name›) STORING
-    │       │         (‹money›) PARTITION BY LIST (‹id›) (PARTITION ‹p1› VALUES IN (‹1›))
-    │       │       StatementTag: CREATE INDEX
-    │       │       TargetMetadata:
-    │       │         SourceElementID: 1
-    │       │         SubWorkID: 1
-    │       │     TargetStatus: 1
+    │   └── • 5 Mutation operations
     │       │
     │       ├── • MakeWriteOnlyIndexDeleteOnly
     │       │     IndexID: 2