Merge cockroachdb#92971 cockroachdb#93551 cockroachdb#93552 cockroach…

…db#93682

92971: sql/schemachanger: enable drop index by default r=fqazi a=fqazi

This patch will enable drop index by default in the declarative schema changer with the following fixes:

Fixes: cockroachdb#93070

1. Return proper index names for errors
2. Some index notices should only be generated when work is done
3. Views and indexes should have to be cleaned together when the index becomes invisible.
4. Drop index should return correct errors in system tables
5. Enable drop index by default and update logic tests

Current limitations for falling back are if there are foreign key constraint dependencies or zone configs

93551: sql: add replication start time to `SHOW TENANT WITH REPLICATION STATUS` r=lidorcarmel a=adityamaru

The start time is a useful piece of information to expose to the end user. It is the lower bound for the data we have replicated.

Informs: cockroachdb#93447

Release note: None

93552: changefeedccl: increase chaos roachtest latency limits r=samiskin a=samiskin

Resolves: cockroachdb#93238

Since our job-level retry MaxBackoff has increased from 10 seconds to 10 minutes, increase the chaos test latency limits.

Release note: None

93682: ci: update bazel builder image r=rickystewart a=cockroach-teamcity

Release note: None
Epic: None


Co-authored-by: Faizan Qazi <[email protected]>
Co-authored-by: adityamaru <[email protected]>
Co-authored-by: Shiranka Miskin <[email protected]>
Co-authored-by: cockroach-teamcity <[email protected]>

build/.bazelbuilderversion

@@ -1 +1 @@
-cockroachdb/bazel:20220328-163955
+cockroachdb/bazel:20221215-030250

pkg/ccl/changefeedccl/BUILD.bazel

@@ -115,6 +115,7 @@ go_library(
         "//pkg/util/ctxgroup",
         "//pkg/util/duration",
         "//pkg/util/encoding/csv",
+        "//pkg/util/envutil",
         "//pkg/util/hlc",
         "//pkg/util/httputil",
         "//pkg/util/humanizeutil",

pkg/ccl/changefeedccl/retry.go

@@ -12,11 +12,13 @@ import (
 	"context"
 	"time"
 
+	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 )
 
-var useFastRetry = false
+var useFastRetry = envutil.EnvOrDefaultBool(
+	"COCKROACH_CHANGEFEED_TESTING_FAST_RETRY", false)
 
 // getRetry returns retry object for changefeed.
 func getRetry(ctx context.Context) Retry {
@@ -30,7 +32,7 @@ func getRetry(ctx context.Context) Retry {
 		opts = retry.Options{
 			InitialBackoff: 5 * time.Millisecond,
 			Multiplier:     2,
-			MaxBackoff:     250 * time.Minute,
+			MaxBackoff:     250 * time.Millisecond,
 		}
 	}
 

pkg/ccl/schemachangerccl/testdata/explain/create_index.rollback_1_of_7

@@ -24,9 +24,8 @@ Schema change plan for rolling back CREATE INDEX ‹id1› ON ‹defaultdb›.pu
            │    ├── PUBLIC           → ABSENT IndexColumn:{DescID: 104, ColumnID: 1, IndexID: 3}
            │    ├── PUBLIC           → ABSENT IndexColumn:{DescID: 104, ColumnID: 2, IndexID: 3}
            │    └── PUBLIC           → ABSENT IndexColumn:{DescID: 104, ColumnID: 3, IndexID: 3}
-           └── 7 Mutation operations
+           └── 6 Mutation operations
                 ├── SetIndexName {"IndexID":2,"Name":"crdb_internal_in...","TableID":104}
-                ├── LogEvent {"TargetStatus":1}
                 ├── MakeIndexAbsent {"IndexID":2,"TableID":104}
                 ├── CreateGCJobForIndex {"IndexID":2,"TableID":104}
                 ├── MakeIndexAbsent {"IndexID":3,"TableID":104}

pkg/ccl/schemachangerccl/testdata/explain/create_index.rollback_2_of_7

@@ -23,10 +23,9 @@ Schema change plan for rolling back CREATE INDEX ‹id1› ON ‹defaultdb›.pu
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 1, IndexID: 3}
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 2, IndexID: 3}
       │    │    └── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 3, IndexID: 3}
-      │    └── 6 Mutation operations
+      │    └── 5 Mutation operations
       │         ├── SetIndexName {"IndexID":2,"Name":"crdb_internal_in...","TableID":104}
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":3,"TableID":104}
-      │         ├── LogEvent {"TargetStatus":1}
       │         ├── MakeIndexAbsent {"IndexID":2,"TableID":104}
       │         ├── SetJobStateOnDescriptor {"DescriptorID":104}
       │         └── UpdateSchemaChangerJob {"IsNonCancelable":true,"RunningStatus":"PostCommitNonRev..."}

pkg/ccl/schemachangerccl/testdata/explain/create_index.rollback_3_of_7

@@ -23,10 +23,9 @@ Schema change plan for rolling back CREATE INDEX ‹id1› ON ‹defaultdb›.pu
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 1, IndexID: 3}
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 2, IndexID: 3}
       │    │    └── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 3, IndexID: 3}
-      │    └── 6 Mutation operations
+      │    └── 5 Mutation operations
       │         ├── SetIndexName {"IndexID":2,"Name":"crdb_internal_in...","TableID":104}
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":3,"TableID":104}
-      │         ├── LogEvent {"TargetStatus":1}
       │         ├── MakeIndexAbsent {"IndexID":2,"TableID":104}
       │         ├── SetJobStateOnDescriptor {"DescriptorID":104}
       │         └── UpdateSchemaChangerJob {"IsNonCancelable":true,"RunningStatus":"PostCommitNonRev..."}

pkg/ccl/schemachangerccl/testdata/explain/create_index.rollback_4_of_7

@@ -23,10 +23,9 @@ Schema change plan for rolling back CREATE INDEX ‹id1› ON ‹defaultdb›.pu
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 1, IndexID: 3}
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 2, IndexID: 3}
       │    │    └── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 3, IndexID: 3}
-      │    └── 6 Mutation operations
+      │    └── 5 Mutation operations
       │         ├── SetIndexName {"IndexID":2,"Name":"crdb_internal_in...","TableID":104}
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":3,"TableID":104}
-      │         ├── LogEvent {"TargetStatus":1}
       │         ├── MakeIndexAbsent {"IndexID":2,"TableID":104}
       │         ├── SetJobStateOnDescriptor {"DescriptorID":104}
       │         └── UpdateSchemaChangerJob {"IsNonCancelable":true,"RunningStatus":"PostCommitNonRev..."}

pkg/ccl/schemachangerccl/testdata/explain/create_index.rollback_5_of_7

@@ -23,10 +23,11 @@ Schema change plan for rolling back CREATE INDEX ‹id1› ON ‹defaultdb›.pu
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 1, IndexID: 3}
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 2, IndexID: 3}
       │    │    └── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 3, IndexID: 3}
-      │    └── 5 Mutation operations
-      │         ├── SetIndexName {"IndexID":2,"Name":"crdb_internal_in...","TableID":104}
+      │    └── 6 Mutation operations
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":3,"TableID":104}
+      │         ├── LogEvent {"TargetStatus":1}
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":2,"TableID":104}
+      │         ├── SetIndexName {"IndexID":2,"Name":"crdb_internal_in...","TableID":104}
       │         ├── SetJobStateOnDescriptor {"DescriptorID":104}
       │         └── UpdateSchemaChangerJob {"IsNonCancelable":true,"RunningStatus":"PostCommitNonRev..."}
       └── Stage 2 of 2 in PostCommitNonRevertiblePhase
@@ -35,8 +36,7 @@ Schema change plan for rolling back CREATE INDEX ‹id1› ON ‹defaultdb›.pu
            │    ├── PUBLIC      → ABSENT IndexData:{DescID: 104, IndexID: 2}
            │    ├── DELETE_ONLY → ABSENT TemporaryIndex:{DescID: 104, IndexID: 3, ConstraintID: 1, SourceIndexID: 1}
            │    └── PUBLIC      → ABSENT IndexData:{DescID: 104, IndexID: 3}
-           └── 7 Mutation operations
-                ├── LogEvent {"TargetStatus":1}
+           └── 6 Mutation operations
                 ├── MakeIndexAbsent {"IndexID":2,"TableID":104}
                 ├── CreateGCJobForIndex {"IndexID":2,"TableID":104}
                 ├── MakeIndexAbsent {"IndexID":3,"TableID":104}

pkg/ccl/schemachangerccl/testdata/explain/create_index.rollback_6_of_7

@@ -23,10 +23,11 @@ Schema change plan for rolling back CREATE INDEX ‹id1› ON ‹defaultdb›.pu
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 1, IndexID: 3}
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 2, IndexID: 3}
       │    │    └── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 3, IndexID: 3}
-      │    └── 5 Mutation operations
-      │         ├── SetIndexName {"IndexID":2,"Name":"crdb_internal_in...","TableID":104}
+      │    └── 6 Mutation operations
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":3,"TableID":104}
+      │         ├── LogEvent {"TargetStatus":1}
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":2,"TableID":104}
+      │         ├── SetIndexName {"IndexID":2,"Name":"crdb_internal_in...","TableID":104}
       │         ├── SetJobStateOnDescriptor {"DescriptorID":104}
       │         └── UpdateSchemaChangerJob {"IsNonCancelable":true,"RunningStatus":"PostCommitNonRev..."}
       └── Stage 2 of 2 in PostCommitNonRevertiblePhase
@@ -35,8 +36,7 @@ Schema change plan for rolling back CREATE INDEX ‹id1› ON ‹defaultdb›.pu
            │    ├── PUBLIC      → ABSENT IndexData:{DescID: 104, IndexID: 2}
            │    ├── DELETE_ONLY → ABSENT TemporaryIndex:{DescID: 104, IndexID: 3, ConstraintID: 1, SourceIndexID: 1}
            │    └── PUBLIC      → ABSENT IndexData:{DescID: 104, IndexID: 3}
-           └── 7 Mutation operations
-                ├── LogEvent {"TargetStatus":1}
+           └── 6 Mutation operations
                 ├── MakeIndexAbsent {"IndexID":2,"TableID":104}
                 ├── CreateGCJobForIndex {"IndexID":2,"TableID":104}
                 ├── MakeIndexAbsent {"IndexID":3,"TableID":104}

pkg/ccl/schemachangerccl/testdata/explain/create_index.rollback_7_of_7

@@ -23,7 +23,8 @@ Schema change plan for rolling back CREATE INDEX ‹id1› ON ‹defaultdb›.pu
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 1, IndexID: 3}
       │    │    ├── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 2, IndexID: 3}
       │    │    └── PUBLIC           → ABSENT      IndexColumn:{DescID: 104, ColumnID: 3, IndexID: 3}
-      │    └── 5 Mutation operations
+      │    └── 6 Mutation operations
+      │         ├── LogEvent {"TargetStatus":1}
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":2,"TableID":104}
       │         ├── SetIndexName {"IndexID":2,"Name":"crdb_internal_in...","TableID":104}
       │         ├── MakeWriteOnlyIndexDeleteOnly {"IndexID":3,"TableID":104}
@@ -35,8 +36,7 @@ Schema change plan for rolling back CREATE INDEX ‹id1› ON ‹defaultdb›.pu
            │    ├── PUBLIC      → ABSENT IndexData:{DescID: 104, IndexID: 2}
            │    ├── DELETE_ONLY → ABSENT TemporaryIndex:{DescID: 104, IndexID: 3, ConstraintID: 1, SourceIndexID: 1}
            │    └── PUBLIC      → ABSENT IndexData:{DescID: 104, IndexID: 3}
-           └── 7 Mutation operations
-                ├── LogEvent {"TargetStatus":1}
+           └── 6 Mutation operations
                 ├── MakeIndexAbsent {"IndexID":2,"TableID":104}
                 ├── CreateGCJobForIndex {"IndexID":2,"TableID":104}
                 ├── MakeIndexAbsent {"IndexID":3,"TableID":104}

pkg/ccl/schemachangerccl/testdata/explain_verbose/create_index.rollback_1_of_7

@@ -65,7 +65,10 @@ EXPLAIN (ddl, verbose) rollback at post-commit stage 1 of 7;
         │   │         rule: "index removed before garbage collection"
         │   │
         │   ├── • IndexName:{DescID: 104, Name: id1, IndexID: 2}
-        │   │     PUBLIC → ABSENT
+        │   │   │ PUBLIC → ABSENT
+        │   │   │
+        │   │   └── • Precedence dependency from DELETE_ONLY SecondaryIndex:{DescID: 104, IndexID: 2, ConstraintID: 0, TemporaryIndexID: 3, SourceIndexID: 1}
+        │   │         rule: "index no longer public before index name"
         │   │
         │   ├── • TemporaryIndex:{DescID: 104, IndexID: 3, ConstraintID: 1, SourceIndexID: 1}
         │   │   │ DELETE_ONLY → ABSENT
@@ -106,31 +109,13 @@ EXPLAIN (ddl, verbose) rollback at post-commit stage 1 of 7;
         │       └── • skip PUBLIC → ABSENT operations
         │             rule: "skip index-column removal ops on index removal"
         │
-        └── • 7 Mutation operations
+        └── • 6 Mutation operations
             │
             ├── • SetIndexName
             │     IndexID: 2
             │     Name: crdb_internal_index_2_name_placeholder
             │     TableID: 104
             │
-            ├── • LogEvent
-            │     Element:
-            │       SecondaryIndex:
-            │         indexId: 2
-            │         sourceIndexId: 1
-            │         tableId: 104
-            │         temporaryIndexId: 3
-            │     EventBase:
-            │       Authorization:
-            │         UserName: root
-            │       Statement: CREATE INDEX ‹id1› ON ‹defaultdb›.public.‹t1› (‹id›, ‹name›) STORING
-            │         (‹money›) PARTITION BY LIST (‹id›) (PARTITION ‹p1› VALUES IN (‹1›))
-            │       StatementTag: CREATE INDEX
-            │       TargetMetadata:
-            │         SourceElementID: 1
-            │         SubWorkID: 1
-            │     TargetStatus: 1
-            │
             ├── • MakeIndexAbsent
             │     IndexID: 2
             │     TableID: 104

pkg/ccl/schemachangerccl/testdata/explain_verbose/create_index.rollback_2_of_7

@@ -59,7 +59,10 @@ EXPLAIN (ddl, verbose) rollback at post-commit stage 2 of 7;
     │   │   │         rule: "skip index-column removal ops on index removal"
     │   │   │
     │   │   ├── • IndexName:{DescID: 104, Name: id1, IndexID: 2}
-    │   │   │     PUBLIC → ABSENT
+    │   │   │   │ PUBLIC → ABSENT
+    │   │   │   │
+    │   │   │   └── • Precedence dependency from DELETE_ONLY SecondaryIndex:{DescID: 104, IndexID: 2, ConstraintID: 0, TemporaryIndexID: 3, SourceIndexID: 1}
+    │   │   │         rule: "index no longer public before index name"
     │   │   │
     │   │   ├── • TemporaryIndex:{DescID: 104, IndexID: 3, ConstraintID: 1, SourceIndexID: 1}
     │   │   │   │ WRITE_ONLY → DELETE_ONLY
@@ -88,7 +91,7 @@ EXPLAIN (ddl, verbose) rollback at post-commit stage 2 of 7;
     │   │       └── • skip PUBLIC → ABSENT operations
     │   │             rule: "skip index-column removal ops on index removal"
     │   │
-    │   └── • 6 Mutation operations
+    │   └── • 5 Mutation operations
     │       │
     │       ├── • SetIndexName
     │       │     IndexID: 2
@@ -99,24 +102,6 @@ EXPLAIN (ddl, verbose) rollback at post-commit stage 2 of 7;
     │       │     IndexID: 3
     │       │     TableID: 104
     │       │
-    │       ├── • LogEvent
-    │       │     Element:
-    │       │       SecondaryIndex:
-    │       │         indexId: 2
-    │       │         sourceIndexId: 1
-    │       │         tableId: 104
-    │       │         temporaryIndexId: 3
-    │       │     EventBase:
-    │       │       Authorization:
-    │       │         UserName: root
-    │       │       Statement: CREATE INDEX ‹id1› ON ‹defaultdb›.public.‹t1› (‹id›, ‹name›) STORING
-    │       │         (‹money›) PARTITION BY LIST (‹id›) (PARTITION ‹p1› VALUES IN (‹1›))
-    │       │       StatementTag: CREATE INDEX
-    │       │       TargetMetadata:
-    │       │         SourceElementID: 1
-    │       │         SubWorkID: 1
-    │       │     TargetStatus: 1
-    │       │
     │       ├── • MakeIndexAbsent
     │       │     IndexID: 2
     │       │     TableID: 104

pkg/ccl/schemachangerccl/testdata/explain_verbose/create_index.rollback_3_of_7

@@ -59,7 +59,10 @@ EXPLAIN (ddl, verbose) rollback at post-commit stage 3 of 7;
     │   │   │         rule: "skip index-column removal ops on index removal"
     │   │   │
     │   │   ├── • IndexName:{DescID: 104, Name: id1, IndexID: 2}
-    │   │   │     PUBLIC → ABSENT
+    │   │   │   │ PUBLIC → ABSENT
+    │   │   │   │
+    │   │   │   └── • Precedence dependency from DELETE_ONLY SecondaryIndex:{DescID: 104, IndexID: 2, ConstraintID: 0, TemporaryIndexID: 3, SourceIndexID: 1}
+    │   │   │         rule: "index no longer public before index name"
     │   │   │
     │   │   ├── • TemporaryIndex:{DescID: 104, IndexID: 3, ConstraintID: 1, SourceIndexID: 1}
     │   │   │   │ WRITE_ONLY → DELETE_ONLY
@@ -88,7 +91,7 @@ EXPLAIN (ddl, verbose) rollback at post-commit stage 3 of 7;
     │   │       └── • skip PUBLIC → ABSENT operations
     │   │             rule: "skip index-column removal ops on index removal"
     │   │
-    │   └── • 6 Mutation operations
+    │   └── • 5 Mutation operations
     │       │
     │       ├── • SetIndexName
     │       │     IndexID: 2
@@ -99,24 +102,6 @@ EXPLAIN (ddl, verbose) rollback at post-commit stage 3 of 7;
     │       │     IndexID: 3
     │       │     TableID: 104
     │       │
-    │       ├── • LogEvent
-    │       │     Element:
-    │       │       SecondaryIndex:
-    │       │         indexId: 2
-    │       │         sourceIndexId: 1
-    │       │         tableId: 104
-    │       │         temporaryIndexId: 3
-    │       │     EventBase:
-    │       │       Authorization:
-    │       │         UserName: root
-    │       │       Statement: CREATE INDEX ‹id1› ON ‹defaultdb›.public.‹t1› (‹id›, ‹name›) STORING
-    │       │         (‹money›) PARTITION BY LIST (‹id›) (PARTITION ‹p1› VALUES IN (‹1›))
-    │       │       StatementTag: CREATE INDEX
-    │       │       TargetMetadata:
-    │       │         SourceElementID: 1
-    │       │         SubWorkID: 1
-    │       │     TargetStatus: 1
-    │       │
     │       ├── • MakeIndexAbsent
     │       │     IndexID: 2
     │       │     TableID: 104