changefeedccl: increase chaos roachtest latency limits

Resolves: cockroachdb#93238

Since our job-level retry MaxBackoff has increased from 10 seconds to 10 minutes, increase the chaos test latency limits.

Release note: None

<what was there before: Previously, ...>
<why it needed to change: This was inadequate because ...>
<what you did about it: To address this, this patch ...>

pkg/ccl/changefeedccl/BUILD.bazel

@@ -115,6 +115,7 @@ go_library(
         "//pkg/util/ctxgroup",
         "//pkg/util/duration",
         "//pkg/util/encoding/csv",
+        "//pkg/util/envutil",
         "//pkg/util/hlc",
         "//pkg/util/httputil",
         "//pkg/util/humanizeutil",

pkg/ccl/changefeedccl/retry.go

@@ -12,11 +12,13 @@ import (
 	"context"
 	"time"
 
+	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 )
 
-var useFastRetry = false
+var useFastRetry = envutil.EnvOrDefaultBool(
+	"COCKROACH_CHANGEFEED_TESTING_FAST_RETRY", false)
 
 // getRetry returns retry object for changefeed.
 func getRetry(ctx context.Context) Retry {
@@ -30,7 +32,7 @@ func getRetry(ctx context.Context) Retry {
 		opts = retry.Options{
 			InitialBackoff: 5 * time.Millisecond,
 			Multiplier:     2,
-			MaxBackoff:     250 * time.Minute,
+			MaxBackoff:     250 * time.Millisecond,
 		}
 	}
 

pkg/cmd/roachtest/tests/cdc.go

@@ -483,7 +483,7 @@ func newCDCTester(ctx context.Context, t test.Test, c cluster.Cluster) cdcTester
 	c.Put(ctx, t.Cockroach(), "./cockroach")
 
 	settings := install.MakeClusterSettings()
-	settings.Env = append(settings.Env, "COCKROACH_EXPERIMENTAL_ENABLE_PER_CHANGEFEED_METRICS=true")
+	settings.Env = append(settings.Env, "COCKROACH_CHANGEFEED_TESTING_FAST_RETRY=true")
 	c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, tester.crdbNodes)
 
 	c.Put(ctx, t.DeprecatedWorkload(), "./workload", tester.workloadNode)