roachtest: codify longer ttl external storage buckets

Previously, the only codified external buckets for roachtests to back up to was
the `cockroachdb-backup-testing` buckets in s3 and gcs which each had a ttl of
1 day. This low ttl is not suitable for roachtests that produce backups that
the test failure investigator may want to inspect. This patch codifies the new
`cockroachdb-backup-testing-long-ttl` buckets in s3 and gcs, which currently
have a ttl of 20 days, the same ttl that team city artifacts have.

This patch also points the c2c, backup-restore/mixed-version, and
disagg-rebalance roachtests to use these new buckets.

Epic: none

Release note: none

build/teamcity/internal/cockroach/nightlies/private_roachtest.sh

@@ -15,5 +15,5 @@ export COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING=1
 export COCKROACH_NO_EXAMPLE_DATABASE=1
 export COCKROACH_AUTO_BALLAST=false
 
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD=gce -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e ROACHTEST_PRIVATE -e ROACHTEST_BUCKET -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e COCKROACH_DEV_LICENSE -e BACKUP_TESTING_BUCKET -e SFUSER -e SFPASSWORD -e COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING -e COCKROACH_NO_EXAMPLE_DATABASE -e COCKROACH_AUTO_BALLAST -e GCE_PROJECT" \
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD=gce -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e ROACHTEST_PRIVATE -e ROACHTEST_BUCKET -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e COCKROACH_DEV_LICENSE -e BACKUP_TESTING_BUCKET -e BACKUP_TESTING_BUCKET_LONG_TTL -e SFUSER -e SFPASSWORD -e COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING -e COCKROACH_NO_EXAMPLE_DATABASE -e  COCKROACH_AUTO_BALLAST -e GCE_PROJECT" \
 			       run_bazel build/teamcity/internal/cockroach/nightlies/private_roachtest_impl.sh

pkg/cmd/roachtest/tests/cluster_to_cluster.go

@@ -703,7 +703,7 @@ func (rd *replicationDriver) backupAfterFingerprintMismatch(
 	if rd.c.Spec().Cloud == spec.AWS {
 		prefix = "s3"
 	}
-	collection := fmt.Sprintf("%s://%s/c2c-fingerprint-mismatch/%s/%s/%s?AUTH=implicit", prefix, testutils.BackupTestingBucket(), rd.rs.name, rd.c.Name(), tenantName)
+	collection := fmt.Sprintf("%s://%s/c2c-fingerprint-mismatch/%s/%s/%s?AUTH=implicit", prefix, testutils.BackupTestingBucketLongTTL(), rd.rs.name, rd.c.Name(), tenantName)
 	fullBackupQuery := fmt.Sprintf("BACKUP INTO '%s' AS OF SYSTEM TIME '%s' with revision_history", collection, startTime.AsOfSystemTime())
 	_, err := conn.ExecContext(ctx, fullBackupQuery)
 	if err != nil {

pkg/cmd/roachtest/tests/disagg_rebalance.go

@@ -39,7 +39,7 @@ func registerDisaggRebalance(r registry.Registry) {
 				t.Skip("disagg-rebalance is only configured to run on AWS")
 			}
 			c.Put(ctx, t.Cockroach(), "./cockroach")
-			s3dir := fmt.Sprintf("s3://%s/disagg-rebalance/%s?AUTH=implicit", testutils.BackupTestingBucket(), c.Name())
+			s3dir := fmt.Sprintf("s3://%s/disagg-rebalance/%s?AUTH=implicit", testutils.BackupTestingBucketLongTTL(), c.Name())
 			startOpts := option.DefaultStartOptsNoBackups()
 			startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, fmt.Sprintf("--experimental-shared-storage=%s", s3dir))
 			c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Range(1, 3))

pkg/cmd/roachtest/tests/mixed_version_backup.go

@@ -946,11 +946,7 @@ func (bc *backupCollection) uri() string {
 	// global namespace represented by the BACKUP_TESTING_BUCKET
 	// bucket. The nonce allows multiple people (or TeamCity builds) to
 	// be running this test without interfering with one another.
-	gcsBackupTestingBucket := os.Getenv("BACKUP_TESTING_BUCKET")
-	if gcsBackupTestingBucket == "" {
-		gcsBackupTestingBucket = "cockroachdb-backup-testing"
-	}
-	return fmt.Sprintf("gs://"+gcsBackupTestingBucket+"/mixed-version/%s_%s?AUTH=implicit", bc.name, bc.nonce)
+	return fmt.Sprintf("gs://%s/mixed-version/%s_%s?AUTH=implicit", testutils.BackupTestingBucketLongTTL(), bc.name, bc.nonce)
 }
 
 func (bc *backupCollection) encryptionOption() *encryptionPassphrase {
@@ -2141,7 +2137,7 @@ func registerBackupMixedVersion(r registry.Registry) {
 		RequiresLicense:   true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			if c.Spec().Cloud != spec.GCE && !c.IsLocal() {
-				t.Skip("uses gs://cockroachdb-backup-testing; see https://github.com/cockroachdb/cockroach/issues/105968")
+				t.Skip("uses gs://cockroachdb-backup-testing-long-ttl; see https://github.com/cockroachdb/cockroach/issues/105968")
 			}
 
 			roachNodes := c.Range(1, c.Spec().NodeCount-1)

pkg/testutils/backup.go

@@ -13,18 +13,35 @@ package testutils
 import "os"
 
 const (
-	defaultBackupBucket       = "cockroachdb-backup-testing"
-	backupTestingBucketEnvVar = "BACKUP_TESTING_BUCKET"
+	defaultBackupBucket              = "cockroachdb-backup-testing"
+	longTTLBackupTestingBucket       = "cockroachdb-backup-testing-long-ttl"
+	backupTestingBucketEnvVar        = "BACKUP_TESTING_BUCKET"
+	backupTestingBucketLongTTLEnvVar = "BACKUP_TESTING_BUCKET_LONG_TTL"
 )
 
-// BackupTestingBucket returns the name of the GCS bucket that should
-// be used in a test run. Most times, this will be the regular public
-// bucket. In private test runs, the name of the bucket is passed
-// through an environment variable.
+// BackupTestingBucket returns the name of the external storage bucket that
+// should be used in a test run. Most times, this will be the regular public
+// bucket. In private test runs, the name of the bucket is passed through an
+// environment variable.
 func BackupTestingBucket() string {
 	if bucket := os.Getenv(backupTestingBucketEnvVar); bucket != "" {
 		return bucket
 	}
 
 	return defaultBackupBucket
 }
+
+// BackupTestingBucketLongTTL returns the name of the external storage bucket
+// that should be used in a test run where the bucket's content may inform a
+// debugging investigation. At the time of this comment, the ttl for the s3 and
+// gcs buckets is 20 days.
+//
+// In private test runs, the name of the bucket is passed through an environment
+// variable.
+func BackupTestingBucketLongTTL() string {
+	if bucket := os.Getenv(backupTestingBucketLongTTLEnvVar); bucket != "" {
+		return bucket
+	}
+
+	return longTTLBackupTestingBucket
+}