From 2bb02ad33364ec2463d6485a9572c9e7e4ddd7dc Mon Sep 17 00:00:00 2001 From: Michael Butler Date: Fri, 8 Sep 2023 15:28:33 -0400 Subject: [PATCH] roachtest: codify longer ttl external storage buckets Previously, the only codified external buckets for roachtests to back up to was the `cockroachdb-backup-testing` buckets in s3 and gcs which each had a ttl of 1 day. This low ttl is not suitable for roachtests that produce backups that the test failure investigator may want to inspect. This patch codifies the new `cockroachdb-backup-testing-long-ttl` buckets in s3 and gcs, which currently have a ttl of 20 days, the same ttl that team city artifacts have. This patch also points the c2c, backup-restore/mixed-version, and disagg-rebalance roachtests to use these new buckets. Epic: none Release note: none --- .../cockroach/nightlies/private_roachtest.sh | 2 +- pkg/cmd/roachtest/tests/cluster_to_cluster.go | 2 +- pkg/cmd/roachtest/tests/disagg_rebalance.go | 2 +- .../roachtest/tests/mixed_version_backup.go | 8 ++--- pkg/testutils/backup.go | 29 +++++++++++++++---- 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/build/teamcity/internal/cockroach/nightlies/private_roachtest.sh b/build/teamcity/internal/cockroach/nightlies/private_roachtest.sh index 1098e44af316..bf1c4ad37b7e 100755 --- a/build/teamcity/internal/cockroach/nightlies/private_roachtest.sh +++ b/build/teamcity/internal/cockroach/nightlies/private_roachtest.sh @@ -15,5 +15,5 @@ export COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING=1 export COCKROACH_NO_EXAMPLE_DATABASE=1 export COCKROACH_AUTO_BALLAST=false -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD=gce -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e ROACHTEST_PRIVATE -e ROACHTEST_BUCKET -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e COCKROACH_DEV_LICENSE -e BACKUP_TESTING_BUCKET -e SFUSER -e SFPASSWORD -e COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING -e COCKROACH_NO_EXAMPLE_DATABASE -e COCKROACH_AUTO_BALLAST -e GCE_PROJECT" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD=gce -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e ROACHTEST_PRIVATE -e ROACHTEST_BUCKET -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e COCKROACH_DEV_LICENSE -e BACKUP_TESTING_BUCKET -e BACKUP_TESTING_BUCKET_LONG_TTL -e SFUSER -e SFPASSWORD -e COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING -e COCKROACH_NO_EXAMPLE_DATABASE -e COCKROACH_AUTO_BALLAST -e GCE_PROJECT" \ run_bazel build/teamcity/internal/cockroach/nightlies/private_roachtest_impl.sh diff --git a/pkg/cmd/roachtest/tests/cluster_to_cluster.go b/pkg/cmd/roachtest/tests/cluster_to_cluster.go index 72c88a8b3fb0..a0bd0b1a5369 100644 --- a/pkg/cmd/roachtest/tests/cluster_to_cluster.go +++ b/pkg/cmd/roachtest/tests/cluster_to_cluster.go @@ -703,7 +703,7 @@ func (rd *replicationDriver) backupAfterFingerprintMismatch( if rd.c.Spec().Cloud == spec.AWS { prefix = "s3" } - collection := fmt.Sprintf("%s://%s/c2c-fingerprint-mismatch/%s/%s/%s?AUTH=implicit", prefix, testutils.BackupTestingBucket(), rd.rs.name, rd.c.Name(), tenantName) + collection := fmt.Sprintf("%s://%s/c2c-fingerprint-mismatch/%s/%s/%s?AUTH=implicit", prefix, testutils.BackupTestingBucketLongTTL(), rd.rs.name, rd.c.Name(), tenantName) fullBackupQuery := fmt.Sprintf("BACKUP INTO '%s' AS OF SYSTEM TIME '%s' with revision_history", collection, startTime.AsOfSystemTime()) _, err := conn.ExecContext(ctx, fullBackupQuery) if err != nil { diff --git a/pkg/cmd/roachtest/tests/disagg_rebalance.go b/pkg/cmd/roachtest/tests/disagg_rebalance.go index ecdd199c8826..fd2bab6dd25d 100644 --- a/pkg/cmd/roachtest/tests/disagg_rebalance.go +++ b/pkg/cmd/roachtest/tests/disagg_rebalance.go @@ -39,7 +39,7 @@ func registerDisaggRebalance(r registry.Registry) { t.Skip("disagg-rebalance is only configured to run on AWS") } c.Put(ctx, t.Cockroach(), "./cockroach") - s3dir := fmt.Sprintf("s3://%s/disagg-rebalance/%s?AUTH=implicit", testutils.BackupTestingBucket(), c.Name()) + s3dir := fmt.Sprintf("s3://%s/disagg-rebalance/%s?AUTH=implicit", testutils.BackupTestingBucketLongTTL(), c.Name()) startOpts := option.DefaultStartOptsNoBackups() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, fmt.Sprintf("--experimental-shared-storage=%s", s3dir)) c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Range(1, 3)) diff --git a/pkg/cmd/roachtest/tests/mixed_version_backup.go b/pkg/cmd/roachtest/tests/mixed_version_backup.go index 853fcc6fc232..9b98ab198fce 100644 --- a/pkg/cmd/roachtest/tests/mixed_version_backup.go +++ b/pkg/cmd/roachtest/tests/mixed_version_backup.go @@ -946,11 +946,7 @@ func (bc *backupCollection) uri() string { // global namespace represented by the BACKUP_TESTING_BUCKET // bucket. The nonce allows multiple people (or TeamCity builds) to // be running this test without interfering with one another. - gcsBackupTestingBucket := os.Getenv("BACKUP_TESTING_BUCKET") - if gcsBackupTestingBucket == "" { - gcsBackupTestingBucket = "cockroachdb-backup-testing" - } - return fmt.Sprintf("gs://"+gcsBackupTestingBucket+"/mixed-version/%s_%s?AUTH=implicit", bc.name, bc.nonce) + return fmt.Sprintf("gs://%s/mixed-version/%s_%s?AUTH=implicit", testutils.BackupTestingBucketLongTTL(), bc.name, bc.nonce) } func (bc *backupCollection) encryptionOption() *encryptionPassphrase { @@ -2141,7 +2137,7 @@ func registerBackupMixedVersion(r registry.Registry) { RequiresLicense: true, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { if c.Spec().Cloud != spec.GCE && !c.IsLocal() { - t.Skip("uses gs://cockroachdb-backup-testing; see https://github.com/cockroachdb/cockroach/issues/105968") + t.Skip("uses gs://cockroachdb-backup-testing-long-ttl; see https://github.com/cockroachdb/cockroach/issues/105968") } roachNodes := c.Range(1, c.Spec().NodeCount-1) diff --git a/pkg/testutils/backup.go b/pkg/testutils/backup.go index d51dc2fb9294..399b2eeec5e1 100644 --- a/pkg/testutils/backup.go +++ b/pkg/testutils/backup.go @@ -13,14 +13,16 @@ package testutils import "os" const ( - defaultBackupBucket = "cockroachdb-backup-testing" - backupTestingBucketEnvVar = "BACKUP_TESTING_BUCKET" + defaultBackupBucket = "cockroachdb-backup-testing" + longTTLBackupTestingBucket = "cockroachdb-backup-testing-long-ttl" + backupTestingBucketEnvVar = "BACKUP_TESTING_BUCKET" + backupTestingBucketLongTTLEnvVar = "BACKUP_TESTING_BUCKET_LONG_TTL" ) -// BackupTestingBucket returns the name of the GCS bucket that should -// be used in a test run. Most times, this will be the regular public -// bucket. In private test runs, the name of the bucket is passed -// through an environment variable. +// BackupTestingBucket returns the name of the external storage bucket that +// should be used in a test run. Most times, this will be the regular public +// bucket. In private test runs, the name of the bucket is passed through an +// environment variable. func BackupTestingBucket() string { if bucket := os.Getenv(backupTestingBucketEnvVar); bucket != "" { return bucket @@ -28,3 +30,18 @@ func BackupTestingBucket() string { return defaultBackupBucket } + +// BackupTestingBucketLongTTL returns the name of the external storage bucket +// that should be used in a test run where the bucket's content may inform a +// debugging investigation. At the time of this comment, the ttl for the s3 and +// gcs buckets is 20 days. +// +// In private test runs, the name of the bucket is passed through an environment +// variable. +func BackupTestingBucketLongTTL() string { + if bucket := os.Getenv(backupTestingBucketLongTTLEnvVar); bucket != "" { + return bucket + } + + return longTTLBackupTestingBucket +}