Skip to content

Commit

Permalink
roachtest: provision 250 MB/s for restore tests on AWS
Browse files Browse the repository at this point in the history
The restore/tpce/* family of tests on AWS max out the default 125 MB/s EBS
throughput. In contrast, similar tests in GCE provision for more throughput and
don't max it out.

This commit bumps the provisioned throughput from 125 MB/s to 250 MB/s in all
restore tests on AWS, so that the tests don't work at the edge of overload.

This both brings some parity between testing on GCE and AWS, and reduces
likelyhood of raft OOMs (which manifest more often when disk is overloaded).

Fixes #106248
Epic: none
Release note: none
  • Loading branch information
pav-kv committed Aug 22, 2023
1 parent 211fc93 commit 93aa9b7
Showing 1 changed file with 9 additions and 8 deletions.
17 changes: 9 additions & 8 deletions pkg/cmd/roachtest/tests/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ func registerRestore(r registry.Registry) {
PrometheusNameSpace, Subsystem: "restore", Name: "duration"}, []string{"test_name"})

withPauseSpecs := restoreSpecs{
hardware: makeHardwareSpecs(hardwareSpecs{}),
hardware: makeHardwareSpecs(hardwareSpecs{ebsThroughput: 250 /* MB/s */}),
backup: makeBackupSpecs(
backupSpecs{workload: tpceRestore{customers: 1000},
version: "v22.2.1"}),
Expand Down Expand Up @@ -268,7 +268,7 @@ func registerRestore(r registry.Registry) {

for _, sp := range []restoreSpecs{
{
hardware: makeHardwareSpecs(hardwareSpecs{}),
hardware: makeHardwareSpecs(hardwareSpecs{ebsThroughput: 250 /* MB/s */}),
backup: makeBackupSpecs(backupSpecs{}),
timeout: 1 * time.Hour,
},
Expand All @@ -289,30 +289,30 @@ func registerRestore(r registry.Registry) {
{
// Benchmarks if per node throughput remains constant if the number of
// nodes doubles relative to default.
hardware: makeHardwareSpecs(hardwareSpecs{nodes: 8}),
hardware: makeHardwareSpecs(hardwareSpecs{nodes: 8, ebsThroughput: 250 /* MB/s */}),
backup: makeBackupSpecs(backupSpecs{}),
timeout: 1 * time.Hour,
},
{
// Benchmarks if per node throughput remains constant if the cluster
// is multi-region.
hardware: makeHardwareSpecs(hardwareSpecs{
nodes: 9,
nodes: 9, ebsThroughput: 250, /* MB/s */
zones: []string{"us-east-2b", "us-west-2b", "eu-west-1b"}}), // These zones are AWS-specific.
backup: makeBackupSpecs(backupSpecs{cloud: spec.AWS}),
timeout: 90 * time.Minute,
},
{
// Benchmarks if per node throughput doubles if the vcpu count doubles
// relative to default.
hardware: makeHardwareSpecs(hardwareSpecs{cpus: 16}),
hardware: makeHardwareSpecs(hardwareSpecs{cpus: 16, ebsThroughput: 250 /* MB/s */}),
backup: makeBackupSpecs(backupSpecs{}),
timeout: 1 * time.Hour,
},
{
// Ensures we can restore a 48 length incremental chain.
// Also benchmarks per node throughput for a long chain.
hardware: makeHardwareSpecs(hardwareSpecs{}),
hardware: makeHardwareSpecs(hardwareSpecs{ebsThroughput: 250 /* MB/s */}),
backup: makeBackupSpecs(backupSpecs{backupsIncluded: 48}),
timeout: 1 * time.Hour,
},
Expand All @@ -329,7 +329,8 @@ func registerRestore(r registry.Registry) {
},
{
// The weekly 32TB Restore test.
hardware: makeHardwareSpecs(hardwareSpecs{nodes: 15, cpus: 16, volumeSize: 5000}),
hardware: makeHardwareSpecs(hardwareSpecs{nodes: 15, cpus: 16, volumeSize: 5000,
ebsThroughput: 250 /* MB/s */}),
backup: makeBackupSpecs(backupSpecs{
version: "v22.2.1",
workload: tpceRestore{customers: 2000000}}),
Expand All @@ -338,7 +339,7 @@ func registerRestore(r registry.Registry) {
},
{
// A teeny weeny 15GB restore that could be used to bisect scale agnostic perf regressions.
hardware: makeHardwareSpecs(hardwareSpecs{}),
hardware: makeHardwareSpecs(hardwareSpecs{ebsThroughput: 250 /* MB/s */}),
backup: makeBackupSpecs(
backupSpecs{workload: tpceRestore{customers: 1000},
version: "v22.2.1"}),
Expand Down

0 comments on commit 93aa9b7

Please sign in to comment.