Skip to content

Commit

Permalink
cmd/roachtest: adjust disk-stalled roachtests TPS calculation
Browse files Browse the repository at this point in the history
Previously, the post-stall TPS calculation included the time that the node was
stalled but before the stall triggered the node's exit. During this period,
overall TPS drops until the gray failure is converted into a hard failure. This
commit adjusts the post-stall TPS calculation to exclude the stalled time when
TPS is expected to tank.

Epic: None
Informs: #97705.
Release note: None
  • Loading branch information
jbowens committed Apr 6, 2023
1 parent 47d9abf commit 0dc1f64
Showing 1 changed file with 7 additions and 9 deletions.
16 changes: 7 additions & 9 deletions pkg/cmd/roachtest/tests/disk_stall.go
Original file line number Diff line number Diff line change
Expand Up @@ -208,14 +208,12 @@ func runDiskStalledDetection(
}

// Let the workload continue after the stall.
{
workloadPauseDur := 10*time.Minute - timeutil.Since(workloadStartAt)
t.Status("letting workload continue for ", workloadPauseDur, " with n1 stalled")
select {
case <-ctx.Done():
t.Fatal(ctx.Err())
case <-time.After(workloadPauseDur):
}
workloadAfterDur := 10*time.Minute - timeutil.Since(workloadStartAt)
t.Status("letting workload continue for ", workloadAfterDur, " with n1 stalled")
select {
case <-ctx.Done():
t.Fatal(ctx.Err())
case <-time.After(workloadAfterDur):
}

{
Expand All @@ -226,7 +224,7 @@ func runDiskStalledDetection(
cum := response.Results[0].Datapoints
totalTxnsPostStall := cum[len(cum)-1].Value - totalTxnsPreStall
preStallTPS := totalTxnsPreStall / stallAt.Sub(workloadStartAt).Seconds()
postStallTPS := totalTxnsPostStall / now.Sub(stallAt).Seconds()
postStallTPS := totalTxnsPostStall / workloadAfterDur.Seconds()
t.L().PrintfCtx(ctx, "%.2f total transactions committed after stall\n", totalTxnsPostStall)
t.L().PrintfCtx(ctx, "pre-stall tps: %.2f, post-stall tps: %.2f\n", preStallTPS, postStallTPS)
if postStallTPS < preStallTPS/2 {
Expand Down

0 comments on commit 0dc1f64

Please sign in to comment.