From 17bc75b93f9d9f6b7f8ace7e46f0ff4c4276db2c Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Mon, 6 Feb 2023 09:54:08 +0000 Subject: [PATCH 1/2] roachtest: use exponential backoff for `change-replicas/mixed-version` retries Epic: none Release note: None --- pkg/cmd/roachtest/tests/mixed_version_change_replicas.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/cmd/roachtest/tests/mixed_version_change_replicas.go b/pkg/cmd/roachtest/tests/mixed_version_change_replicas.go index 7a7944641dd7..d58c97c0609a 100644 --- a/pkg/cmd/roachtest/tests/mixed_version_change_replicas.go +++ b/pkg/cmd/roachtest/tests/mixed_version_change_replicas.go @@ -21,6 +21,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/util/randutil" + "github.com/cockroachdb/cockroach/pkg/util/retry" "github.com/stretchr/testify/require" ) @@ -119,8 +120,14 @@ func runChangeReplicasMixedVersion(ctx context.Context, t test.Test, c cluster.C conn := u.c.Conn(ctx, t.L(), gateway) defer conn.Close() + retryOpts := retry.Options{ + InitialBackoff: 100 * time.Millisecond, + MaxBackoff: 5 * time.Second, + Multiplier: 2, + MaxRetries: 8, + } var rangeErrors map[int]string - for attempt := 1; attempt <= 5; attempt++ { + for r := retry.StartWithCtx(ctx, retryOpts); r.Next(); { if errCount := len(rangeErrors); errCount > 0 { t.L().Printf("%d ranges failed, retrying", errCount) } From 5cd80f1ef2e3e4402630cfbefab8af565bf5a94a Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Mon, 13 Mar 2023 13:48:25 +0000 Subject: [PATCH 2/2] roachtest: maybe deflake `change-replicas/mixed-version` Epic: none Release note: None --- pkg/cmd/roachtest/tests/mixed_version_change_replicas.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/cmd/roachtest/tests/mixed_version_change_replicas.go b/pkg/cmd/roachtest/tests/mixed_version_change_replicas.go index d58c97c0609a..ff73271b7c75 100644 --- a/pkg/cmd/roachtest/tests/mixed_version_change_replicas.go +++ b/pkg/cmd/roachtest/tests/mixed_version_change_replicas.go @@ -124,10 +124,11 @@ func runChangeReplicasMixedVersion(ctx context.Context, t test.Test, c cluster.C InitialBackoff: 100 * time.Millisecond, MaxBackoff: 5 * time.Second, Multiplier: 2, - MaxRetries: 8, + MaxRetries: 12, } var rangeErrors map[int]string for r := retry.StartWithCtx(ctx, retryOpts); r.Next(); { + setReplicateQueueEnabled(false) if errCount := len(rangeErrors); errCount > 0 { t.L().Printf("%d ranges failed, retrying", errCount) } @@ -152,6 +153,10 @@ func runChangeReplicasMixedVersion(ctx context.Context, t test.Test, c cluster.C if len(rangeErrors) == 0 { break } + // The failure may be caused by conflicts with ongoing configuration + // changes by the replicate queue, so we re-enable it and let it run + // for a bit before the next retry. + setReplicateQueueEnabled(true) } if len(rangeErrors) > 0 {