Merge #55239

55239: roachtest: restart node in kv/quiesence unconditionally r=knz a=tbg When the test fails, restart the node anyway, so we don't get distracted by a faux "dead node". Release note: None Co-authored-by: Tobias Grieger <[email protected]>
cockroachdb · Oct 8, 2020 · 3cceec6 · 3cceec6
2 parents 7bc2bf9 + 38d6d71
commit 3cceec6
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 7 deletions.
diff --git a/pkg/cmd/roachtest/inconsistency.go b/pkg/cmd/roachtest/inconsistency.go
@@ -88,6 +88,8 @@ func runInconsistency(ctx context.Context, t *test, c *cluster) {
 		return nil
 	})
 
+	time.Sleep(10 * time.Second) // wait for n1-n3 to all be known as live to each other
+
 	// set an aggressive consistency check interval, but only now (that we're
 	// reasonably sure all nodes are live, etc). This makes sure that the consistency
 	// check runs against all three nodes. If it targeted only two nodes, a random

diff --git a/pkg/cmd/roachtest/kv.go b/pkg/cmd/roachtest/kv.go
@@ -23,7 +23,6 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/kv"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror"
-	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/ts/tspb"
 	"github.com/cockroachdb/cockroach/pkg/util/httputil"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
@@ -328,6 +327,7 @@ func registerKVQuiescenceDead(r *testRegistry) {
 				// other earlier kv invocation's footsteps.
 				run(kv+" --seed 2 {pgurl:1}", true)
 			})
+			c.Start(ctx, t, c.Node(nodes)) // satisfy dead node detector, even if test fails below
 
 			if minFrac, actFrac := 0.8, qpsOneDown/qpsAllUp; actFrac < minFrac {
 				t.Fatalf(
@@ -336,19 +336,18 @@ func registerKVQuiescenceDead(r *testRegistry) {
 				)
 			}
 			t.l.Printf("QPS went from %.2f to %2.f with one node down\n", qpsAllUp, qpsOneDown)
-			c.Start(ctx, t, c.Node(nodes)) // satisfy dead node detector
 		},
 	})
 }
 
 func registerKVGracefulDraining(r *testRegistry) {
 	r.Add(testSpec{
-		Name:    "kv/gracefuldraining/nodes=3",
-		Owner:   OwnerKV,
-		Cluster: makeClusterSpec(4),
+		Name:        "kv/gracefuldraining/nodes=3",
+		Owner:       OwnerKV,
+		Cluster:     makeClusterSpec(4),
+		Skip:        "flaky",
+		SkipDetails: "https://github.com/cockroachdb/cockroach/issues/53760",
 		Run: func(ctx context.Context, t *test, c *cluster) {
-			skip.UnderRace(t, "race builds make this test exceed its timeout")
-
 			nodes := c.spec.NodeCount - 1
 			c.Put(ctx, cockroach, "./cockroach", c.Range(1, nodes))
 			c.Put(ctx, workload, "./workload", c.Node(nodes+1))