Merge pull request #70737 from cockroachdb/blathers/backport-release-…

…21.2-70723

release-21.2: kv: deflake TestTenantRateLimiter

pkg/kv/kvserver/client_tenant_test.go

@@ -162,6 +162,10 @@ func TestTenantRateLimiter(t *testing.T) {
 	ctx := context.Background()
 	defer s.Stopper().Stop(ctx)
 
+	// Set a small rate limit so the test doesn't take a long time.
+	runner := sqlutils.MakeSQLRunner(sqlDB)
+	runner.Exec(t, `SET CLUSTER SETTING kv.tenant_rate_limiter.rate_limit = 200`)
+
 	tenantID := serverutils.TestTenantID()
 	codec := keys.MakeSQLCodec(tenantID)
 
@@ -222,8 +226,7 @@ func TestTenantRateLimiter(t *testing.T) {
 
 	// Create some tooling to read and verify metrics off of the prometheus
 	// endpoint.
-	sqlutils.MakeSQLRunner(sqlDB).Exec(t,
-		`SET CLUSTER SETTING server.child_metrics.enabled = true`)
+	runner.Exec(t, `SET CLUSTER SETTING server.child_metrics.enabled = true`)
 	httpClient, err := s.GetHTTPClient()
 	require.NoError(t, err)
 	getMetrics := func() string {
@@ -245,5 +248,9 @@ func TestTenantRateLimiter(t *testing.T) {
 
 	// Ensure that the metric for the admitted requests reflects the number of
 	// admitted requests.
-	require.Contains(t, getMetrics(), makeMetricStr(int64(tooManyWrites)))
+	// TODO(radu): this is fragile because a background write could sneak in and
+	// the count wouldn't match exactly.
+	m := getMetrics()
+	exp := makeMetricStr(int64(tooManyWrites))
+	require.Contains(t, m, exp, "could not find %s in metrics: \n%s\n", exp, m)
 }