From f7d4b984af9dbae58703bfbfaec98dd56019bb08 Mon Sep 17 00:00:00 2001 From: Ibrahim Kettaneh Date: Wed, 20 Nov 2024 15:07:08 -0500 Subject: [PATCH 1/3] kvcoord: print traced query when failing the TestProxyTracing This commit prints the traced query when TestProxyTracing test fails. This should help with debugging the failures. References: #135493 Release note: None --- .../kvcoord/dist_sender_server_test.go | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go b/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go index d83149539f03..fc1baf535816 100644 --- a/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go +++ b/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go @@ -4914,6 +4914,22 @@ func TestProxyTracing(t *testing.T) { return nil } + printTrace := func() { + t.Log("started printing a trace") + rows, err := conn.QueryContext(ctx, "SELECT message, tag, location FROM [SHOW TRACE FOR SESSION]") + require.NoError(t, err) + defer rows.Close() + + // Iterate over the results and print them + for rows.Next() { + var msg, tag, loc string + err := rows.Scan(&msg, &tag, &loc) + require.NoError(t, err) + t.Logf("msg: %s, tag: %s, loc: %s", msg, tag, loc) + } + require.NoError(t, rows.Err()) + } + // Wait until the leaseholder for the test table ranges are on n3. testutils.SucceedsSoon(t, func() error { return checkLeaseCount(3, numRanges) @@ -4923,7 +4939,6 @@ func TestProxyTracing(t *testing.T) { _, err = conn.Exec("SET TRACING = on; SELECT FROM t where i = 987654321; SET TRACING = off") require.NoError(t, err) - // Expect the "proxy request complete" message to be in the trace and that it // comes from the proxy node n2. var msg, tag, loc string @@ -4933,6 +4948,10 @@ func TestProxyTracing(t *testing.T) { AND location LIKE '%server/node%' AND tag LIKE '%n2%'`, ).Scan(&msg, &tag, &loc); err != nil { + // If we fail for any reason, print the trace to help debugging. + printTrace() + // Make sure that node 3 still holds the leases. + require.NoError(t, checkLeaseCount(3, numRanges)) if errors.Is(err, gosql.ErrNoRows) { t.Fatalf("request succeeded without proxying") } From 39e2cc2ccf764b5ca94892579eacadb70b73f2a3 Mon Sep 17 00:00:00 2001 From: Ibrahim Kettaneh Date: Wed, 20 Nov 2024 17:29:27 -0500 Subject: [PATCH 2/3] kvcoord: disable deadlock/race for expiration leases TestProxyTracing This commit disables deadlock/race runs for TestProxyTracing when running with expiration leases. The main reason is that it consumes resources unnecessarily. References: #135493 Release note: None --- pkg/kv/kvclient/kvcoord/dist_sender_server_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go b/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go index fc1baf535816..e8fa63fed9ac 100644 --- a/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go +++ b/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go @@ -4829,6 +4829,8 @@ func TestProxyTracing(t *testing.T) { st := cluster.MakeTestingClusterSettings() switch leaseType { case roachpb.LeaseExpiration: + skip.UnderRace(t, "too slow") + skip.UnderDeadlock(t, "too slow") kvserver.ExpirationLeasesOnly.Override(ctx, &st.SV, true) case roachpb.LeaseEpoch: // With epoch leases this test doesn't work reliably. It passes From c43cf5fadf91263f07f48b304964bdff44d26e65 Mon Sep 17 00:00:00 2001 From: Ibrahim Kettaneh Date: Wed, 20 Nov 2024 17:03:39 -0500 Subject: [PATCH 3/3] kvcoord: Disable follower reads for TestProxyTracing This commit disables follower reads for the test TestProxyTracing. We noticed that sometimes, the test gets slow, and by the time we issue a read request, it's served via follower reads instead of proxying it to the leaseholder. Fixes: #135493 Release note: None --- pkg/kv/kvclient/kvcoord/dist_sender_server_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go b/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go index e8fa63fed9ac..b558f67d315f 100644 --- a/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go +++ b/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go @@ -4847,6 +4847,9 @@ func TestProxyTracing(t *testing.T) { } kvserver.RangefeedEnabled.Override(ctx, &st.SV, true) kvserver.RangeFeedRefreshInterval.Override(ctx, &st.SV, 10*time.Millisecond) + // Disable follower reads to ensure that the request is proxied, and not + // answered locally due to follower reads. + kvserver.FollowerReadsEnabled.Override(ctx, &st.SV, false) closedts.TargetDuration.Override(ctx, &st.SV, 10*time.Millisecond) closedts.SideTransportCloseInterval.Override(ctx, &st.SV, 10*time.Millisecond)