From 2c0de90521b1efa21952725f21c218cecf57d358 Mon Sep 17 00:00:00 2001 From: Ibrahim Kettaneh Date: Wed, 20 Nov 2024 15:07:08 -0500 Subject: [PATCH 1/2] kvcoord: print traced query when failing the TestProxyTracing This commit prints the traced query when TestProxyTracing test fails. This should help with debugging the failures. References: #135493 Release note: None --- .../kvcoord/dist_sender_server_test.go | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go b/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go index 2ee1256ba5be..3a320aadb2f9 100644 --- a/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go +++ b/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go @@ -4883,6 +4883,22 @@ func TestProxyTracing(t *testing.T) { return nil } + printTrace := func() { + t.Log("started printing a trace") + rows, err := conn.QueryContext(ctx, "SELECT message, tag, location FROM [SHOW TRACE FOR SESSION]") + require.NoError(t, err) + defer rows.Close() + + // Iterate over the results and print them + for rows.Next() { + var msg, tag, loc string + err := rows.Scan(&msg, &tag, &loc) + require.NoError(t, err) + t.Logf("msg: %s, tag: %s, loc: %s", msg, tag, loc) + } + require.NoError(t, rows.Err()) + } + // Wait until the leaseholder for the test table ranges are on n3. testutils.SucceedsSoon(t, func() error { return checkLeaseCount(3, numRanges) @@ -4907,6 +4923,10 @@ func TestProxyTracing(t *testing.T) { AND location LIKE '%server/node%' AND tag LIKE '%n2%'`, ).Scan(&msg, &tag, &loc); err != nil { + // If we fail for any reason, print the trace to help debugging. + printTrace() + // Make sure that node 3 still holds the leases. + require.NoError(t, checkLeaseCount(3, numRanges)) if errors.Is(err, gosql.ErrNoRows) { t.Fatalf("request succeeded without proxying") } From 66aaff873513677bb5f4c6949678148bce1de5b0 Mon Sep 17 00:00:00 2001 From: Ibrahim Kettaneh Date: Wed, 20 Nov 2024 17:03:39 -0500 Subject: [PATCH 2/2] kvcoord: Disable follower reads for TestProxyTracing This commit disables follower reads for the test TestProxyTracing. We noticed that sometimes, the test gets slow, and by the time we issue a read request, it's served via follower reads instead of proxying it to the leaseholder. Fixes: #135493 Release note: None --- pkg/kv/kvclient/kvcoord/dist_sender_server_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go b/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go index 3a320aadb2f9..0e627ac41561 100644 --- a/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go +++ b/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go @@ -4814,6 +4814,9 @@ func TestProxyTracing(t *testing.T) { kvserver.ExpirationLeasesOnly.Override(ctx, &st.SV, true) kvserver.RangefeedEnabled.Override(ctx, &st.SV, true) kvserver.RangeFeedRefreshInterval.Override(ctx, &st.SV, 10*time.Millisecond) + // Disable follower reads to ensure that the request is proxied, and not + // answered locally due to follower reads. + kvserver.FollowerReadsEnabled.Override(ctx, &st.SV, false) closedts.TargetDuration.Override(ctx, &st.SV, 10*time.Millisecond) closedts.SideTransportCloseInterval.Override(ctx, &st.SV, 10*time.Millisecond)