ccl/sqlproxyccl: fix TestWatchPods under stressrace

Fixes cockroachdb#69220.
Regression from cockroachdb#67452.

In cockroachdb#67452, we omitted DRAINING pods from the tenant directory. Whenever a pod
goes into the DRAINING state, the pod watcher needs time to update the
directory. Not waiting for that while calling EnsureTenantAddr produces a
stale result. This commit updates TestWatchPods by polling on EnsureTenantAddr
until that has been completed.

Release note: None

pkg/ccl/sqlproxyccl/tenant/directory_test.go

@@ -101,9 +101,10 @@ func TestWatchPods(t *testing.T) {
 
 	// Ensure that all addresses have been cleared from the directory, since
 	// it should only return RUNNING addresses.
-	addrs, err := dir.LookupTenantAddrs(ctx, tenantID)
-	require.NoError(t, err)
-	require.Empty(t, addrs)
+	require.Eventually(t, func() bool {
+		addrs, _ := dir.LookupTenantAddrs(ctx, tenantID)
+		return len(addrs) == 0
+	}, 10*time.Second, 100*time.Millisecond)
 
 	// Now shut the tenant directory down.
 	processes := tds.Get(tenantID)
@@ -120,10 +121,12 @@ func TestWatchPods(t *testing.T) {
 	require.Equal(t, addr, pod.Addr)
 	require.Equal(t, tenant.DELETING, pod.State)
 
-	require.Eventually(t, func() bool {
-		addrs, _ := dir.LookupTenantAddrs(ctx, tenantID)
-		return len(addrs) == 0
-	}, 10*time.Second, 100*time.Millisecond)
+	// We know that the directory should have been emptied earlier since we
+	// don't add DRAINING pods to the directory, so putting the pod into the
+	// DELETING state should not make a difference.
+	addrs, err := dir.LookupTenantAddrs(ctx, tenantID)
+	require.NoError(t, err)
+	require.Empty(t, addrs)
 
 	// Resume tenant again by a direct call to the directory server
 	_, err = tds.EnsurePod(ctx, &tenant.EnsurePodRequest{tenantID.ToUint64()})