Fix flaky unit tests

Addresses issues causing failures in TestCache_Backoff, TestTeleportProcess_reconnectToAuth
and TestResourceWatcher_Backoff. By utilizing FakeClock.BlockUntil tests ensure that the clock
will not be advanced until retry.After has been called.

lib/cache/cache_test.go

@@ -2193,8 +2193,11 @@ func TestCache_Backoff(t *testing.T) {
 			require.GreaterOrEqual(t, duration, stepMin)
 			require.LessOrEqual(t, duration, stepMax)
 
+			// wait for cache to get to retry.After
+			clock.BlockUntil(1)
+
 			// add some extra to the duration to ensure the retry occurs
-			clock.Advance(duration * 3)
+			clock.Advance(p.cache.MaxRetryPeriod)
 		case <-time.After(time.Minute):
 			t.Fatalf("timeout waiting for event")
 		}

lib/service/service_test.go

@@ -545,6 +545,7 @@ func TestTeleportProcess_reconnectToAuth(t *testing.T) {
 	cfg.Auth.Enabled = false
 	cfg.Proxy.Enabled = false
 	cfg.SSH.Enabled = true
+	cfg.MaxRetryPeriod = defaults.MaxWatcherBackoff
 	process, err := NewTeleport(cfg)
 	require.NoError(t, err)
 
@@ -567,8 +568,12 @@ func TestTeleportProcess_reconnectToAuth(t *testing.T) {
 
 			require.GreaterOrEqual(t, duration, stepMin)
 			require.LessOrEqual(t, duration, stepMax)
+
+			// wait for connection to get to retry.After
+			clock.BlockUntil(1)
+
 			// add some extra to the duration to ensure the retry occurs
-			clock.Advance(duration * 3)
+			clock.Advance(cfg.MaxRetryPeriod)
 		case <-time.After(time.Minute):
 			t.Fatalf("timeout waiting for failure")
 		}

lib/services/watcher_test.go

@@ -82,8 +82,12 @@ func TestResourceWatcher_Backoff(t *testing.T) {
 
 			require.GreaterOrEqual(t, duration, stepMin)
 			require.LessOrEqual(t, duration, stepMax)
+
+			// wait for watcher to get to retry.After
+			clock.BlockUntil(1)
+
 			// add some extra to the duration to ensure the retry occurs
-			clock.Advance(duration * 3)
+			clock.Advance(w.MaxRetryPeriod)
 		case <-time.After(time.Minute):
 			t.Fatalf("timeout waiting for reset")
 		}