Add mechanism to disable automatic rollbacks (#19748)

When testing the rollback mechanism, there's two categories of tests
typically written:

 1. Ones in which the rollback manager is entirely left alone, which
    usually are a bit slower and less predictable. However, it is still
    sufficient in many scenarios.
 2. Ones in which the rollback manager is explicitly probed by tests
    and "stepped" to achieve the next rollback.

Here, without a mechanism to fully disable the rollback manager's
periodic ticker (without affecting its ability to work!) we'll continue
to see races of the sort:

>     --- FAIL: TestRevocationQueue (50.95s)
>     panic: sync: WaitGroup is reused before previous Wait has returned [recovered]
>         panic: sync: WaitGroup is reused before previous Wait has returned

This allows us to disable the ticker, returning control to the test
suite entirely.

Signed-off-by: Alexander Scheel <[email protected]>

vault/rollback.go

@@ -47,6 +47,7 @@ type RollbackManager struct {
 	shutdown     bool
 	shutdownCh   chan struct{}
 	shutdownLock sync.Mutex
+	stopTicker   chan struct{}
 	quitContext  context.Context
 
 	core *Core
@@ -70,6 +71,7 @@ func NewRollbackManager(ctx context.Context, logger log.Logger, backendsFunc fun
 		inflight:    make(map[string]*rollbackState),
 		doneCh:      make(chan struct{}),
 		shutdownCh:  make(chan struct{}),
+		stopTicker:  make(chan struct{}),
 		quitContext: ctx,
 		core:        core,
 	}
@@ -94,6 +96,10 @@ func (m *RollbackManager) Stop() {
 	m.inflightAll.Wait()
 }
 
+func (m *RollbackManager) StopTicker() {
+	close(m.stopTicker)
+}
+
 // run is a long running routine to periodically invoke rollback
 func (m *RollbackManager) run() {
 	m.logger.Info("starting rollback manager")
@@ -108,6 +114,10 @@ func (m *RollbackManager) run() {
 		case <-m.shutdownCh:
 			m.logger.Info("stopping rollback manager")
 			return
+
+		case <-m.stopTicker:
+			m.logger.Info("stopping rollback manager ticker for tests")
+			tick.Stop()
 		}
 	}
 }

vault/testing.go

@@ -1002,6 +1002,10 @@ func (c *TestClusterCore) stop() error {
 	return nil
 }
 
+func (c *TestClusterCore) StopAutomaticRollbacks() {
+	c.rollback.StopTicker()
+}
+
 func (c *TestClusterCore) GrabRollbackLock() {
 	// Ensure we don't hold this lock while there are in flight rollbacks.
 	c.rollback.inflightAll.Wait()