Merge #73155

73155: rpc: avoid use-after-Stop via RPC endpoints r=erikgrinaker a=tbg This avoids a class of crashes on shutdown that occur when an RPC is handled after the stopper has already stopped (and in the process has stopped the Stores' pebble instances. These crashes are probably very rare in production but hit with some regularity in unit tests, for an example see: #68395 (comment) I was also able to hit this locally within a few minutes by running `make stress PKG=./pkg/kv/kvserver/ TESTS=TestStrictGCEnforcement/protected_timestamps_are_respected`. Fixes #73154. Release note: None Co-authored-by: Tobias Grieger <[email protected]>
cockroachdb · Dec 9, 2021 · 3bed3a6 · 3bed3a6
2 parents d848eff + 8c24367
commit 3bed3a6
Show file tree

Hide file tree

Showing 5 changed files with 232 additions and 121 deletions.
diff --git a/pkg/kv/kvserver/closedts/sidetransport/sender_test.go b/pkg/kv/kvserver/closedts/sidetransport/sender_test.go
@@ -309,7 +309,13 @@ func (s *sideTransportGRPCServer) addr() net.Addr {
 
 func newMockSideTransportGRPCServer(stopper *stop.Stopper) (*sideTransportGRPCServer, error) {
 	receiver := newMockReceiver()
-	stopper.AddCloser(receiver)
+	if err := stopper.RunAsyncTask(context.Background(), "stopper-watcher", func(ctx context.Context) {
+		// We can't use a Closer since the receiver will be blocking inside of a task.
+		<-stopper.ShouldQuiesce()
+		receiver.Close()
+	}); err != nil {
+		return nil, err
+	}
 	server, err := newMockSideTransportGRPCServerWithOpts(stopper, receiver)
 	if err != nil {
 		return nil, err