diff --git a/pkg/controller/main-controller.go b/pkg/controller/main-controller.go
index aeb4e1c3d50..dc3f7d8592b 100644
--- a/pkg/controller/main-controller.go
+++ b/pkg/controller/main-controller.go
@@ -492,12 +492,12 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha
 	for {
 		select {
 		case oerr := <-notificationChannel:
-			if !errors.Is(oerr.Err, http.ErrServerClosed) {
+			if oerr != nil && !errors.Is(oerr.Err, http.ErrServerClosed) {
 				klog.Errorf("STS API Server stopped: %v, going to restart", oerr.Err)
 				go c.startSTSAPIServer(ctx, notificationChannel)
 			}
 		case err := <-upgradeServerChannel:
-			if err != http.ErrServerClosed {
+			if err != nil && !errors.Is(err, http.ErrServerClosed) {
 				klog.Errorf("Upgrade Server stopped: %v, going to restart", err)
 				upgradeServerChannel = c.startUpgradeServer()
 			}
@@ -584,8 +584,24 @@ func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error {
 				leaderRun(ctx, c, threadiness, stopCh, notificationChannel)
 			},
 			OnStoppedLeading: func() {
-				// we can do cleanup here
-				klog.Infof("leader lost: %s", c.podName)
+				klog.Infof("leader lost, removing any leader labels that I '%s' might have", c.podName)
+				p := []patchAnnotation{{
+					Op:   "remove",
+					Path: "/metadata/labels/operator",
+				}}
+
+				payloadBytes, err := json.Marshal(p)
+				if err != nil {
+					klog.Errorf("failed to marshal patch: %#v", err)
+				} else {
+					c.kubeClientSet.CoreV1().Pods(leaseLockNamespace).Patch(ctx, c.podName, types.JSONPatchType, payloadBytes, metav1.PatchOptions{})
+				}
+				// Even if Stop() is called twice, stopping it here ensures the sync handler no longer is handling events,
+				// in case SIGTERM fails or the controller takes longer to exit.
+				c.Stop()
+				if err := syscall.Kill(os.Getpid(), syscall.SIGTERM); err != nil {
+					klog.Errorf("error sending SIGTERM: %v", err)
+				}
 			},
 			OnNewLeader: func(identity string) {
 				// we're notified when new leader elected