server: clear tenant cookies when tenant is missing

Previously, when we had cookies in an HTTP Request referencing a tenant, and
that tenant didn't exist, we'd be stuck in an error loop constantly attempting
to start a tenant that didn't exist.

This commit updates the error log in `server_controller.go` to clear cookies
for multi-tenant sessions when we return an error in order to break the browser
out of the bad update cycle and have it either continue happily with an
insecure cluster that doesn't need a session anyway, or request a new one by
routing the user to the login page.

Resolves: #95365
Resolves: #95366
Epic: CRDB-14545

Release note: None

pkg/server/server_controller.go

@@ -252,6 +252,21 @@ func (c *serverController) httpMux(w http.ResponseWriter, r *http.Request) {
 	s, err := c.getOrCreateServer(ctx, tenantName)
 	if err != nil {
 		log.Warningf(ctx, "unable to start server for tenant %q: %v", tenantName, err)
+		// Clear session and tenant cookies after all logouts have completed.
+		http.SetCookie(w, &http.Cookie{
+			Name:     MultitenantSessionCookieName,
+			Value:    "",
+			Path:     "/",
+			HttpOnly: true,
+			Expires:  timeutil.Unix(0, 0),
+		})
+		http.SetCookie(w, &http.Cookie{
+			Name:     TenantSelectCookieName,
+			Value:    "",
+			Path:     "/",
+			HttpOnly: false,
+			Expires:  timeutil.Unix(0, 0),
+		})
 		w.WriteHeader(http.StatusInternalServerError)
 		return
 	}