added more detailed logs around ES communication failure

cmd/fleet/error.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"os"
+	"strings"
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
 	"github.com/elastic/fleet-server/v7/internal/pkg/limit"
@@ -144,6 +145,17 @@ func NewErrorResp(err error) errResp {
 		}
 	}
 
+	// Check if we have encountered a connectivity error
+	// Predicate taken from https://github.com/golang/go/blob/go1.17.5/src/net/dial_test.go#L798
+	if strings.Contains(err.Error(), "connection refused") {
+		return errResp{
+			http.StatusServiceUnavailable,
+			"ServiceUnavailable",
+			"Fleet server unable to communicate with Elasticsearch",
+			zerolog.InfoLevel,
+		}
+	}
+
 	// Default
 	return errResp{
 		StatusCode: http.StatusBadRequest,

internal/pkg/coordinator/monitor.go

@@ -128,11 +128,17 @@ func (m *monitorT) Run(ctx context.Context) (err error) {
 	// Start timer loop to ensure leadership
 	lT := time.NewTimer(m.checkInterval)
 	defer lT.Stop()
+
+	// Keep track of errored statuses
+	erroredOnLastRequest := false
+	numFailedRequests := 0
 	for {
 		select {
 		case hits := <-s.Output():
 			err = m.handlePolicies(ctx, hits)
 			if err != nil {
+				erroredOnLastRequest = true
+				numFailedRequests++
 				m.log.Warn().Err(err).Msgf("Encountered an error while policy leadership changes; continuing to retry.")
 			}
 		case <-mT.C:
@@ -141,13 +147,20 @@ func (m *monitorT) Run(ctx context.Context) (err error) {
 		case <-lT.C:
 			err = m.ensureLeadership(ctx)
 			if err != nil {
+				erroredOnLastRequest = true
+				numFailedRequests++
 				m.log.Warn().Err(err).Msgf("Encountered an error while checking/assigning policy leaders; continuing to retry.")
 			}
 			lT.Reset(m.checkInterval)
 		case <-ctx.Done():
 			m.releaseLeadership()
 			return ctx.Err()
 		}
+		if err == nil && erroredOnLastRequest {
+			erroredOnLastRequest = false
+			m.log.Info().Msgf("Policy leader monitor successfully recovered after %d attempts", numFailedRequests)
+			numFailedRequests = 0
+		}
 	}
 }