services/horizon: Fix remaining HTTP error logging (#3730)

In #3674 `canceling statement due to conflict with recovery` errors were
silenced so Horizon does not report expected errors with `level=error`. However,
`bad connection` errors that occur from time to time (around 0.001% of all
requests in the last 24h in horizon.stellar.org likely after conflict with
recovery error) were missed but trigger alerts.

services/horizon/internal/httpx/server.go

@@ -58,6 +58,7 @@ func init() {
 	problem.RegisterError(context.Canceled, hProblem.ServiceUnavailable)
 	problem.RegisterError(db.ErrCancelled, hProblem.ServiceUnavailable)
 	problem.RegisterError(db.ErrConflictWithRecovery, hProblem.ServiceUnavailable)
+	problem.RegisterError(db.ErrBadConnection, hProblem.ServiceUnavailable)
 }
 
 func NewServer(serverConfig ServerConfig, routerConfig RouterConfig, ledgerState *ledger.State) (*Server, error) {

support/db/main.go

@@ -38,6 +38,9 @@ var (
 	// read replica cancels the query due to conflict with about-to-be-applied
 	// WAL entries (https://www.postgresql.org/docs/current/hot-standby.html).
 	ErrConflictWithRecovery = errors.New("canceling statement due to conflict with recovery")
+	// ErrBadConnection is an error returned when driver returns `bad connection`
+	// error.
+	ErrBadConnection = errors.New("bad connection")
 )
 
 // Conn represents a connection to a single database.

support/db/session.go

@@ -236,6 +236,8 @@ func (s *Session) replaceWithKnownError(err error) error {
 		return ErrCancelled
 	case strings.Contains(err.Error(), "pq: canceling statement due to conflict with recovery"):
 		return ErrConflictWithRecovery
+	case strings.Contains(err.Error(), "driver: bad connection"):
+		return ErrBadConnection
 	default:
 		return nil
 	}

support/render/problem/problem.go

@@ -73,7 +73,8 @@ const (
 	// LogNoErrors indicates that the Problem instance should not log any errors
 	LogNoErrors = LogFilter(iota)
 	// LogUnknownErrors indicates that the Problem instance should only log errors
-	// which are not registered
+	// which are not registered with level=error but will also log known errors with
+	// level=warn.
 	LogUnknownErrors = LogFilter(iota)
 	// LogAllErrors indicates that the Problem instance should log all errors
 	LogAllErrors = LogFilter(iota)
@@ -187,6 +188,8 @@ func (ps *Problem) Render(ctx context.Context, w http.ResponseWriter, err error)
 				ps.reportFn(ctx, err)
 			}
 			problem = ServerError
+		} else if ps.filter == LogUnknownErrors {
+			ps.log.Ctx(ctx).WithStack(err).Warn(err)
 		}
 	}