diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 1f59e11d2d28..7ddff964a087 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -43,6 +43,7 @@ import ( "github.com/grafana/loki/pkg/storage/chunk" "github.com/grafana/loki/pkg/storage/stores/shipper/compactor" "github.com/grafana/loki/pkg/tracing" + serverutil "github.com/grafana/loki/pkg/util/server" "github.com/grafana/loki/pkg/validation" ) @@ -211,6 +212,7 @@ func New(cfg Config) (*Loki, error) { } loki.setupAuthMiddleware() + loki.setupGRPCRecoveryMiddleware() if err := loki.setupModuleManager(); err != nil { return nil, err } @@ -235,6 +237,11 @@ func (t *Loki) setupAuthMiddleware() { }) } +func (t *Loki) setupGRPCRecoveryMiddleware() { + t.Cfg.Server.GRPCMiddleware = append(t.Cfg.Server.GRPCMiddleware, serverutil.RecoveryGRPCUnaryInterceptor) + t.Cfg.Server.GRPCStreamMiddleware = append(t.Cfg.Server.GRPCStreamMiddleware, serverutil.RecoveryGRPCStreamInterceptor) +} + func newDefaultConfig() *Config { defaultConfig := &Config{} defaultFS := flag.NewFlagSet("", flag.PanicOnError) diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index c334cf8501b4..fd9a6fb712db 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -238,12 +238,16 @@ func (t *Loki) initIngester() (_ services.Service, err error) { if err != nil { return } - logproto.RegisterPusherServer(t.Server.GRPC, t.Ingester) logproto.RegisterQuerierServer(t.Server.GRPC, t.Ingester) logproto.RegisterIngesterServer(t.Server.GRPC, t.Ingester) - t.Server.HTTP.Path("/flush").Handler(http.HandlerFunc(t.Ingester.FlushHandler)) - t.Server.HTTP.Methods("POST").Path("/ingester/flush_shutdown").Handler(http.HandlerFunc(t.Ingester.ShutdownHandler)) + + httpMiddleware := middleware.Merge( + serverutil.RecoveryHTTPMiddleware, + ) + t.Server.HTTP.Path("/flush").Handler(httpMiddleware.Wrap(http.HandlerFunc(t.Ingester.FlushHandler))) + t.Server.HTTP.Methods("POST").Path("/ingester/flush_shutdown").Handler(httpMiddleware.Wrap(http.HandlerFunc(t.Ingester.ShutdownHandler))) + return t.Ingester, nil } diff --git a/pkg/querier/worker_service.go b/pkg/querier/worker_service.go index 6650ae24c747..9203a459e67b 100644 --- a/pkg/querier/worker_service.go +++ b/pkg/querier/worker_service.go @@ -104,6 +104,7 @@ func InitWorkerService( // HTTP router with middleware to parse the tenant ID from the HTTP header and inject it into the // request context, as well as make sure any x-www-url-formencoded params are correctly parsed httpMiddleware := middleware.Merge( + serverutil.RecoveryHTTPMiddleware, authMiddleware, serverutil.NewPrepopulateMiddleware(), )