Skip to content

Commit

Permalink
queue_manager: add histogram info to error logs (prometheus#14326)
Browse files Browse the repository at this point in the history
Signed-off-by: Piotr Gwizdala <[email protected]>
  • Loading branch information
thampiotr authored Jun 20, 2024
1 parent dbd29df commit d782533
Showing 1 changed file with 11 additions and 6 deletions.
17 changes: 11 additions & 6 deletions storage/remote/queue_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package remote
import (
"context"
"errors"
"fmt"
"math"
"strconv"
"sync"
Expand Down Expand Up @@ -1224,12 +1225,16 @@ func (s *shards) stop() {
// Force an unclean shutdown.
s.hardShutdown()
<-s.done
if dropped := s.samplesDroppedOnHardShutdown.Load(); dropped > 0 {
level.Error(s.qm.logger).Log("msg", "Failed to flush all samples on shutdown", "count", dropped)
}
if dropped := s.exemplarsDroppedOnHardShutdown.Load(); dropped > 0 {
level.Error(s.qm.logger).Log("msg", "Failed to flush all exemplars on shutdown", "count", dropped)

// Log error for any dropped samples, exemplars, or histograms.
logDroppedError := func(t string, counter atomic.Uint32) {
if dropped := counter.Load(); dropped > 0 {
level.Error(s.qm.logger).Log("msg", fmt.Sprintf("Failed to flush all %s on shutdown", t), "count", dropped)
}
}
logDroppedError("samples", s.samplesDroppedOnHardShutdown)
logDroppedError("exemplars", s.exemplarsDroppedOnHardShutdown)
logDroppedError("histograms", s.histogramsDroppedOnHardShutdown)
}

// enqueue data (sample or exemplar). If the shard is full, shutting down, or
Expand Down Expand Up @@ -1537,7 +1542,7 @@ func (s *shards) sendSamples(ctx context.Context, samples []prompb.TimeSeries, s
begin := time.Now()
err := s.sendSamplesWithBackoff(ctx, samples, sampleCount, exemplarCount, histogramCount, pBuf, buf)
if err != nil {
level.Error(s.qm.logger).Log("msg", "non-recoverable error", "count", sampleCount, "exemplarCount", exemplarCount, "err", err)
level.Error(s.qm.logger).Log("msg", "non-recoverable error", "count", sampleCount, "exemplarCount", exemplarCount, "histogramCount", histogramCount, "err", err)
s.qm.metrics.failedSamplesTotal.Add(float64(sampleCount))
s.qm.metrics.failedExemplarsTotal.Add(float64(exemplarCount))
s.qm.metrics.failedHistogramsTotal.Add(float64(histogramCount))
Expand Down

0 comments on commit d782533

Please sign in to comment.