sidecar: time limit requests to Prometheus remote read api

cmd/thanos/sidecar.go

@@ -37,6 +37,8 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri
 	promURL := cmd.Flag("prometheus.url", "URL at which to reach Prometheus's API. For better performance use local network.").
 		Default("http://localhost:9090").URL()
 
+	promRetention := modelDuration(cmd.Flag("prometheus.retention", "A limit on how much retention to query from Prometheus. 0d means query all TSDB data found on disk").Default("0d"))
+
 	dataDir := cmd.Flag("tsdb.path", "Data directory of TSDB.").
 		Default("./data").String()
 
@@ -71,6 +73,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri
 			*clientCA,
 			*httpBindAddr,
 			*promURL,
+			time.Duration(*promRetention),
 			*dataDir,
 			objStoreConfig,
 			rl,
@@ -90,6 +93,7 @@ func runSidecar(
 	clientCA string,
 	httpBindAddr string,
 	promURL *url.URL,
+	promRetention time.Duration,
 	dataDir string,
 	objStoreConfig *pathOrContent,
 	reloader *reloader.Reloader,
@@ -100,8 +104,9 @@ func runSidecar(
 
 		// Start out with the full time range. The shipper will constrain it later.
 		// TODO(fabxc): minimum timestamp is never adjusted if shipping is disabled.
-		mint: 0,
-		maxt: math.MaxInt64,
+		mint:  0,
+		maxt:  math.MaxInt64,
+		limit: promRetention,
 	}
 
 	confContentYaml, err := objStoreConfig.Content()
@@ -318,6 +323,7 @@ type promMetadata struct {
 	mtx    sync.Mutex
 	mint   int64
 	maxt   int64
+	limit  time.Duration
 	labels labels.Labels
 }
 
@@ -337,8 +343,21 @@ func (s *promMetadata) UpdateLabels(ctx context.Context, logger log.Logger) erro
 func (s *promMetadata) UpdateTimestamps(mint int64, maxt int64) {
 	s.mtx.Lock()
 	defer s.mtx.Unlock()
+	var limitt int64
+
+	// Calculate our limit on how far back in time we query Prometheus.
+	// Long term retention should be accessible via a Store component,
+	// therefore, its possible we can ignore anything older than a few days on
+	// the local Prometheus.  This is disabled if limit == 0.
+	if s.limit > 0 {
+		limitt = int64(model.Now().Add(-s.limit))
+	}
 
-	s.mint = mint
+	if mint < limitt {
+		s.mint = limitt
+	} else {
+		s.mint = mint
+	}
 	s.maxt = maxt
 }
 

docs/components/sidecar.md

@@ -104,6 +104,9 @@ Flags:
       --prometheus.url=http://localhost:9090
                                  URL at which to reach Prometheus's API. For
                                  better performance use local network.
+      --prometheus.retention=0d  A limit on how much retention to query from
+                                 Prometheus. 0d means query all TSDB data found
+                                 on disk
       --tsdb.path="./data"       Data directory of TSDB.
       --reloader.config-file=""  Config file watched by the reloader.
       --reloader.config-envsubst-file=""

pkg/store/prometheus.go

@@ -125,6 +125,14 @@ func (p *PrometheusStore) Series(r *storepb.SeriesRequest, s storepb.Store_Serie
 		return nil
 	}
 	q := prompb.Query{StartTimestampMs: r.MinTime, EndTimestampMs: r.MaxTime}
+	// Check to see if we can limit the mint/maxt for a more efficient query.
+	mint, maxt := p.timestamps()
+	if mint > q.StartTimestampMs {
+		q.StartTimestampMs = mint
+	}
+	if maxt < q.EndTimestampMs {
+		q.EndTimestampMs = maxt
+	}
 
 	// TODO(fabxc): import common definitions from prompb once we have a stable gRPC
 	// query API there.

pkg/store/prometheus_test.go

@@ -58,7 +58,11 @@ func testPrometheusStoreSeriesE2e(t *testing.T, prefix string) {
 	proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar,
 		func() labels.Labels {
 			return labels.FromStrings("region", "eu-west")
-		}, nil)
+		},
+		func() (int64, int64) {
+			return baseT, math.MaxInt64
+		},
+	)
 	testutil.Ok(t, err)
 
 	// Query all three samples except for the first one. Since we round up queried data
@@ -136,7 +140,11 @@ func TestPrometheusStore_LabelValues_e2e(t *testing.T) {
 	u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr()))
 	testutil.Ok(t, err)
 
-	proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, getExternalLabels, nil)
+	proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, getExternalLabels,
+		func() (int64, int64) {
+			return 0, math.MaxInt64
+		},
+	)
 	testutil.Ok(t, err)
 
 	resp, err := proxy.LabelValues(ctx, &storepb.LabelValuesRequest{
@@ -170,7 +178,11 @@ func TestPrometheusStore_ExternalLabelValues_e2e(t *testing.T) {
 	u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr()))
 	testutil.Ok(t, err)
 
-	proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, getExternalLabels, nil)
+	proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar, getExternalLabels,
+		func() (int64, int64) {
+			return 0, math.MaxInt64
+		},
+	)
 	testutil.Ok(t, err)
 
 	resp, err := proxy.LabelValues(ctx, &storepb.LabelValuesRequest{
@@ -217,7 +229,11 @@ func TestPrometheusStore_Series_MatchExternalLabel_e2e(t *testing.T) {
 	proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar,
 		func() labels.Labels {
 			return labels.FromStrings("region", "eu-west")
-		}, nil)
+		},
+		func() (int64, int64) {
+			return baseT, math.MaxInt64
+		},
+	)
 	testutil.Ok(t, err)
 	srv := newStoreSeriesServer(ctx)
 
@@ -345,7 +361,12 @@ func TestPrometheusStore_Series_SplitSamplesIntoChunksWithMaxSizeOfUint16_e2e(t
 		proxy, err := NewPrometheusStore(nil, nil, u, component.Sidecar,
 			func() labels.Labels {
 				return labels.FromStrings("region", "eu-west")
-			}, nil)
+			},
+			func() (int64, int64) {
+				return 0, math.MaxInt64
+			},
+		)
+
 		testutil.Ok(t, err)
 
 		return proxy