Skip to content

Commit

Permalink
Add a new prometheus.ready_timeout CLI option to the sidecar
Browse files Browse the repository at this point in the history
The current timeout while the Prometheus instance is starting up is a constant 10 minutes.
As reading the WAL can take a long time we would like to set a custom timeout value, so the Thanos
sidecar container is not erroring out every 10 minutes.
  • Loading branch information
Andras Ferencz-Szabo committed Oct 17, 2019
1 parent 48a8fb6 commit 6b3685b
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 4 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ We use *breaking* word for marking changes that are not backward compatible (rel

## Unreleased

### Added

- [#1660](https://github.com/thanos-io/thanos/pull/1660) Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus startups up.

## [v0.8.1](https://github.com/thanos-io/thanos/releases/tag/v0.8.1) - 2019.10.14

### Fixed
Expand Down
11 changes: 7 additions & 4 deletions cmd/thanos/sidecar.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@ import (
"gopkg.in/alecthomas/kingpin.v2"
)

const waitForExternalLabelsTimeout = 10 * time.Minute

func registerSidecar(m map[string]setupFunc, app *kingpin.Application) {
cmd := app.Command(component.Sidecar.String(), "sidecar for Prometheus server")

Expand All @@ -43,6 +41,9 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) {
promURL := cmd.Flag("prometheus.url", "URL at which to reach Prometheus's API. For better performance use local network.").
Default("http://localhost:9090").URL()

promReadyTimeout := cmd.Flag("prometheus.ready_timeout", "Maximum time to wait for the Prometheus instance to start up").
Default("10m").Duration()

dataDir := cmd.Flag("tsdb.path", "Data directory of TSDB.").
Default("./data").String()

Expand Down Expand Up @@ -81,6 +82,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) {
*clientCA,
*httpBindAddr,
*promURL,
*promReadyTimeout,
*dataDir,
objStoreConfig,
rl,
Expand All @@ -102,6 +104,7 @@ func runSidecar(
clientCA string,
httpBindAddr string,
promURL *url.URL,
promReadyTimeout time.Duration,
dataDir string,
objStoreConfig *extflag.PathOrContent,
reloader *reloader.Reloader,
Expand Down Expand Up @@ -268,7 +271,7 @@ func runSidecar(
g.Add(func() error {
defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client")

extLabelsCtx, cancel := context.WithTimeout(ctx, waitForExternalLabelsTimeout)
extLabelsCtx, cancel := context.WithTimeout(ctx, promReadyTimeout)
defer cancel()

if err := runutil.Retry(2*time.Second, extLabelsCtx.Done(), func() error {
Expand All @@ -277,7 +280,7 @@ func runSidecar(
}
return nil
}); err != nil {
return errors.Wrapf(err, "aborting as no external labels found after waiting %s", waitForExternalLabelsTimeout)
return errors.Wrapf(err, "aborting as no external labels found after waiting %s", promReadyTimeout)
}

var s *shipper.Shipper
Expand Down
3 changes: 3 additions & 0 deletions docs/components/sidecar.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,9 @@ Flags:
--prometheus.url=http://localhost:9090
URL at which to reach Prometheus's API. For
better performance use local network.
--prometheus.ready_timeout=10m
Maximum time to wait for the Prometheus
instance to start up
--tsdb.path="./data" Data directory of TSDB.
--reloader.config-file="" Config file watched by the reloader.
--reloader.config-envsubst-file=""
Expand Down

0 comments on commit 6b3685b

Please sign in to comment.