Document downsample component

cmd/thanos/bucket.go

@@ -73,6 +73,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin
 	registerBucketInspect(m, cmd, name, objStoreConfig)
 	registerBucketWeb(m, cmd, name, objStoreConfig)
 	registerBucketReplicate(m, cmd, name, objStoreConfig)
+	registerBucketDownsample(m, cmd, name, objStoreConfig)
 }
 
 func registerBucketVerify(m map[string]setupFunc, root *kingpin.CmdClause, name string, objStoreConfig *extflag.PathOrContent) {
@@ -424,6 +425,21 @@ func registerBucketReplicate(m map[string]setupFunc, root *kingpin.CmdClause, na
 
 }
 
+func registerBucketDownsample(m map[string]setupFunc, root *kingpin.CmdClause, name string, objStoreConfig *extflag.PathOrContent) {
+
+	comp := component.Downsample
+	cmd := root.Command(comp.String(), "continuously downsamples blocks in an object store bucket")
+
+	httpAddr, httpGracePeriod := regHTTPFlags(cmd)
+
+	dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings.").
+		Default("./data").String()
+
+	m[name+" "+comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error {
+		return RunDownsample(g, logger, reg, *httpAddr, time.Duration(*httpGracePeriod), *dataDir, objStoreConfig, comp)
+	}
+}
+
 // refresh metadata from remote storage periodically and update UI.
 func refresh(ctx context.Context, logger log.Logger, bucketUI *ui.Bucket, duration time.Duration, timeout time.Duration, name string, reg *prometheus.Registry, objStoreConfig *extflag.PathOrContent) error {
 	confContentYaml, err := objStoreConfig.Content()

cmd/thanos/downsample.go

@@ -13,7 +13,6 @@ import (
 	"github.com/go-kit/kit/log/level"
 	"github.com/oklog/run"
 	"github.com/oklog/ulid"
-	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -31,25 +30,8 @@ import (
 	"github.com/thanos-io/thanos/pkg/prober"
 	"github.com/thanos-io/thanos/pkg/runutil"
 	httpserver "github.com/thanos-io/thanos/pkg/server/http"
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
-func registerDownsample(m map[string]setupFunc, app *kingpin.Application) {
-	comp := component.Downsample
-	cmd := app.Command(comp.String(), "continuously downsamples blocks in an object store bucket")
-
-	httpAddr, httpGracePeriod := regHTTPFlags(cmd)
-
-	dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings.").
-		Default("./data").String()
-
-	objStoreConfig := regCommonObjStoreFlags(cmd, "", true)
-
-	m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error {
-		return runDownsample(g, logger, reg, *httpAddr, time.Duration(*httpGracePeriod), *dataDir, objStoreConfig, comp)
-	}
-}
-
 type DownsampleMetrics struct {
 	downsamples        *prometheus.CounterVec
 	downsampleFailures *prometheus.CounterVec
@@ -70,7 +52,7 @@ func newDownsampleMetrics(reg *prometheus.Registry) *DownsampleMetrics {
 	return m
 }
 
-func runDownsample(
+func RunDownsample(
 	g *run.Group,
 	logger log.Logger,
 	reg *prometheus.Registry,

cmd/thanos/main.go

@@ -61,7 +61,6 @@ func main() {
 	registerRule(cmds, app)
 	registerCompact(cmds, app)
 	registerBucket(cmds, app, "bucket")
-	registerDownsample(cmds, app)
 	registerReceive(cmds, app)
 	registerChecks(cmds, app, "check")
 

docs/components/bucket.md

@@ -78,6 +78,9 @@ Subcommands:
     Replicate data from one object storage to another. NOTE: Currently it works
     only with Thanos blocks (meta.json has to have Thanos metadata).
 
+  bucket downsample [<flags>]
+    continuously downsamples blocks in an object store bucket
+
 
 ```
 
@@ -389,3 +392,71 @@ Flags:
       --single-run               Run replication only one time, then exit.
 
 ```
+
+### Downsample
+
+The downsample component of Thanos implements the downsample API on top of historical data in an object storage bucket. It continuously downsamples blocks in an object store bucket as a service.
+
+```bash
+$ thanos downsample \
+    --data-dir        "/local/state/data/dir" \
+    --objstore.config-file "bucket.yml"
+```
+
+The content of `bucket.yml`:
+
+```yaml
+type: GCS
+config:
+  bucket: example-bucket
+```
+
+#### Flags
+
+[embedmd]:# (flags/bucket_downsample.txt $)
+```$
+usage: thanos bucket downsample [<flags>]
+
+continuously downsamples blocks in an object store bucket
+
+Flags:
+  -h, --help                  Show context-sensitive help (also try --help-long
+                              and --help-man).
+      --version               Show application version.
+      --log.level=info        Log filtering level.
+      --log.format=logfmt     Log format to use. Possible options: logfmt or
+                              json.
+      --tracing.config-file=<file-path>
+                              Path to YAML file with tracing configuration. See
+                              format details:
+                              https://thanos.io/tracing.md/#configuration
+      --tracing.config=<content>
+                              Alternative to 'tracing.config-file' flag (lower
+                              priority). Content of YAML file with tracing
+                              configuration. See format details:
+                              https://thanos.io/tracing.md/#configuration
+      --objstore.config-file=<file-path>
+                              Path to YAML file that contains object store
+                              configuration. See format details:
+                              https://thanos.io/storage.md/#configuration
+      --objstore.config=<content>
+                              Alternative to 'objstore.config-file' flag (lower
+                              priority). Content of YAML file that contains
+                              object store configuration. See format details:
+                              https://thanos.io/storage.md/#configuration
+      --http-address="0.0.0.0:10902"
+                              Listen host:port for HTTP endpoints.
+      --http-grace-period=2m  Time to wait after an interrupt received for HTTP
+                              Server.
+      --data-dir="./data"     Data directory in which to cache blocks and
+                              process downsamplings.
+
+```
+
+#### Probes
+
+- Thanos downsample exposes two endpoints for probing.
+  - `/-/healthy` starts as soon as initial setup completed.
+  - `/-/ready` starts after all the bootstrapping completed (e.g object store bucket connection) and ready to serve traffic.
+
+> NOTE: Metric endpoint starts immediately so, make sure you set up readiness probe on designated HTTP `/-/ready` path.

scripts/genflagdocs.sh

@@ -40,7 +40,7 @@ for x in "${commands[@]}"; do
     ./thanos "${x}" --help &> "docs/components/flags/${x}.txt"
 done
 
-bucketCommands=("verify" "ls" "inspect" "web" "replicate")
+bucketCommands=("verify" "ls" "inspect" "web" "replicate" "downsample")
 for x in "${bucketCommands[@]}"; do
     ./thanos bucket "${x}" --help &> "docs/components/flags/bucket_${x}.txt"
 done