Update docs/components/compact.md

accept PR suggestions

Co-Authored-By: Bartlomiej Plotka <[email protected]>
Signed-off-by: Ivan Kiselev <[email protected]>

CHANGELOG.md

@@ -12,13 +12,17 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 ## Unreleased
 
 ## Added
-- [#1538](https://github.com/thanos-io/thanos/pull/1538) Added `/-/ready` and `/-/healthy` endpoints to Thanos Rule.
--[1533](https://github.com/thanos-io/thanos/pull/1533) Thanos inspect now supports the timeout flag.
+- [#1540](https://github.com/thanos-io/thanos/pull/1540) Thanos Downsample added `/-/ready` and `/-/healthy` endpoints.
+- [#1538](https://github.com/thanos-io/thanos/pull/1538) Thanos Rule added `/-/ready` and `/-/healthy` endpoints.
+- [#1537](https://github.com/thanos-io/thanos/pull/1537) Thanos Receive added `/-/ready` and `/-/healthy` endpoints.
+- [#1460](https://github.com/thanos-io/thanos/pull/1460) Thanos Store Added `/-/ready` and `/-/healthy` endpoints.
+- [#1534](https://github.com/thanos-io/thanos/pull/1534) Thanos Query Added `/-/ready` and `/-/healthy` endpoints.
+- [#1533](https://github.com/thanos-io/thanos/pull/1533) Thanos inspect now supports the timeout flag.
 
 ### Fixed
 
--[#1525](https://github.com/thanos-io/thanos/pull/1525) Thanos now deletes block's file in correct order allowing to detect partial blocks without problems.
--[#1505](https://github.com/thanos-io/thanos/pull/1505) Thanos store now removes invalid local cache blocks.
+- [#1525](https://github.com/thanos-io/thanos/pull/1525) Thanos now deletes block's file in correct order allowing to detect partial blocks without problems.
+- [#1505](https://github.com/thanos-io/thanos/pull/1505) Thanos store now removes invalid local cache blocks.
 
 ## v0.7.0 - 2019.09.02
 

MAINTAINERS.md

@@ -7,10 +7,11 @@
 | Frederic Branczyk     | [email protected]    | `@brancz`                | [@brancz](https://github.com/brancz)                       |
 | Giedrius Statkevičius | [email protected] | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS)                 |
 | Povilas Versockas     | [email protected]  | `@povilasv`              | [@povilasv](https://github.com/povilasv)                   |
+| Matthias Loibl        | [email protected] | `@metalmatze`            | [@metalmatze](https://github.com/metalmatze)               |
 
 We are bunch of people from different companies with various interests and skills.
 We are from different parts of Europe: Germany, Lithuania, Poland and UK. 
-We have something in common though: We all share the love for OpenSource, Golang, Prometheus, :coffee: and Observability topics.
+We have something in common though: We all share the love for OpenSource, Go, Prometheus, :coffee: and Observability topics.
 
 As either Software Developers or SRE (or both!) we've chosen to maintain (mostly in our free time) Thanos, the de facto way to scale awesome [Prometheus](https://prometheus.io) project.
 
@@ -96,4 +97,5 @@ maintainer team.
 
 ## Initial authors
 
-Fabian Reinartz @fabxc and Bartłomiej Płotka @bwplotka
+Fabian Reinartz @fabxc and Bartłomiej Płotka @bwplotka
+

cmd/thanos/bucket.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/thanos-io/thanos/pkg/block"
 	"github.com/thanos-io/thanos/pkg/block/metadata"
+	"github.com/thanos-io/thanos/pkg/compact"
 	extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http"
 	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/objstore/client"
@@ -50,7 +51,7 @@ var (
 		sort.Strings(s)
 		return s
 	}
-	inspectColumns = []string{"ULID", "FROM", "UNTIL", "RANGE", "UNTIL-COMP", "#SERIES", "#SAMPLES", "#CHUNKS", "COMP-LEVEL", "COMP-FAILED", "LABELS", "RESOLUTION", "SOURCE"}
+	inspectColumns = []string{"ULID", "FROM", "UNTIL", "RANGE", "UNTIL-DOWN", "#SERIES", "#SAMPLES", "#CHUNKS", "COMP-LEVEL", "COMP-FAILED", "LABELS", "RESOLUTION", "SOURCE"}
 )
 
 func registerBucket(m map[string]setupFunc, app *kingpin.Application, name string) {
@@ -420,13 +421,10 @@ func printTable(blockMetas []*metadata.Meta, selectorLabels labels.Labels, sortB
 		}
 
 		timeRange := time.Duration((blockMeta.MaxTime - blockMeta.MinTime) * int64(time.Millisecond))
-		// Calculate how long it takes until the next compaction.
-		untilComp := "-"
-		if blockMeta.Thanos.Downsample.Resolution == 0 { // data currently raw, downsample if range >= 40 hours
-			untilComp = (time.Duration(40*60*60*1000*time.Millisecond) - timeRange).String()
-		}
-		if blockMeta.Thanos.Downsample.Resolution == 5*60*1000 { // data currently 5m resolution, downsample if range >= 10 days
-			untilComp = (time.Duration(10*24*60*60*1000*time.Millisecond) - timeRange).String()
+
+		untilDown := "-"
+		if until, err := compact.UntilNextDownsampling(blockMeta); err == nil {
+			untilDown = until.String()
 		}
 		var labels []string
 		for _, key := range getKeysAlphabetically(blockMeta.Thanos.Labels) {
@@ -438,7 +436,7 @@ func printTable(blockMetas []*metadata.Meta, selectorLabels labels.Labels, sortB
 		line = append(line, time.Unix(blockMeta.MinTime/1000, 0).Format("02-01-2006 15:04:05"))
 		line = append(line, time.Unix(blockMeta.MaxTime/1000, 0).Format("02-01-2006 15:04:05"))
 		line = append(line, timeRange.String())
-		line = append(line, untilComp)
+		line = append(line, untilDown)
 		line = append(line, p.Sprintf("%d", blockMeta.Stats.NumSeries))
 		line = append(line, p.Sprintf("%d", blockMeta.Stats.NumSamples))
 		line = append(line, p.Sprintf("%d", blockMeta.Stats.NumChunks))

cmd/thanos/compact.go

@@ -168,9 +168,9 @@ func runCompact(
 	downsampleMetrics := newDownsampleMetrics(reg)
 
 	statusProber := prober.NewProber(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
-	// Initiate default HTTP listener providing metrics endpoint and readiness/liveness probes.
+	// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
 	if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, component); err != nil {
-		return errors.Wrap(err, "create default HTTP server with readiness prober")
+		return errors.Wrap(err, "schedule HTTP server with probes")
 	}
 
 	confContentYaml, err := objStoreConfig.Content()

cmd/thanos/downsample.go

@@ -22,12 +22,14 @@ import (
 	"github.com/thanos-io/thanos/pkg/component"
 	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/objstore/client"
+	"github.com/thanos-io/thanos/pkg/prober"
 	"github.com/thanos-io/thanos/pkg/runutil"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
-func registerDownsample(m map[string]setupFunc, app *kingpin.Application, name string) {
-	cmd := app.Command(name, "continuously downsamples blocks in an object store bucket")
+func registerDownsample(m map[string]setupFunc, app *kingpin.Application) {
+	comp := component.Downsample
+	cmd := app.Command(comp.String(), "continuously downsamples blocks in an object store bucket")
 
 	httpAddr := regHTTPAddrFlag(cmd)
 
@@ -36,8 +38,8 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application, name s
 
 	objStoreConfig := regCommonObjStoreFlags(cmd, "", true)
 
-	m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error {
-		return runDownsample(g, logger, reg, *httpAddr, *dataDir, objStoreConfig)
+	m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error {
+		return runDownsample(g, logger, reg, *httpAddr, *dataDir, objStoreConfig, comp)
 	}
 }
 
@@ -71,6 +73,7 @@ func runDownsample(
 	httpBindAddr string,
 	dataDir string,
 	objStoreConfig *pathOrContent,
+	comp component.Component,
 ) error {
 	confContentYaml, err := objStoreConfig.Content()
 	if err != nil {
@@ -90,13 +93,14 @@ func runDownsample(
 	}()
 
 	metrics := newDownsampleMetrics(reg)
-
+	statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
 	// Start cycle of syncing blocks from the bucket and garbage collecting the bucket.
 	{
 		ctx, cancel := context.WithCancel(context.Background())
 
 		g.Add(func() error {
 			defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client")
+			statusProber.SetReady()
 
 			level.Info(logger).Log("msg", "start first pass of downsampling")
 
@@ -116,8 +120,9 @@ func runDownsample(
 		})
 	}
 
-	if err := metricHTTPListenGroup(g, logger, reg, httpBindAddr); err != nil {
-		return err
+	// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
+	if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, comp); err != nil {
+		return errors.Wrap(err, "schedule HTTP server with probe")
 	}
 
 	level.Info(logger).Log("msg", "starting downsample node")
@@ -165,13 +170,13 @@ func downsampleBucket(
 
 	for _, m := range metas {
 		switch m.Thanos.Downsample.Resolution {
-		case 0:
+		case downsample.ResLevel0:
 			continue
-		case 5 * 60 * 1000:
+		case downsample.ResLevel1:
 			for _, id := range m.Compaction.Sources {
 				sources5m[id] = struct{}{}
 			}
-		case 60 * 60 * 1000:
+		case downsample.ResLevel2:
 			for _, id := range m.Compaction.Sources {
 				sources1h[id] = struct{}{}
 			}
@@ -182,7 +187,7 @@ func downsampleBucket(
 
 	for _, m := range metas {
 		switch m.Thanos.Downsample.Resolution {
-		case 0:
+		case downsample.ResLevel0:
 			missing := false
 			for _, id := range m.Compaction.Sources {
 				if _, ok := sources5m[id]; !ok {
@@ -196,16 +201,16 @@ func downsampleBucket(
 			// Only downsample blocks once we are sure to get roughly 2 chunks out of it.
 			// NOTE(fabxc): this must match with at which block size the compactor creates downsampled
 			// blocks. Otherwise we may never downsample some data.
-			if m.MaxTime-m.MinTime < 40*60*60*1000 {
+			if m.MaxTime-m.MinTime < downsample.DownsampleRange0 {
 				continue
 			}
-			if err := processDownsampling(ctx, logger, bkt, m, dir, 5*60*1000); err != nil {
+			if err := processDownsampling(ctx, logger, bkt, m, dir, downsample.ResLevel1); err != nil {
 				metrics.downsampleFailures.WithLabelValues(compact.GroupKey(*m)).Inc()
 				return errors.Wrap(err, "downsampling to 5 min")
 			}
 			metrics.downsamples.WithLabelValues(compact.GroupKey(*m)).Inc()
 
-		case 5 * 60 * 1000:
+		case downsample.ResLevel1:
 			missing := false
 			for _, id := range m.Compaction.Sources {
 				if _, ok := sources1h[id]; !ok {
@@ -219,10 +224,10 @@ func downsampleBucket(
 			// Only downsample blocks once we are sure to get roughly 2 chunks out of it.
 			// NOTE(fabxc): this must match with at which block size the compactor creates downsampled
 			// blocks. Otherwise we may never downsample some data.
-			if m.MaxTime-m.MinTime < 10*24*60*60*1000 {
+			if m.MaxTime-m.MinTime < downsample.DownsampleRange1 {
 				continue
 			}
-			if err := processDownsampling(ctx, logger, bkt, m, dir, 60*60*1000); err != nil {
+			if err := processDownsampling(ctx, logger, bkt, m, dir, downsample.ResLevel2); err != nil {
 				metrics.downsampleFailures.WithLabelValues(compact.GroupKey(*m))
 				return errors.Wrap(err, "downsampling to 60 min")
 			}

cmd/thanos/flags.go

@@ -118,7 +118,7 @@ func regCommonTracingFlags(app *kingpin.Application) *pathOrContent {
 	fileFlagName := fmt.Sprintf("tracing.config-file")
 	contentFlagName := fmt.Sprintf("tracing.config")
 
-	help := fmt.Sprintf("Path to YAML file that contains tracing configuration. See fomrat details: https://thanos.io/tracing.md/#configuration ")
+	help := fmt.Sprintf("Path to YAML file that contains tracing configuration. See format details: https://thanos.io/tracing.md/#configuration ")
 	tracingConfFile := app.Flag(fileFlagName, help).PlaceHolder("<tracing.config-yaml-path>").String()
 
 	help = fmt.Sprintf("Alternative to '%s' flag. Tracing configuration in YAML. See format details: https://thanos.io/tracing.md/#configuration", fileFlagName)

cmd/thanos/main.go

@@ -74,13 +74,13 @@ func main() {
 
 	cmds := map[string]setupFunc{}
 	registerSidecar(cmds, app)
-	registerStore(cmds, app, "store")
+	registerStore(cmds, app)
 	registerQuery(cmds, app)
 	registerRule(cmds, app)
 	registerCompact(cmds, app)
 	registerBucket(cmds, app, "bucket")
-	registerDownsample(cmds, app, "downsample")
-	registerReceive(cmds, app, "receive")
+	registerDownsample(cmds, app)
+	registerReceive(cmds, app)
 	registerChecks(cmds, app, "check")
 
 	cmd, err := app.Parse(os.Args[1:])
@@ -333,27 +333,6 @@ func newStoreGRPCServer(logger log.Logger, reg *prometheus.Registry, tracer open
 	return s
 }
 
-// TODO Remove once all components are migrated to the new scheduleHTTPServer.
-// metricHTTPListenGroup is a run.Group that servers HTTP endpoint with only Prometheus metrics.
-func metricHTTPListenGroup(g *run.Group, logger log.Logger, reg *prometheus.Registry, httpBindAddr string) error {
-	mux := http.NewServeMux()
-	registerMetrics(mux, reg)
-	registerProfile(mux)
-
-	l, err := net.Listen("tcp", httpBindAddr)
-	if err != nil {
-		return errors.Wrap(err, "listen metrics address")
-	}
-
-	g.Add(func() error {
-		level.Info(logger).Log("msg", "Listening for metrics", "address", httpBindAddr)
-		return errors.Wrap(http.Serve(l, mux), "serve metrics")
-	}, func(error) {
-		runutil.CloseWithLogOnErr(logger, l, "metric listener")
-	})
-	return nil
-}
-
 // scheduleHTTPServer starts a run.Group that servers HTTP endpoint with default endpoints providing Prometheus metrics,
 // profiling and liveness/readiness probes.
 func scheduleHTTPServer(g *run.Group, logger log.Logger, reg *prometheus.Registry, readinessProber *prober.Prober, httpBindAddr string, handler http.Handler, comp component.Component) error {

cmd/thanos/query.go

@@ -414,9 +414,9 @@ func runQuery(
 
 		api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins)
 
-		// Initiate default HTTP listener providing metrics endpoint and readiness/liveness probes.
+		// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
 		if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, router, comp); err != nil {
-			return errors.Wrap(err, "create default HTTP server with readiness prober")
+			return errors.Wrap(err, "schedule HTTP server with probes")
 		}
 	}
 	// Start query (proxy) gRPC StoreAPI.