From 5fa40d7d1cb105387448061173e067752d96dedd Mon Sep 17 00:00:00 2001 From: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> Date: Thu, 7 Mar 2024 17:53:49 +0300 Subject: [PATCH 1/2] downsample: retry objstore related errors Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> --- cmd/thanos/downsample.go | 5 +++-- pkg/compact/compact.go | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 3d6b129261..ec84fc3d35 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -21,6 +21,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/thanos-io/thanos/pkg/compact" "github.com/thanos-io/objstore" "github.com/thanos-io/objstore/client" @@ -358,7 +359,7 @@ func processDownsampling( err := block.Download(ctx, logger, bkt, m.ULID, bdir, objstore.WithFetchConcurrency(blockFilesConcurrency)) if err != nil { - return errors.Wrapf(err, "download block %s", m.ULID) + return compact.NewRetryError(errors.Wrapf(err, "download block %s", m.ULID)) } level.Info(logger).Log("msg", "downloaded block", "id", m.ULID, "duration", time.Since(begin), "duration_ms", time.Since(begin).Milliseconds()) @@ -419,7 +420,7 @@ func processDownsampling( err = block.Upload(ctx, logger, bkt, resdir, hashFunc) if err != nil { - return errors.Wrapf(err, "upload downsampled block %s", id) + return compact.NewRetryError(errors.Wrapf(err, "upload downsampled block %s", id)) } level.Info(logger).Log("msg", "uploaded block", "id", id, "duration", time.Since(begin), "duration_ms", time.Since(begin).Milliseconds()) diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index edf822434c..ad5c92bdb9 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -967,6 +967,10 @@ type RetryError struct { err error } +func NewRetryError(err error) error { + return retry(err) +} + func retry(err error) error { if IsHaltError(err) { return err From 4c1019448267c446e467de6c9c01bc198e524d7d Mon Sep 17 00:00:00 2001 From: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> Date: Thu, 7 Mar 2024 17:56:13 +0300 Subject: [PATCH 2/2] downsample: retry objstore related errors Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69b05049f8..a81ab8cf24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#7122](https://github.com/thanos-io/thanos/pull/7122) Store Gateway: Fix lazy expanded postings estimate base cardinality using posting group with remove keys. ### Added +- [#7194](https://github.com/thanos-io/thanos/pull/7194) Downsample: retry objstore related errors - [#7105](https://github.com/thanos-io/thanos/pull/7105) Rule: add flag `--query.enable-x-functions` to allow usage of extended promql functions (xrate, xincrease, xdelta) in loaded rules - [#6867](https://github.com/thanos-io/thanos/pull/6867) Query UI: Tenant input box added to the Query UI, in order to be able to specify which tenant the query should use. - [#7175](https://github.com/thanos-io/thanos/pull/7175): Query: Add `--query.mode=distributed` which enables the new distributed mode of the Thanos query engine.