From 51e2ab84513f6e8942314e437904527f02d5bb5e Mon Sep 17 00:00:00 2001 From: Martin Dickson Date: Fri, 5 Jul 2019 16:48:19 +0100 Subject: [PATCH] sidecar: wait on ext labels before uploading (#1287) * wait on ext labels before uploading * bump timeout --- cmd/thanos/sidecar.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index ac33a89731..ef7c5ea5fb 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -248,6 +248,17 @@ func runSidecar( g.Add(func() error { defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") + extLabelsTimeout := 10 * time.Minute + extLabelsCtx, _ := context.WithTimeout(ctx, extLabelsTimeout) + if err := runutil.Retry(2*time.Second, extLabelsCtx.Done(), func() error { + if len(m.Labels()) == 0 { + return errors.New("not uploading as no external labels are configured yet - is Prometheus healthy/reachable?") + } + return nil + }); err != nil { + return errors.Wrapf(err, "aborting as no external labels found after waiting %s", extLabelsTimeout) + } + var s *shipper.Shipper if uploadCompacted { s = shipper.NewWithCompacted(logger, reg, dataDir, bkt, m.Labels, metadata.SidecarSource)