From c8e60b8911494fbad916d5d7f3fb43ddd1ba1d32 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Thu, 20 Jul 2023 15:56:35 +0100 Subject: [PATCH 01/11] [FEAT] Add uploaded bytes metric Signed-off-by: rita.canavarro --- CHANGELOG.md | 1 + pkg/shipper/shipper.go | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5ac24d927..4038693173 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -272,6 +272,7 @@ NOTE: Querier's `query.promql-engine` flag enabling new PromQL engine is now unh - [#5741](https://github.com/thanos-io/thanos/pull/5741) Query: add metrics on how much data is being selected by downstream Store APIs. - [#5673](https://github.com/thanos-io/thanos/pull/5673) Receive: Reload tenant limit configuration on file change. - [#5749](https://github.com/thanos-io/thanos/pull/5749) Query Frontend: Added small LRU cache to cache query analysis results. +- [#6500](https://github.com/thanos-io/thanos/pull/6500) Shipper: Add metric `thanos_shipper_uploaded_bytes_total` for number of total uploaded blocks. ### Changed diff --git a/pkg/shipper/shipper.go b/pkg/shipper/shipper.go index d3562d87b1..e567678a1a 100644 --- a/pkg/shipper/shipper.go +++ b/pkg/shipper/shipper.go @@ -38,6 +38,7 @@ type metrics struct { uploads prometheus.Counter uploadFailures prometheus.Counter uploadedCompacted prometheus.Gauge + uploadedBytes prometheus.Counter } func newMetrics(reg prometheus.Registerer) *metrics { @@ -63,6 +64,11 @@ func newMetrics(reg prometheus.Registerer) *metrics { Name: "thanos_shipper_upload_compacted_done", Help: "If 1 it means shipper uploaded all compacted blocks from the filesystem.", }) + m.uploadedBytes = promauto.With(reg).NewCounter(prometheus.CounterOpts{ + Name: "thanos_shipper_uploaded_bytes_total", + Help: "Total number of uploaded bytes.", + }) + return &m } @@ -383,7 +389,25 @@ func (s *Shipper) upload(ctx context.Context, meta *metadata.Meta) error { if err := meta.WriteToDir(s.logger, updir); err != nil { return errors.Wrap(err, "write meta file") } - return block.Upload(ctx, s.logger, s.bucket, updir, s.hashFunc) + + err := block.Upload(ctx, s.logger, s.bucket, updir, s.hashFunc) + + if err != nil { + return errors.Wrap(err, "while upploading the block") + } + + files, err := block.GatherFileStats(updir, s.hashFunc, s.logger) + + if err != nil { + //The block upload should not stop due to issues gathering data for a metric + return nil + } + + for _, x := range files { + s.metrics.uploadedBytes.Add(float64(x.SizeBytes)) + } + + return nil } // blockMetasFromOldest returns the block meta of each block found in dir From 5968d0530ee8f0b2c7fe774276bc839a3ffcc54e Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Thu, 20 Jul 2023 16:15:59 +0100 Subject: [PATCH 02/11] [FEAT] Add PR number to log Signed-off-by: rita.canavarro --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4038693173..472980a5f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -272,7 +272,7 @@ NOTE: Querier's `query.promql-engine` flag enabling new PromQL engine is now unh - [#5741](https://github.com/thanos-io/thanos/pull/5741) Query: add metrics on how much data is being selected by downstream Store APIs. - [#5673](https://github.com/thanos-io/thanos/pull/5673) Receive: Reload tenant limit configuration on file change. - [#5749](https://github.com/thanos-io/thanos/pull/5749) Query Frontend: Added small LRU cache to cache query analysis results. -- [#6500](https://github.com/thanos-io/thanos/pull/6500) Shipper: Add metric `thanos_shipper_uploaded_bytes_total` for number of total uploaded blocks. +- [#6544](https://github.com/thanos-io/thanos/pull/6500) Shipper: Add metric `thanos_shipper_uploaded_bytes_total` for number of total uploaded blocks. ### Changed From 0a5a9503d218ba607561c084a4132dc07a26ee97 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Thu, 20 Jul 2023 16:33:02 +0100 Subject: [PATCH 03/11] [FIX] Log msg Signed-off-by: rita.canavarro --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 472980a5f0..ceec4844b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -272,7 +272,7 @@ NOTE: Querier's `query.promql-engine` flag enabling new PromQL engine is now unh - [#5741](https://github.com/thanos-io/thanos/pull/5741) Query: add metrics on how much data is being selected by downstream Store APIs. - [#5673](https://github.com/thanos-io/thanos/pull/5673) Receive: Reload tenant limit configuration on file change. - [#5749](https://github.com/thanos-io/thanos/pull/5749) Query Frontend: Added small LRU cache to cache query analysis results. -- [#6544](https://github.com/thanos-io/thanos/pull/6500) Shipper: Add metric `thanos_shipper_uploaded_bytes_total` for number of total uploaded blocks. +- [#6544](https://github.com/thanos-io/thanos/pull/6500) Shipper: Add metric `thanos_shipper_uploaded_bytes_total` for the number of total uploaded blocks. ### Changed From 981dbb99e84ba8707f9729ca3acddc4c9191aec0 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Fri, 21 Jul 2023 11:23:13 +0100 Subject: [PATCH 04/11] [FEAT] Clean code Signed-off-by: rita.canavarro --- pkg/shipper/shipper.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/shipper/shipper.go b/pkg/shipper/shipper.go index e567678a1a..36ecb20846 100644 --- a/pkg/shipper/shipper.go +++ b/pkg/shipper/shipper.go @@ -66,7 +66,7 @@ func newMetrics(reg prometheus.Registerer) *metrics { }) m.uploadedBytes = promauto.With(reg).NewCounter(prometheus.CounterOpts{ Name: "thanos_shipper_uploaded_bytes_total", - Help: "Total number of uploaded bytes.", + Help: "Total number of uploaded bytes from TSDB blocks.", }) return &m @@ -391,19 +391,17 @@ func (s *Shipper) upload(ctx context.Context, meta *metadata.Meta) error { } err := block.Upload(ctx, s.logger, s.bucket, updir, s.hashFunc) - if err != nil { return errors.Wrap(err, "while upploading the block") } - files, err := block.GatherFileStats(updir, s.hashFunc, s.logger) - + fileStats, err := block.GatherFileStats(updir, s.hashFunc, s.logger) if err != nil { - //The block upload should not stop due to issues gathering data for a metric + // The block upload should not stop due to issues gathering data for a metric. return nil } - for _, x := range files { + for _, x := range fileStats { s.metrics.uploadedBytes.Add(float64(x.SizeBytes)) } From 9d8be2ff3c85d2686fa3da7dbd91a9f36121180e Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Fri, 21 Jul 2023 14:35:31 +0100 Subject: [PATCH 05/11] [FIX] Remove shadow code Signed-off-by: rita.canavarro --- pkg/shipper/shipper.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/shipper/shipper.go b/pkg/shipper/shipper.go index 36ecb20846..0f7646fb11 100644 --- a/pkg/shipper/shipper.go +++ b/pkg/shipper/shipper.go @@ -390,13 +390,11 @@ func (s *Shipper) upload(ctx context.Context, meta *metadata.Meta) error { return errors.Wrap(err, "write meta file") } - err := block.Upload(ctx, s.logger, s.bucket, updir, s.hashFunc) - if err != nil { - return errors.Wrap(err, "while upploading the block") + err := block.Upload(ctx, s.logger, s.bucket, updir, s.hashFunc); if err != nil { + return errors.Wrap(err, "while uploading the block") } - fileStats, err := block.GatherFileStats(updir, s.hashFunc, s.logger) - if err != nil { + fileStats, err := block.GatherFileStats(updir, s.hashFunc, s.logger); if err != nil { // The block upload should not stop due to issues gathering data for a metric. return nil } From 7f97b2eb49ee5adb0e68a3b2b241a8bec4cc6e60 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Fri, 21 Jul 2023 15:25:58 +0100 Subject: [PATCH 06/11] [FIX] Go format Signed-off-by: rita.canavarro --- pkg/shipper/shipper.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/shipper/shipper.go b/pkg/shipper/shipper.go index 0f7646fb11..5c64b93a67 100644 --- a/pkg/shipper/shipper.go +++ b/pkg/shipper/shipper.go @@ -390,11 +390,13 @@ func (s *Shipper) upload(ctx context.Context, meta *metadata.Meta) error { return errors.Wrap(err, "write meta file") } - err := block.Upload(ctx, s.logger, s.bucket, updir, s.hashFunc); if err != nil { + err := block.Upload(ctx, s.logger, s.bucket, updir, s.hashFunc) + if err != nil { return errors.Wrap(err, "while uploading the block") } - fileStats, err := block.GatherFileStats(updir, s.hashFunc, s.logger); if err != nil { + fileStats, err := block.GatherFileStats(updir, s.hashFunc, s.logger) + if err != nil { // The block upload should not stop due to issues gathering data for a metric. return nil } From 78d8d84deb6064e28a1d41ba14d784611f51f398 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 11 Sep 2023 16:22:09 +0100 Subject: [PATCH 07/11] [FEAT] Update objstore Signed-off-by: rita.canavarro --- CHANGELOG.md | 2 +- pkg/shipper/shipper.go | 23 +---------------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ceec4844b5..b28748d9fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -272,7 +272,7 @@ NOTE: Querier's `query.promql-engine` flag enabling new PromQL engine is now unh - [#5741](https://github.com/thanos-io/thanos/pull/5741) Query: add metrics on how much data is being selected by downstream Store APIs. - [#5673](https://github.com/thanos-io/thanos/pull/5673) Receive: Reload tenant limit configuration on file change. - [#5749](https://github.com/thanos-io/thanos/pull/5749) Query Frontend: Added small LRU cache to cache query analysis results. -- [#6544](https://github.com/thanos-io/thanos/pull/6500) Shipper: Add metric `thanos_shipper_uploaded_bytes_total` for the number of total uploaded blocks. +- [#6544](https://github.com/thanos-io/thanos/pull/6500) Objstore: Update objstore to latest version which adds a new metric regarding uploaded TSDB bytes ### Changed diff --git a/pkg/shipper/shipper.go b/pkg/shipper/shipper.go index 5c64b93a67..4d0d77cc16 100644 --- a/pkg/shipper/shipper.go +++ b/pkg/shipper/shipper.go @@ -38,7 +38,6 @@ type metrics struct { uploads prometheus.Counter uploadFailures prometheus.Counter uploadedCompacted prometheus.Gauge - uploadedBytes prometheus.Counter } func newMetrics(reg prometheus.Registerer) *metrics { @@ -64,10 +63,6 @@ func newMetrics(reg prometheus.Registerer) *metrics { Name: "thanos_shipper_upload_compacted_done", Help: "If 1 it means shipper uploaded all compacted blocks from the filesystem.", }) - m.uploadedBytes = promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "thanos_shipper_uploaded_bytes_total", - Help: "Total number of uploaded bytes from TSDB blocks.", - }) return &m } @@ -389,23 +384,7 @@ func (s *Shipper) upload(ctx context.Context, meta *metadata.Meta) error { if err := meta.WriteToDir(s.logger, updir); err != nil { return errors.Wrap(err, "write meta file") } - - err := block.Upload(ctx, s.logger, s.bucket, updir, s.hashFunc) - if err != nil { - return errors.Wrap(err, "while uploading the block") - } - - fileStats, err := block.GatherFileStats(updir, s.hashFunc, s.logger) - if err != nil { - // The block upload should not stop due to issues gathering data for a metric. - return nil - } - - for _, x := range fileStats { - s.metrics.uploadedBytes.Add(float64(x.SizeBytes)) - } - - return nil + return block.Upload(ctx, s.logger, s.bucket, updir, s.hashFunc) } // blockMetasFromOldest returns the block meta of each block found in dir From 2317b307ac2e0f860cfa58d23d5d9994838b2528 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 12 Sep 2023 12:51:43 +0100 Subject: [PATCH 08/11] [FEAT] Update objstore package Signed-off-by: rita.canavarro --- go.mod | 2 +- go.sum | 4 ++-- pkg/shipper/shipper.go | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 7843a69f4c..be47d64ae0 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( github.com/prometheus/prometheus v0.46.1-0.20230818184859-4d8e380269da github.com/sony/gobreaker v0.5.0 github.com/stretchr/testify v1.8.4 - github.com/thanos-io/objstore v0.0.0-20230804084840-c042a6a16c58 + github.com/thanos-io/objstore v0.0.0-20230908084555-8d397d4d88e7 github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible // indirect diff --git a/go.sum b/go.sum index 54398c806f..2044edb79a 100644 --- a/go.sum +++ b/go.sum @@ -906,8 +906,8 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4 github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= -github.com/thanos-io/objstore v0.0.0-20230804084840-c042a6a16c58 h1:4cDXsvm3mb1NvW1B1qJ9/fy6h+OOYit0h8oVA957hLM= -github.com/thanos-io/objstore v0.0.0-20230804084840-c042a6a16c58/go.mod h1:oJ82xgcBDzGJrEgUsjlTj6n01+ZWUMMUR8BlZzX5xDE= +github.com/thanos-io/objstore v0.0.0-20230908084555-8d397d4d88e7 h1:P1mukL6u3wKv4gRLjhnEYltZf8k5dXkE7y7UvEJo0fU= +github.com/thanos-io/objstore v0.0.0-20230908084555-8d397d4d88e7/go.mod h1:oJ82xgcBDzGJrEgUsjlTj6n01+ZWUMMUR8BlZzX5xDE= github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e h1:kwsFCU8eSkZehbrAN3nXPw5RdMHi/Bok/y8l2C4M+gk= github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e/go.mod h1:+T/ZYNCGybT6eTsGGvVtGb63nT1cvUmH6MjqRrcQoKw= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= diff --git a/pkg/shipper/shipper.go b/pkg/shipper/shipper.go index 4d0d77cc16..d3562d87b1 100644 --- a/pkg/shipper/shipper.go +++ b/pkg/shipper/shipper.go @@ -63,7 +63,6 @@ func newMetrics(reg prometheus.Registerer) *metrics { Name: "thanos_shipper_upload_compacted_done", Help: "If 1 it means shipper uploaded all compacted blocks from the filesystem.", }) - return &m } From 1e0e03c3d9ab766395fcf7b4d384645c9b176367 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 12 Sep 2023 14:18:27 +0100 Subject: [PATCH 09/11] [FEAT] Update storage.md Signed-off-by: rita.canavarro --- docs/storage.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/storage.md b/docs/storage.md index 24634ad3c7..f1f331cb40 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -357,6 +357,7 @@ type: AZURE config: storage_account: "" storage_account_key: "" + storage_connection_string: "" container: "" endpoint: "" user_assigned_id: "" From e96749e17e565390d2a4616362fd92ae1beb7009 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 12 Sep 2023 14:45:38 +0100 Subject: [PATCH 10/11] [FEAT] Update erroring bucket Signed-off-by: rita.canavarro --- cmd/thanos/main_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/thanos/main_test.go b/cmd/thanos/main_test.go index 7feca6162b..e3f17e6cdf 100644 --- a/cmd/thanos/main_test.go +++ b/cmd/thanos/main_test.go @@ -77,9 +77,9 @@ func (b *erroringBucket) IsObjNotFoundErr(err error) bool { return b.bkt.IsObjNotFoundErr(err) } -// IsCustomerManagedKeyError returns true if error means that customer managed key is invalid. -func (b *erroringBucket) IsCustomerManagedKeyError(err error) bool { - return b.bkt.IsCustomerManagedKeyError(err) +// IsAccessDeniedErr returns true if error means that access was denied. +func (b *erroringBucket) IsAccessDeniedErr(err error) bool { + return b.bkt.IsAccessDeniedErr(err) } // Attributes returns information about the specified object. From 9eeca84f9e1ca424ab7e0bbf67b940dbe1c97a01 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 12 Sep 2023 14:47:51 +0100 Subject: [PATCH 11/11] [FEAT] Update erroring bucket Signed-off-by: rita.canavarro --- cmd/thanos/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/thanos/main_test.go b/cmd/thanos/main_test.go index e3f17e6cdf..3d6a9ee85f 100644 --- a/cmd/thanos/main_test.go +++ b/cmd/thanos/main_test.go @@ -77,7 +77,7 @@ func (b *erroringBucket) IsObjNotFoundErr(err error) bool { return b.bkt.IsObjNotFoundErr(err) } -// IsAccessDeniedErr returns true if error means that access was denied. +// IsAccessDeniedErr returns true if error means that access to the object was denied. func (b *erroringBucket) IsAccessDeniedErr(err error) bool { return b.bkt.IsAccessDeniedErr(err) }