From 10536c8577c77e2e5d9fe3e7017ef7458207e558 Mon Sep 17 00:00:00 2001 From: Pedro Tanaka Date: Thu, 25 Apr 2024 17:37:56 +0200 Subject: [PATCH 1/2] Query|Receiver|Store: Do not log full request on ProxyStore by default We had a problem on our production where a sudden increase in requests with long matchers was putting our receivers under a lot of pressure. Upon checking profiles we saw that the problem was calls to Log() Signed-off-by: Pedro Tanaka --- pkg/store/proxy.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index 666f130ece..50a508d783 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -283,7 +283,10 @@ func (s *ProxyStore) TSDBInfos() []infopb.TSDBInfo { func (s *ProxyStore) Series(originalRequest *storepb.SeriesRequest, srv storepb.Store_SeriesServer) error { // TODO(bwplotka): This should be part of request logger, otherwise it does not make much sense. Also, could be // tiggered by tracing span to reduce cognitive load. - reqLogger := log.With(s.logger, "component", "proxy", "request", originalRequest.String()) + reqLogger := log.With(s.logger, "component", "proxy") + if s.debugLogging { + reqLogger = log.With(reqLogger, "request", originalRequest.String()) + } match, matchers, err := matchesExternalLabels(originalRequest.Matchers, s.selectorLabels) if err != nil { From 43519f5001dbf9d24f05840f4fd27f9def76a9ce Mon Sep 17 00:00:00 2001 From: Pedro Tanaka Date: Thu, 25 Apr 2024 17:49:01 +0200 Subject: [PATCH 2/2] Adding changelog Signed-off-by: Pedro Tanaka --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28a369c548..bde408da39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#7123](https://github.com/thanos-io/thanos/pull/7123) Rule: Change default Alertmanager API version to v2. - [#7223](https://github.com/thanos-io/thanos/pull/7223) Automatic detection of memory limits and configure GOMEMLIMIT to match. - [#7283](https://github.com/thanos-io/thanos/pull/7283) Compact: *breaking :warning:* Replace group with resolution in compact downsample metrics to avoid cardinality explosion with large numbers of groups. +- [#7305](https://github.com/thanos-io/thanos/pull/7305) Query|Receiver: Do not log full request on ProxyStore by default. ### Removed