From 349db6f0def032561b4c738ef03bf4633c37e640 Mon Sep 17 00:00:00 2001 From: Pedro Tanaka Date: Fri, 26 Apr 2024 07:50:28 +0200 Subject: [PATCH] Query|Receiver: Do not log full request on ProxyStore by default (#7305) * Query|Receiver|Store: Do not log full request on ProxyStore by default We had a problem on our production where a sudden increase in requests with long matchers was putting our receivers under a lot of pressure. Upon checking profiles we saw that the problem was calls to Log() Signed-off-by: Pedro Tanaka * Adding changelog Signed-off-by: Pedro Tanaka --------- Signed-off-by: Pedro Tanaka Signed-off-by: mluffman --- CHANGELOG.md | 1 + pkg/store/proxy.go | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28a369c548..bde408da39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#7123](https://github.com/thanos-io/thanos/pull/7123) Rule: Change default Alertmanager API version to v2. - [#7223](https://github.com/thanos-io/thanos/pull/7223) Automatic detection of memory limits and configure GOMEMLIMIT to match. - [#7283](https://github.com/thanos-io/thanos/pull/7283) Compact: *breaking :warning:* Replace group with resolution in compact downsample metrics to avoid cardinality explosion with large numbers of groups. +- [#7305](https://github.com/thanos-io/thanos/pull/7305) Query|Receiver: Do not log full request on ProxyStore by default. ### Removed diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index 666f130ece..50a508d783 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -283,7 +283,10 @@ func (s *ProxyStore) TSDBInfos() []infopb.TSDBInfo { func (s *ProxyStore) Series(originalRequest *storepb.SeriesRequest, srv storepb.Store_SeriesServer) error { // TODO(bwplotka): This should be part of request logger, otherwise it does not make much sense. Also, could be // tiggered by tracing span to reduce cognitive load. - reqLogger := log.With(s.logger, "component", "proxy", "request", originalRequest.String()) + reqLogger := log.With(s.logger, "component", "proxy") + if s.debugLogging { + reqLogger = log.With(reqLogger, "request", originalRequest.String()) + } match, matchers, err := matchesExternalLabels(originalRequest.Matchers, s.selectorLabels) if err != nil {