From 909a1f6f8610fba5bf4a447dc025ceed4942f4fb Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 19 Jan 2022 17:08:30 +0100 Subject: [PATCH] Do no use x-opaque-id for deduplicating kibana requests deprecated log messages originating from kibana's requests should not be deduplicated with the use of x-opaque-id because kibana populates it with UUID. closes #82810 --- .../common/logging/RateLimitingFilter.java | 14 ++++++++------ .../common/logging/RateLimitingFilterTests.java | 16 +++++++++++++++- .../xpack/deprecation/Deprecation.java | 4 ++-- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/logging/RateLimitingFilter.java b/server/src/main/java/org/elasticsearch/common/logging/RateLimitingFilter.java index cdcf629b40e95..a444047cb136a 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/RateLimitingFilter.java +++ b/server/src/main/java/org/elasticsearch/common/logging/RateLimitingFilter.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.common.logging.DeprecatedMessage.ELASTIC_ORIGIN_FIELD_NAME; import static org.elasticsearch.common.logging.DeprecatedMessage.KEY_FIELD_NAME; import static org.elasticsearch.common.logging.DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME; @@ -35,13 +36,13 @@ * This filter works by using a lruKeyCache - a set of keys which prevents a second message with the same key to be logged. * The lruKeyCache has a size limited to 128, which when breached will remove the oldest entries. * - * It is possible to disable use of `x-opaque-id` as a key with {@link RateLimitingFilter#setUseXOpaqueId(boolean) } + * It is possible to disable use of `x-opaque-id` as a key with {@link RateLimitingFilter#setUseXOpaqueIdEnabled(boolean) } * @see Log4j2 Filters */ @Plugin(name = "RateLimitingFilter", category = Node.CATEGORY, elementType = Filter.ELEMENT_TYPE) public class RateLimitingFilter extends AbstractFilter { - - private volatile boolean useXOpaqueId = true; + // a flag to disable/enable use of xOpaqueId controlled by changing cluster setting + private volatile boolean useXOpaqueIdEnabled = true; private final Set lruKeyCache = Collections.newSetFromMap(Collections.synchronizedMap(new LinkedHashMap<>() { @Override @@ -76,7 +77,8 @@ public Result filter(Message message) { private String getKey(ESLogMessage esLogMessage) { final String key = esLogMessage.get(KEY_FIELD_NAME); - if (useXOpaqueId) { + final String productOrigin = esLogMessage.get(ELASTIC_ORIGIN_FIELD_NAME); + if (useXOpaqueIdEnabled && productOrigin.equals("kibana") == false) { String xOpaqueId = esLogMessage.get(X_OPAQUE_ID_FIELD_NAME); return xOpaqueId + key; } @@ -101,7 +103,7 @@ public static RateLimitingFilter createFilter( return new RateLimitingFilter(match, mismatch); } - public void setUseXOpaqueId(boolean useXOpaqueId) { - this.useXOpaqueId = useXOpaqueId; + public void setUseXOpaqueIdEnabled(boolean useXOpaqueIdEnabled) { + this.useXOpaqueIdEnabled = useXOpaqueIdEnabled; } } diff --git a/server/src/test/java/org/elasticsearch/common/logging/RateLimitingFilterTests.java b/server/src/test/java/org/elasticsearch/common/logging/RateLimitingFilterTests.java index 61e6b3453ea1b..f6748e7e42cf6 100644 --- a/server/src/test/java/org/elasticsearch/common/logging/RateLimitingFilterTests.java +++ b/server/src/test/java/org/elasticsearch/common/logging/RateLimitingFilterTests.java @@ -150,7 +150,7 @@ public void testFilterCanBeReset() { public void testMessagesXOpaqueIsIgnoredWhenDisabled() { RateLimitingFilter filter = new RateLimitingFilter(); - filter.setUseXOpaqueId(false); + filter.setUseXOpaqueIdEnabled(false); filter.start(); // Should NOT be rate-limited because it's not in the cache @@ -169,4 +169,18 @@ public void testMessagesXOpaqueIsIgnoredWhenDisabled() { message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 1", "opaque-id 1", "productName", "msg 0"); assertThat(filter.filter(message), equalTo(Result.ACCEPT)); } + + public void testXOpaqueIdNotBeingUsedFromKibanaOriginatingRequests() { + RateLimitingFilter filter = new RateLimitingFilter(); + filter.setUseXOpaqueIdEnabled(true); + filter.start(); + + // Should NOT be rate-limited because it's not in the cache + Message message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 0", "opaque-id 0", "kibana", "msg 0"); + assertThat(filter.filter(message), equalTo(Result.ACCEPT)); + + // Should NOT be rate-limited even though the x-opaque-id is unique because it originates from kibana + message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 0", "opaque-id 1", "kibana", "msg 0"); + assertThat(filter.filter(message), equalTo(Result.ACCEPT)); + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java index b4331748d197a..e523077b140bf 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java @@ -109,9 +109,9 @@ public Collection createComponents( final RateLimitingFilter rateLimitingFilterForIndexing = new RateLimitingFilter(); // enable on start. - rateLimitingFilterForIndexing.setUseXOpaqueId(USE_X_OPAQUE_ID_IN_FILTERING.get(environment.settings())); + rateLimitingFilterForIndexing.setUseXOpaqueIdEnabled(USE_X_OPAQUE_ID_IN_FILTERING.get(environment.settings())); clusterService.getClusterSettings() - .addSettingsUpdateConsumer(USE_X_OPAQUE_ID_IN_FILTERING, rateLimitingFilterForIndexing::setUseXOpaqueId); + .addSettingsUpdateConsumer(USE_X_OPAQUE_ID_IN_FILTERING, rateLimitingFilterForIndexing::setUseXOpaqueIdEnabled); final DeprecationIndexingComponent component = DeprecationIndexingComponent.createDeprecationIndexingComponent( client,