Do no use x-opaque-id for deduplicating kibana requests

deprecated log messages originating from kibana's requests should not be deduplicated with the use of x-opaque-id because kibana populates it with UUID. closes elastic#82810
pgomulka · Jan 19, 2022 · 909a1f6 · 909a1f6
1 parent 05ddae8
commit 909a1f6
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 9 deletions.
diff --git a/server/src/main/java/org/elasticsearch/common/logging/RateLimitingFilter.java b/server/src/main/java/org/elasticsearch/common/logging/RateLimitingFilter.java
@@ -25,6 +25,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.common.logging.DeprecatedMessage.ELASTIC_ORIGIN_FIELD_NAME;
 import static org.elasticsearch.common.logging.DeprecatedMessage.KEY_FIELD_NAME;
 import static org.elasticsearch.common.logging.DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME;
 
@@ -35,13 +36,13 @@
  * This filter works by using a lruKeyCache - a set of keys which prevents a second message with the same key to be logged.
  * The lruKeyCache has a size limited to 128, which when breached will remove the oldest entries.
  *
- * It is possible to disable use of `x-opaque-id` as a key with {@link RateLimitingFilter#setUseXOpaqueId(boolean) }
+ * It is possible to disable use of `x-opaque-id` as a key with {@link RateLimitingFilter#setUseXOpaqueIdEnabled(boolean) }
  * @see <a href="https://logging.apache.org/log4j/2.x/manual/filters.htmlf">Log4j2 Filters</a>
  */
 @Plugin(name = "RateLimitingFilter", category = Node.CATEGORY, elementType = Filter.ELEMENT_TYPE)
 public class RateLimitingFilter extends AbstractFilter {
-
-    private volatile boolean useXOpaqueId = true;
+    // a flag to disable/enable use of xOpaqueId controlled by changing cluster setting
+    private volatile boolean useXOpaqueIdEnabled = true;
 
     private final Set<String> lruKeyCache = Collections.newSetFromMap(Collections.synchronizedMap(new LinkedHashMap<>() {
         @Override
@@ -76,7 +77,8 @@ public Result filter(Message message) {
 
     private String getKey(ESLogMessage esLogMessage) {
         final String key = esLogMessage.get(KEY_FIELD_NAME);
-        if (useXOpaqueId) {
+        final String productOrigin = esLogMessage.get(ELASTIC_ORIGIN_FIELD_NAME);
+        if (useXOpaqueIdEnabled && productOrigin.equals("kibana") == false) {
             String xOpaqueId = esLogMessage.get(X_OPAQUE_ID_FIELD_NAME);
             return xOpaqueId + key;
         }
@@ -101,7 +103,7 @@ public static RateLimitingFilter createFilter(
         return new RateLimitingFilter(match, mismatch);
     }
 
-    public void setUseXOpaqueId(boolean useXOpaqueId) {
-        this.useXOpaqueId = useXOpaqueId;
+    public void setUseXOpaqueIdEnabled(boolean useXOpaqueIdEnabled) {
+        this.useXOpaqueIdEnabled = useXOpaqueIdEnabled;
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/logging/RateLimitingFilterTests.java b/server/src/test/java/org/elasticsearch/common/logging/RateLimitingFilterTests.java
@@ -150,7 +150,7 @@ public void testFilterCanBeReset() {
 
     public void testMessagesXOpaqueIsIgnoredWhenDisabled() {
         RateLimitingFilter filter = new RateLimitingFilter();
-        filter.setUseXOpaqueId(false);
+        filter.setUseXOpaqueIdEnabled(false);
         filter.start();
 
         // Should NOT be rate-limited because it's not in the cache
@@ -169,4 +169,18 @@ public void testMessagesXOpaqueIsIgnoredWhenDisabled() {
         message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 1", "opaque-id 1", "productName", "msg 0");
         assertThat(filter.filter(message), equalTo(Result.ACCEPT));
     }
+
+    public void testXOpaqueIdNotBeingUsedFromKibanaOriginatingRequests() {
+        RateLimitingFilter filter = new RateLimitingFilter();
+        filter.setUseXOpaqueIdEnabled(true);
+        filter.start();
+
+        // Should NOT be rate-limited because it's not in the cache
+        Message message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 0", "opaque-id 0", "kibana", "msg 0");
+        assertThat(filter.filter(message), equalTo(Result.ACCEPT));
+
+        // Should NOT be rate-limited even though the x-opaque-id is unique because it originates from kibana
+        message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 0", "opaque-id 1", "kibana", "msg 0");
+        assertThat(filter.filter(message), equalTo(Result.ACCEPT));
+    }
 }
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java
@@ -109,9 +109,9 @@ public Collection<Object> createComponents(
 
         final RateLimitingFilter rateLimitingFilterForIndexing = new RateLimitingFilter();
         // enable on start.
-        rateLimitingFilterForIndexing.setUseXOpaqueId(USE_X_OPAQUE_ID_IN_FILTERING.get(environment.settings()));
+        rateLimitingFilterForIndexing.setUseXOpaqueIdEnabled(USE_X_OPAQUE_ID_IN_FILTERING.get(environment.settings()));
         clusterService.getClusterSettings()
-            .addSettingsUpdateConsumer(USE_X_OPAQUE_ID_IN_FILTERING, rateLimitingFilterForIndexing::setUseXOpaqueId);
+            .addSettingsUpdateConsumer(USE_X_OPAQUE_ID_IN_FILTERING, rateLimitingFilterForIndexing::setUseXOpaqueIdEnabled);
 
         final DeprecationIndexingComponent component = DeprecationIndexingComponent.createDeprecationIndexingComponent(
             client,