Change deprecation logs data stream name (#68737)

More fixes to deprecation log indexing so that the data stream name and document contents are more ECS-compatible.
elastic · Apr 13, 2021 · fb1921c · fb1921c
1 parent d3d7220
commit fb1921c
Show file tree

Hide file tree

Showing 12 changed files with 65 additions and 56 deletions.
diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties
@@ -62,7 +62,8 @@ appender.deprecation_rolling.type = RollingFile
 appender.deprecation_rolling.name = deprecation_rolling
 appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json
 appender.deprecation_rolling.layout.type = ECSJsonLayout
-appender.deprecation_rolling.layout.dataset = elasticsearch.deprecation
+# Intentionally follows a different pattern to above
+appender.deprecation_rolling.layout.dataset = deprecation.elasticsearch
 appender.deprecation_rolling.filter.rate_limit.type = RateLimitingFilter
 
 appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.json.gz

diff --git a/...ransport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java b/...ransport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java
@@ -118,7 +118,7 @@ public void testTemplateExists() throws IOException {
                 request.setOptions(expectWarnings(
                     "legacy template [template] has index patterns [*] matching patterns from existing composable templates " +
                     "[.deprecation-indexing-template,.slm-history,.watch-history-14,ilm-history,logs," +
-                    "metrics,synthetics] with patterns (.deprecation-indexing-template => [.logs-deprecation-elasticsearch]," +
+                    "metrics,synthetics] with patterns (.deprecation-indexing-template => [.logs-deprecation.elasticsearch-default]," +
                     ".slm-history => [.slm-history-5*]," +
                     ".watch-history-14 => [.watcher-history-14*],ilm-history => [ilm-history-5*]," +
                     "logs => [logs-*-*],metrics => [metrics-*-*],synthetics => [synthetics-*-*]" +

diff --git a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java
@@ -92,14 +92,15 @@ public void testDeprecatedMessageWithoutXOpaqueId() throws IOException {
 
             assertThat(jsonLogs, contains(
                 allOf(
-                    hasEntry("event.dataset", "elasticsearch.deprecation"),
+                    hasEntry("event.dataset", "deprecation.elasticsearch"),
                     hasEntry("log.level", "DEPRECATION"),
                     hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
                     hasEntry("elasticsearch.cluster.name", "elasticsearch"),
                     hasEntry("elasticsearch.node.name", "sample-name"),
                     hasEntry("message", "deprecated message1"),
                     hasEntry("data_stream.type", "logs"),
-                    hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
+                    hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
+                    hasEntry("data_stream.namespace", "default"),
                     hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
                     hasEntry(DeprecatedMessage.KEY_FIELD_NAME, "a key"),
                     not(hasKey(DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME)),
@@ -131,8 +132,9 @@ public void testCompatibleLog() throws Exception {
                     contains(
                         allOf(
                             hasEntry("log.level", "DEPRECATION"),
-                            hasEntry("event.dataset", "elasticsearch.deprecation"),
-                            hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
+                            hasEntry("event.dataset", "deprecation.elasticsearch"),
+                            hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
+                            hasEntry("data_stream.namespace", "default"),
                             hasEntry("data_stream.type", "logs"),
                             hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
                             hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
@@ -146,8 +148,9 @@ public void testCompatibleLog() throws Exception {
                         allOf(
                             hasEntry("log.level", "DEPRECATION"),
                             // event.dataset and data_stream.dataset have to be the same across the data stream
-                            hasEntry("event.dataset", "elasticsearch.deprecation"),
-                            hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
+                            hasEntry("event.dataset", "deprecation.elasticsearch"),
+                            hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
+                            hasEntry("data_stream.namespace", "default"),
                             hasEntry("data_stream.type", "logs"),
                             hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
                             hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
@@ -165,6 +168,7 @@ public void testCompatibleLog() throws Exception {
             assertWarnings("deprecated message1", "compatible API message");
         });
     }
+
     public void testParseFieldEmittingDeprecatedLogs() throws Exception {
         withThreadContext(threadContext -> {
             threadContext.putHeader(Task.X_OPAQUE_ID, "someId");
@@ -193,8 +197,9 @@ public void testParseFieldEmittingDeprecatedLogs() throws Exception {
                         // deprecation log for field deprecated_name
                         allOf(
                             hasEntry("log.level", "DEPRECATION"),
-                            hasEntry("event.dataset", "elasticsearch.deprecation"),
-                            hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
+                            hasEntry("event.dataset", "deprecation.elasticsearch"),
+                            hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
+                            hasEntry("data_stream.namespace", "default"),
                             hasEntry("data_stream.type", "logs"),
                             hasEntry("log.logger", "org.elasticsearch.deprecation.common.ParseField"),
                             hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
@@ -208,8 +213,9 @@ public void testParseFieldEmittingDeprecatedLogs() throws Exception {
                         // deprecation log for field deprecated_name2 (note it is not being throttled)
                         allOf(
                             hasEntry("log.level", "DEPRECATION"),
-                            hasEntry("event.dataset", "elasticsearch.deprecation"),
-                            hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
+                            hasEntry("event.dataset", "deprecation.elasticsearch"),
+                            hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
+                            hasEntry("data_stream.namespace", "default"),
                             hasEntry("data_stream.type", "logs"),
                             hasEntry("log.logger", "org.elasticsearch.deprecation.common.ParseField"),
                             hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
@@ -220,11 +226,12 @@ public void testParseFieldEmittingDeprecatedLogs() throws Exception {
                             hasEntry(DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME, "someId"),
                             hasEntry("elasticsearch.event.category", "api")
                         ),
-                        //compatible  log line
+                        // compatible log line
                         allOf(
                             hasEntry("log.level", "DEPRECATION"),
-                            hasEntry("event.dataset", "elasticsearch.deprecation"),
-                            hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
+                            hasEntry("event.dataset", "deprecation.elasticsearch"),
+                            hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
+                            hasEntry("data_stream.namespace", "default"),
                             hasEntry("data_stream.type", "logs"),
                             hasEntry("log.logger", "org.elasticsearch.deprecation.common.ParseField"),
                             hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
@@ -263,14 +270,15 @@ public void testDeprecatedMessage() throws Exception {
                     jsonLogs,
                     contains(
                         allOf(
-                            hasEntry("event.dataset", "elasticsearch.deprecation"),
+                            hasEntry("event.dataset", "deprecation.elasticsearch"),
                             hasEntry("log.level", "DEPRECATION"),
                             hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
                             hasEntry("elasticsearch.cluster.name", "elasticsearch"),
                             hasEntry("elasticsearch.node.name", "sample-name"),
                             hasEntry("message", "deprecated message1"),
                             hasEntry("data_stream.type", "logs"),
-                            hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
+                            hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
+                            hasEntry("data_stream.namespace", "default"),
                             hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
                             hasEntry(DeprecatedMessage.KEY_FIELD_NAME, "someKey"),
                             hasEntry(DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME, "someId"),
@@ -422,7 +430,6 @@ public void testStacktrace() throws IOException {
         }
     }
 
-
     public void testJsonInStacktraceMessageIsNotSplitted() throws IOException {
         final Logger testLogger = LogManager.getLogger("test");
 
@@ -454,7 +461,6 @@ public void testJsonInStacktraceMessageIsNotSplitted() throws IOException {
         }
     }
 
-
     public void testDuplicateLogMessages() throws Exception {
         final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger("org.elasticsearch.test");
 
@@ -473,7 +479,7 @@ public void testDuplicateLogMessages() throws Exception {
 
                 assertThat(jsonLogs, contains(
                     allOf(
-                        hasEntry("event.dataset", "elasticsearch.deprecation"),
+                        hasEntry("event.dataset", "deprecation.elasticsearch"),
                         hasEntry("log.level", "DEPRECATION"),
                         hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
                         hasEntry("elasticsearch.cluster.name", "elasticsearch"),
@@ -505,7 +511,7 @@ public void testDuplicateLogMessages() throws Exception {
                     jsonLogs,
                     contains(
                         allOf(
-                            hasEntry("event.dataset", "elasticsearch.deprecation"),
+                            hasEntry("event.dataset", "deprecation.elasticsearch"),
                             hasEntry("log.level", "DEPRECATION"),
                             hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
                             hasEntry("elasticsearch.cluster.name", "elasticsearch"),
@@ -515,7 +521,7 @@ public void testDuplicateLogMessages() throws Exception {
                             hasEntry("elasticsearch.event.category", "other")
                         ),
                         allOf(
-                            hasEntry("event.dataset", "elasticsearch.deprecation"),
+                            hasEntry("event.dataset", "deprecation.elasticsearch"),
                             hasEntry("log.level", "DEPRECATION"),
                             hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
                             hasEntry("elasticsearch.cluster.name", "elasticsearch"),
@@ -555,7 +561,6 @@ private void setupLogging(final String config, final Settings settings) throws I
         LogConfigurator.configure(environment);
     }
 
-
     private Matcher<JsonLogLine> logLine(String type, Level level, String nodeName, String component, String message) {
         return logLine(mapOfParamsToCheck(type, level, nodeName, component, message));
     }

diff --git a/...-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties b/...-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties
@@ -13,14 +13,16 @@ appender.file.layout.dataset = elasticsearch.file
 appender.deprecated.type = File
 appender.deprecated.name = deprecated
 appender.deprecated.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecated.json
+# Intentionally follows a different pattern to above
 appender.deprecated.layout.type = ECSJsonLayout
-appender.deprecated.layout.dataset = elasticsearch.deprecation
+appender.deprecated.layout.dataset = deprecation.elasticsearch
 appender.deprecated.filter.rate_limit.type = RateLimitingFilter
 
 appender.deprecatedconsole.type = Console
 appender.deprecatedconsole.name = deprecatedconsole
 appender.deprecatedconsole.layout.type = ECSJsonLayout
-appender.deprecatedconsole.layout.dataset = elasticsearch.deprecation
+# Intentionally follows a different pattern to above
+appender.deprecatedconsole.layout.dataset = deprecation.elasticsearch
 appender.deprecatedconsole.filter.rate_limit.type = RateLimitingFilter
 
 

diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java
@@ -43,7 +43,7 @@ private static ESLogMessage getEsLogMessage(
         String messagePattern,
         Object[] args) {
         ESLogMessage esLogMessage = new ESLogMessage(messagePattern, args)
-            .field("data_stream.dataset", "elasticsearch.deprecation")
+            .field("data_stream.dataset", "deprecation.elasticsearch")
             .field("data_stream.type", "logs")
             .field("data_stream.namespace", "default")
             .field("ecs.version", ECS_VERSION)

diff --git a/...c/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json b/...c/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json
@@ -3,8 +3,8 @@
     "hot": {
       "actions": {
         "rollover": {
-          "max_primary_shard_size": "50gb",
-          "max_age": "30d"
+          "max_primary_shard_size": "10gb",
+          "max_age": "14d"
         }
       }
     }

diff --git a/...src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json b/...src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json
@@ -53,7 +53,7 @@
     }
   },
   "_meta": {
-    "description": "default mappings for ES deprecation logs index template installed by x-pack",
+    "description": "default mappings for Stack deprecation logs index template installed by x-pack",
     "managed": true
   },
   "version": ${xpack.deprecation.indexing.template.version}

diff --git a/...src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json b/...src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json
@@ -13,7 +13,7 @@
     }
   },
   "_meta": {
-    "description": "default settings for ES deprecation logs index template installed by x-pack",
+    "description": "default settings for Stack deprecation logs index template installed by x-pack",
     "managed": true
   },
   "version": ${xpack.deprecation.indexing.template.version}

diff --git a/...src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json b/...src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json
@@ -1,5 +1,5 @@
 {
-  "index_patterns": [".logs-deprecation-elasticsearch"],
+  "index_patterns": [".logs-deprecation.*"],
   "priority": 1000,
   "data_stream": {
     "hidden": true
@@ -10,7 +10,7 @@
   ],
   "allow_auto_create": true,
   "_meta": {
-    "description": "default template for ES deprecation logs index template installed by x-pack",
+    "description": "default template for Stack deprecation logs index template installed by x-pack",
     "managed": true
   },
   "version": ${xpack.deprecation.indexing.template.version}