Add option to extend the length of OpenSearchJsonLayout logger (opens…

…earch-project#6569) * Extend the length of OpenSearchJsonLayout logger from 10 to 100k Signed-off-by: Boris Djurdjevic <[email protected]> * Add changelog entry for OpenSearchJsonLayout change Signed-off-by: Boris Djurdjevic <[email protected]> * Changelog: Add link to opensearch-project#6569 Signed-off-by: Boris Djurdjevic <[email protected]> * Adapt OpenSearchJsonLayoutTests for opensearch-project#6569 Signed-off-by: Boris Djurdjevic <[email protected]> * implement configurable truncation limit in OpenSearchJsonLayout logger Signed-off-by: Boris Djurdjevic <[email protected]> * changelog: Move opensearch-project#6569 PR from changes to additions Signed-off-by: Boris Djurdjevic <[email protected]> * move PR opensearch-project#6569 changelog entry to Unreleased 2.x Signed-off-by: Boris Djurdjevic <[email protected]> --------- Signed-off-by: Boris Djurdjevic <[email protected]>
mingshl · Mar 9, 2023 · 13d336c · 13d336c
1 parent 16535b9
commit 13d336c
Show file tree

Hide file tree

Showing 3 changed files with 72 additions and 7 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -76,6 +76,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add GeoTile and GeoHash Grid aggregations on GeoShapes. ([#5589](https://github.com/opensearch-project/OpenSearch/pull/5589))
 - Disallow multiple data paths for search nodes ([#6427](https://github.com/opensearch-project/OpenSearch/pull/6427))
 - [Segment Replication] Allocation and rebalancing based on average primary shard count per index ([#6422](https://github.com/opensearch-project/OpenSearch/pull/6422))
+- The truncation limit of the OpenSearchJsonLayout logger is now configurable ([#6569](https://github.com/opensearch-project/OpenSearch/pull/6569))
 - Add 'base_path' setting to File System Repository ([#6558](https://github.com/opensearch-project/OpenSearch/pull/6558))
 - Return success on DeletePits when no PITs exist. ([#6544](https://github.com/opensearch-project/OpenSearch/pull/6544))
 

diff --git a/qa/logging-config/src/test/java/org/opensearch/common/logging/OpenSearchJsonLayoutTests.java b/qa/logging-config/src/test/java/org/opensearch/common/logging/OpenSearchJsonLayoutTests.java
@@ -66,6 +66,46 @@ public void testLayout() {
                 "%exceptionAsJson }" + System.lineSeparator()));
     }
 
+    public void testWithMaxMessageLengthLayout() {
+        OpenSearchJsonLayout server = OpenSearchJsonLayout.newBuilder()
+            .setType("server")
+            .setMaxMessageLength(42)
+            .build();
+        String conversionPattern = server.getPatternLayout().getConversionPattern();
+
+        assertThat(conversionPattern, Matchers.equalTo(
+            "{" +
+                "\"type\": \"server\", " +
+                "\"timestamp\": \"%d{yyyy-MM-dd'T'HH:mm:ss,SSSZZ}\", " +
+                "\"level\": \"%p\", " +
+                "\"component\": \"%c{1.}\", " +
+                "\"cluster.name\": \"${sys:opensearch.logs.cluster_name}\", " +
+                "\"node.name\": \"%node_name\", " +
+                "\"message\": \"%notEmpty{%enc{%marker}{JSON} }%enc{%.-42m}{JSON}\"" +
+                "%notEmpty{, %node_and_cluster_id }" +
+                "%exceptionAsJson }" + System.lineSeparator()));
+    }
+
+    public void testWithUnrestrictedMaxMessageLengthLayout() {
+        OpenSearchJsonLayout server = OpenSearchJsonLayout.newBuilder()
+            .setType("server")
+            .setMaxMessageLength(0)
+            .build();
+        String conversionPattern = server.getPatternLayout().getConversionPattern();
+
+        assertThat(conversionPattern, Matchers.equalTo(
+            "{" +
+                "\"type\": \"server\", " +
+                "\"timestamp\": \"%d{yyyy-MM-dd'T'HH:mm:ss,SSSZZ}\", " +
+                "\"level\": \"%p\", " +
+                "\"component\": \"%c{1.}\", " +
+                "\"cluster.name\": \"${sys:opensearch.logs.cluster_name}\", " +
+                "\"node.name\": \"%node_name\", " +
+                "\"message\": \"%notEmpty{%enc{%marker}{JSON} }%enc{%m}{JSON}\"" +
+                "%notEmpty{, %node_and_cluster_id }" +
+                "%exceptionAsJson }" + System.lineSeparator()));
+    }
+
     public void testLayoutWithAdditionalFields() {
         OpenSearchJsonLayout server = OpenSearchJsonLayout.newBuilder()
                                           .setType("server")

diff --git a/server/src/main/java/org/opensearch/common/logging/OpenSearchJsonLayout.java b/server/src/main/java/org/opensearch/common/logging/OpenSearchJsonLayout.java
@@ -83,6 +83,9 @@
  * <p>
  * The value taken from %OpenSearchMessageField{message} has to be a simple escaped JSON value and is populated in subclasses of
  * <code>OpenSearchLogMessage</code>
+ * <p>
+ * The <code>message</code> field is truncated at 10000 characters by default. This limit can be controlled by setting the
+ * <code>appender.logger.layout.maxmessagelength</code> to the desired limit or to <code>0</code> to disable truncation.
  *
  * @opensearch.internal
  */
@@ -91,26 +94,34 @@ public class OpenSearchJsonLayout extends AbstractStringLayout {
 
     private final PatternLayout patternLayout;
 
-    protected OpenSearchJsonLayout(String typeName, Charset charset, String[] opensearchMessageFields) {
+    protected OpenSearchJsonLayout(String typeName, Charset charset, String[] opensearchMessageFields, int maxMessageLength) {
         super(charset);
         this.patternLayout = PatternLayout.newBuilder()
-            .withPattern(pattern(typeName, opensearchMessageFields))
+            .withPattern(pattern(typeName, opensearchMessageFields, maxMessageLength))
             .withAlwaysWriteExceptions(false)
             .build();
     }
 
-    private String pattern(String type, String[] opensearchMessageFields) {
+    private String pattern(String type, String[] opensearchMessageFields, int maxMessageLength) {
         if (Strings.isEmpty(type)) {
             throw new IllegalArgumentException("layout parameter 'type_name' cannot be empty");
         }
+
+        String messageFormat = "%m";
+        if (maxMessageLength < 0) {
+            throw new IllegalArgumentException("layout parameter 'maxmessagelength' cannot be a negative number");
+        } else if (maxMessageLength > 0) {
+            messageFormat = "%.-" + Integer.toString(maxMessageLength) + "m";
+        }
+
         Map<String, Object> map = new LinkedHashMap<>();
         map.put("type", inQuotes(type));
         map.put("timestamp", inQuotes("%d{yyyy-MM-dd'T'HH:mm:ss,SSSZZ}"));
         map.put("level", inQuotes("%p"));
         map.put("component", inQuotes("%c{1.}"));
         map.put("cluster.name", inQuotes("${sys:opensearch.logs.cluster_name}"));
         map.put("node.name", inQuotes("%node_name"));
-        map.put("message", inQuotes("%notEmpty{%enc{%marker}{JSON} }%enc{%.-10000m}{JSON}"));
+        map.put("message", inQuotes("%notEmpty{%enc{%marker}{JSON} }%enc{" + messageFormat + "}{JSON}"));
 
         for (String key : opensearchMessageFields) {
             map.put(key, inQuotes("%OpenSearchMessageField{" + key + "}"));
@@ -162,8 +173,8 @@ private String inQuotes(String s) {
     }
 
     @PluginFactory
-    public static OpenSearchJsonLayout createLayout(String type, Charset charset, String[] opensearchmessagefields) {
-        return new OpenSearchJsonLayout(type, charset, opensearchmessagefields);
+    public static OpenSearchJsonLayout createLayout(String type, Charset charset, String[] opensearchmessagefields, int maxMessageLength) {
+        return new OpenSearchJsonLayout(type, charset, opensearchmessagefields, maxMessageLength);
     }
 
     PatternLayout getPatternLayout() {
@@ -188,14 +199,18 @@ public static class Builder<B extends OpenSearchJsonLayout.Builder<B>> extends A
         @PluginAttribute("opensearchmessagefields")
         private String opensearchMessageFields;
 
+        @PluginAttribute(value = "maxmessagelength", defaultInt = 10000)
+        private int maxMessageLength;
+
         public Builder() {
             setCharset(StandardCharsets.UTF_8);
+            setMaxMessageLength(10000);
         }
 
         @Override
         public OpenSearchJsonLayout build() {
             String[] split = Strings.isNullOrEmpty(opensearchMessageFields) ? new String[] {} : opensearchMessageFields.split(",");
-            return OpenSearchJsonLayout.createLayout(type, charset, split);
+            return OpenSearchJsonLayout.createLayout(type, charset, split, maxMessageLength);
         }
 
         public Charset getCharset() {
@@ -224,6 +239,15 @@ public B setOpenSearchMessageFields(String opensearchMessageFields) {
             this.opensearchMessageFields = opensearchMessageFields;
             return asBuilder();
         }
+
+        public int getMaxMessageLength() {
+            return maxMessageLength;
+        }
+
+        public B setMaxMessageLength(final int maxMessageLength) {
+            this.maxMessageLength = maxMessageLength;
+            return asBuilder();
+        }
     }
 
     @PluginBuilderFactory