From 06393d6b078d01300fd8df18ca4e5ee1e4ce2e30 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Mon, 16 Sep 2019 18:03:45 +0200 Subject: [PATCH] Serialize stack traces in error.stack_trace depends on elastic/ecs#562 --- README.md | 8 +- .../co/elastic/logging/EcsJsonSerializer.java | 163 ++++++++++++++++-- .../logging/AbstractEcsLoggingTest.java | 14 +- .../logging/EcsJsonSerializerTest.java | 89 ++++++++++ .../co/elastic/logging/log4j/EcsLayout.java | 13 +- .../logging/log4j/Log4jEcsLayoutTest.java | 1 + log4j2-ecs-layout/README.md | 26 ++- .../co/elastic/logging/log4j2/EcsLayout.java | 47 ++--- .../logging/log4j2/Log4j2EcsLayoutTest.java | 1 + .../src/test/resources/log4j2-test.xml | 2 +- logback-ecs-encoder/README.md | 12 +- .../elastic/logging/logback/EcsEncoder.java | 27 +-- .../logging/logback/EcsEncoderTest.java | 1 + .../src/test/resources/logback-config.xml | 1 + 14 files changed, 328 insertions(+), 77 deletions(-) create mode 100644 ecs-logging-core/src/test/java/co/elastic/logging/EcsJsonSerializerTest.java diff --git a/README.md b/README.md index 59ec85df..d1c8927e 100644 --- a/README.md +++ b/README.md @@ -74,10 +74,12 @@ We recommend using this library to log into a JSON log file and let Filebeat sen |ECS field | Log4j2 API | |----------|-------------| |[`@timestamp`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html) | [`LogEvent#getTimeMillis()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getTimeMillis()) | -| [`log.level`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) | [`LogEvent#getLevel()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getLevel()) | -|[`log.logger`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html)|[`LogEvent#getLoggerName(`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getLoggerName())| +|[`log.level`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) | [`LogEvent#getLevel()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getLevel()) | +|[`log.logger`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html)|[`LogEvent#getLoggerName()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getLoggerName())| |[`message`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html)|[`LogEvent#getMessage()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getMessage())| -|[`message`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html)|[`LogEvent#getThrown()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getThrown())| +|[`error.code`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html)|[`Throwable#getClass()`](https://docs.oracle.com/javase/7/docs/api/java/lang/Object.html#getClass())| +|[`error.message`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html)|[`Throwable#getStackTrace()`](https://docs.oracle.com/javase/7/docs/api/java/lang/Throwable.html#getMessage())| +|[`error.stack_trace`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html)|[`Throwable#getStackTrace()`](https://docs.oracle.com/javase/7/docs/api/java/lang/Throwable.html#getStackTrace())| |[`process.thread.name`](https://www.elastic.co/guide/en/ecs/current/ecs-process.html)|[`LogEvent#getThreadName()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getThreadName()) | |[`labels`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html)|[`LogEvent#getContextMap()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextMap())| |[`tags`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html)|[`LogEvent#getContextStack()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextStack())| diff --git a/ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java b/ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java index cbfa1128..9372099a 100644 --- a/ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java +++ b/ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java @@ -25,7 +25,7 @@ package co.elastic.logging; import java.io.PrintWriter; -import java.io.StringWriter; +import java.io.Writer; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -35,6 +35,8 @@ public class EcsJsonSerializer { public static final List DEFAULT_TOP_LEVEL_LABELS = Arrays.asList("trace.id", "transaction.id", "span.id"); private static final TimestampSerializer TIMESTAMP_SERIALIZER = new TimestampSerializer(); + private static final ThreadLocal messageStringBuilder = new ThreadLocal(); + private static final String NEW_LINE = System.getProperty("line.separator"); public static CharSequence toNullSafeString(final CharSequence s) { return s == null ? "" : s; @@ -48,8 +50,7 @@ public static void serializeObjectStart(StringBuilder builder, long timeMillis) } public static void serializeObjectEnd(StringBuilder builder) { - // last char is always a comma (,) - builder.setLength(builder.length() - 1); + removeIfEndsWith(builder, ","); builder.append('}'); builder.append('\n'); } @@ -68,13 +69,9 @@ public static void serializeThreadName(StringBuilder builder, String threadName) } } - public static void serializeFormattedMessage(StringBuilder builder, String message, Throwable t) { + public static void serializeFormattedMessage(StringBuilder builder, String message) { builder.append("\"message\":\""); JsonUtils.quoteAsString(message, builder); - if (t != null) { - builder.append("\\n"); - JsonUtils.quoteAsString(formatThrowable(t), builder); - } builder.append("\", "); } @@ -134,22 +131,162 @@ public static void serializeLabels(StringBuilder builder, Map labels, } } - public static void serializeException(StringBuilder builder, Throwable thrown) { + public static void serializeException(StringBuilder builder, Throwable thrown, boolean stackTraceAsArray) { if (thrown != null) { builder.append("\"error.code\":\""); JsonUtils.quoteAsString(thrown.getClass().getName(), builder); builder.append("\","); builder.append("\"error.message\":\""); - JsonUtils.quoteAsString(formatThrowable(thrown), builder); + JsonUtils.quoteAsString(thrown.getMessage(), builder); builder.append("\","); + if (stackTraceAsArray) { + builder.append("\"error.stack_trace\":[").append(NEW_LINE); + formatThrowableAsArray(builder, thrown); + builder.append(NEW_LINE).append("]"); + } else { + builder.append("\"error.stack_trace\":\""); + JsonUtils.quoteAsString(formatThrowable(thrown), builder); + builder.append("\""); + } } } + public static void serializeException(StringBuilder builder, String exceptionClassName, String exceptionMessage, String stackTrace, boolean stackTraceAsArray) { + builder.append("\"error.code\":\""); + JsonUtils.quoteAsString(exceptionClassName, builder); + builder.append("\","); + builder.append("\"error.message\":\""); + JsonUtils.quoteAsString(exceptionMessage, builder); + builder.append("\","); + if (stackTraceAsArray) { + builder.append("\"error.stack_trace\":[").append(NEW_LINE); + for (String line : stackTrace.split("\\n")) { + appendQuoted(builder, line); + } + builder.append(NEW_LINE).append("]"); + } else { + builder.append("\"error.stack_trace\":\""); + JsonUtils.quoteAsString(stackTrace, builder); + builder.append("\""); + } + } + + private static void appendQuoted(StringBuilder builder, CharSequence content) { + builder.append('"'); + JsonUtils.quoteAsString(content, builder); + builder.append('"'); + } + private static CharSequence formatThrowable(final Throwable throwable) { - StringWriter sw = new StringWriter(2048); - final PrintWriter pw = new PrintWriter(sw); + StringBuilder buffer = getMessageStringBuilder(); + final PrintWriter pw = new PrintWriter(new StringBuilderWriter(buffer)); throwable.printStackTrace(pw); pw.flush(); - return sw.toString(); + return buffer; + } + + private static void formatThrowableAsArray(final StringBuilder jsonBuilder, final Throwable throwable) { + final StringBuilder buffer = getMessageStringBuilder(); + final PrintWriter pw = new PrintWriter(new StringBuilderWriter(buffer), true) { + @Override + public void println() { + flush(); + jsonBuilder.append("\t\""); + JsonUtils.quoteAsString(buffer, jsonBuilder); + jsonBuilder.append("\","); + jsonBuilder.append(NEW_LINE); + buffer.setLength(0); + } + }; + throwable.printStackTrace(pw); + removeIfEndsWith(jsonBuilder, NEW_LINE); + removeIfEndsWith(jsonBuilder, ","); + } + + public static void removeIfEndsWith(StringBuilder sb, String ending) { + if (endsWith(sb, ending)) { + sb.setLength(sb.length() - ending.length()); + } + } + + public static boolean endsWith(StringBuilder sb, String ending) { + int endingLength = ending.length(); + int startIndex = sb.length() - endingLength; + if (startIndex < 0) { + return false; + } + for (int i = 0; i < endingLength; i++) { + if (sb.charAt(startIndex + i) != ending.charAt(i)) { + return false; + } + } + return true; + } + + public static StringBuilder getMessageStringBuilder() { + StringBuilder result = messageStringBuilder.get(); + if (result == null) { + result = new StringBuilder(1024); + messageStringBuilder.set(result); + } + result.setLength(0); + return result; + } + + private static class StringBuilderWriter extends Writer { + + private final StringBuilder buffer; + + StringBuilderWriter(StringBuilder buffer) { + this.buffer = buffer; + } + + @Override + public Writer append(CharSequence csq) { + buffer.append(csq); + return this; + } + + @Override + public void write(String str) { + buffer.append(str); + } + + @Override + public void write(String str, int off, int len) { + buffer.append(str, off, len); + } + + @Override + public Writer append(CharSequence csq, int start, int end) { + buffer.append(csq, start, end); + return this; + } + + @Override + public Writer append(char c) { + buffer.append(c); + return this; + } + + @Override + public void write(int c) { + buffer.append((char) c); + } + + @Override + public void write(char[] cbuf, int off, int len) { + buffer.append(cbuf, off, len); + } + + @Override + public void flush() { + + } + + @Override + public void close() { + + } } } diff --git a/ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java b/ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java index 142ef5aa..10cf08e9 100644 --- a/ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java +++ b/ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java @@ -27,16 +27,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.assertj.core.data.TemporalOffset; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.within; @@ -88,7 +85,12 @@ void testTopLevelLabels() throws Exception { void testLogException() throws Exception { error("test", new RuntimeException("test")); assertThat(getLastLogLine().get("log.level").textValue()).isEqualTo("ERROR"); - assertThat(getLastLogLine().get("message").textValue()).contains("at co.elastic.logging.AbstractEcsLoggingTest.testLogException"); + assertThat(getLastLogLine().get("error.message").textValue()).isEqualTo("test"); + assertThat(getLastLogLine().get("error.code").textValue()).isEqualTo(RuntimeException.class.getName()); + String stackTrace = StreamSupport.stream(getLastLogLine().get("error.stack_trace").spliterator(), false) + .map(JsonNode::textValue) + .collect(Collectors.joining("\n", "", "\n")); + assertThat(stackTrace).contains("at co.elastic.logging.AbstractEcsLoggingTest.testLogException"); } public abstract void putMdc(String key, String value); diff --git a/ecs-logging-core/src/test/java/co/elastic/logging/EcsJsonSerializerTest.java b/ecs-logging-core/src/test/java/co/elastic/logging/EcsJsonSerializerTest.java new file mode 100644 index 00000000..48e20f0f --- /dev/null +++ b/ecs-logging-core/src/test/java/co/elastic/logging/EcsJsonSerializerTest.java @@ -0,0 +1,89 @@ +/*- + * #%L + * Java ECS logging + * %% + * Copyright (C) 2019 Elastic and contributors + * %% + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * #L% + */ +package co.elastic.logging; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static org.assertj.core.api.Assertions.assertThat; + +class EcsJsonSerializerTest { + + @Test + void serializeExceptionAsString() throws IOException { + Exception exception = new Exception("foo"); + StringBuilder jsonBuilder = new StringBuilder(); + jsonBuilder.append('{'); + EcsJsonSerializer.serializeException(jsonBuilder, exception, false); + jsonBuilder.append('}'); + JsonNode jsonNode = new ObjectMapper().readTree(jsonBuilder.toString()); + + assertThat(jsonNode.get("error.code").textValue()).isEqualTo(exception.getClass().getName()); + assertThat(jsonNode.get("error.message").textValue()).isEqualTo("foo"); + StringWriter stringWriter = new StringWriter(); + exception.printStackTrace(new PrintWriter(stringWriter)); + assertThat(jsonNode.get("error.stack_trace").textValue()).isEqualTo(stringWriter.toString()); + } + + @Test + void serializeExceptionAsArray() throws IOException { + Exception exception = new Exception("foo"); + StringBuilder jsonBuilder = new StringBuilder(); + jsonBuilder.append('{'); + EcsJsonSerializer.serializeException(jsonBuilder, exception, true); + jsonBuilder.append('}'); + System.out.println(jsonBuilder); + JsonNode jsonNode = new ObjectMapper().readTree(jsonBuilder.toString()); + + assertThat(jsonNode.get("error.code").textValue()).isEqualTo(exception.getClass().getName()); + assertThat(jsonNode.get("error.message").textValue()).isEqualTo("foo"); + StringWriter stringWriter = new StringWriter(); + exception.printStackTrace(new PrintWriter(stringWriter)); + assertThat(StreamSupport.stream(jsonNode.get("error.stack_trace").spliterator(), false) + .map(JsonNode::textValue) + .collect(Collectors.joining("\n", "", "\n"))) + .isEqualTo(stringWriter.toString()); + } + + @Test + void testRemoveIfEndsWith() { + assertRemoveIfEndsWith("", "foo", ""); + assertRemoveIfEndsWith("foobar", "foo", "foobar"); + assertRemoveIfEndsWith("barfoo", "foo", "bar"); + } + + private void assertRemoveIfEndsWith(String builder, String ending, String expected) { + StringBuilder sb = new StringBuilder(builder); + EcsJsonSerializer.removeIfEndsWith(sb, ending); + assertThat(sb.toString()).isEqualTo(expected); + } +} diff --git a/log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java b/log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java index 3ff9e0fc..7751fa34 100644 --- a/log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java +++ b/log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java @@ -27,12 +27,14 @@ import co.elastic.logging.EcsJsonSerializer; import org.apache.log4j.Layout; import org.apache.log4j.spi.LoggingEvent; +import org.apache.log4j.spi.ThrowableInformation; import java.util.HashSet; import java.util.Set; public class EcsLayout extends Layout { + private boolean stackTraceAsArray = false; private String serviceName; private Set topLevelLabels = new HashSet(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS); @@ -41,13 +43,16 @@ public String format(LoggingEvent event) { StringBuilder builder = new StringBuilder(); EcsJsonSerializer.serializeObjectStart(builder, event.getTimeStamp()); EcsJsonSerializer.serializeLogLevel(builder, event.getLevel().toString()); - Throwable thrown = event.getThrowableInformation() != null ? event.getThrowableInformation().getThrowable() : null; - EcsJsonSerializer.serializeFormattedMessage(builder, event.getRenderedMessage(), thrown); + EcsJsonSerializer.serializeFormattedMessage(builder, event.getRenderedMessage()); EcsJsonSerializer.serializeServiceName(builder, serviceName); EcsJsonSerializer.serializeThreadName(builder, event.getThreadName()); EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName()); EcsJsonSerializer.serializeLabels(builder, event.getProperties(), topLevelLabels); EcsJsonSerializer.serializeTag(builder, event.getNDC()); + ThrowableInformation throwableInformation = event.getThrowableInformation(); + if (throwableInformation != null) { + EcsJsonSerializer.serializeException(builder, throwableInformation.getThrowable(), stackTraceAsArray); + } EcsJsonSerializer.serializeObjectEnd(builder); return builder.toString(); } @@ -65,4 +70,8 @@ public void activateOptions() { public void setServiceName(String serviceName) { this.serviceName = serviceName; } + + public void setStackTraceAsArray(boolean stackTraceAsArray) { + this.stackTraceAsArray = stackTraceAsArray; + } } diff --git a/log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java b/log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java index 613de258..78c87a3f 100644 --- a/log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java +++ b/log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java @@ -53,6 +53,7 @@ void setUp() { logger.addAppender(appender); ecsLayout = new EcsLayout(); ecsLayout.setServiceName("test"); + ecsLayout.setStackTraceAsArray(true); } @BeforeEach diff --git a/log4j2-ecs-layout/README.md b/log4j2-ecs-layout/README.md index 1dac33fa..d866b925 100644 --- a/log4j2-ecs-layout/README.md +++ b/log4j2-ecs-layout/README.md @@ -19,13 +19,6 @@ Add a dependency to your application Instead of the usual ``, use ``. -If you want to include [Markers](https://logging.apache.org/log4j/2.0/manual/markers.html) as tags, -set the `includeMarkers` attribute to `true` (default: `false`). - -``` - -``` - ## Example ```xml @@ -46,3 +39,22 @@ set the `includeMarkers` attribute to `true` (default: `false`). ``` + +## Layout Parameters + +|Parameter name |Type |Default|Description| +|-----------------|-------|-------|-----------| +|serviceName |String | |Sets the `service.name` field so you can filter your logs by a particular service | +|includeMarkers |boolean|`false`|Log [Markers](https://logging.apache.org/log4j/2.0/manual/markers.html) as `tags` | +|stackTraceAsArray|boolean|`false`|Serializes the `error.stack_trace` as a JSON array where each element is in a new line to improve readability. Note that this requires a slightly more complex Filebeat setup. See also https://github.com/elastic/java-ecs-logging/blob/master/README.md#TODO| + +To include any custom field in the output, use following syntax: + +```xml + + + + +``` + +Custom fields are included in the order they are declared. The values support [lookups](https://logging.apache.org/log4j/2.x/manual/lookups.html). \ No newline at end of file diff --git a/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java b/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java index c11565b9..061e35db 100644 --- a/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java +++ b/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java @@ -41,13 +41,11 @@ import org.apache.logging.log4j.core.layout.Encoder; import org.apache.logging.log4j.core.lookup.StrSubstitutor; import org.apache.logging.log4j.core.util.KeyValuePair; -import org.apache.logging.log4j.core.util.StringBuilderWriter; import org.apache.logging.log4j.message.MapMessage; import org.apache.logging.log4j.message.Message; import org.apache.logging.log4j.util.StringBuilderFormattable; import org.apache.logging.log4j.util.TriConsumer; -import java.io.PrintWriter; import java.nio.charset.Charset; import java.util.Arrays; import java.util.Collection; @@ -59,7 +57,6 @@ @Plugin(name = "EcsLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE) public class EcsLayout extends AbstractStringLayout { - private static final ThreadLocal messageStringBuilder = new ThreadLocal(); public static final Charset UTF_8 = Charset.forName("UTF-8"); private final TriConsumer WRITE_KEY_VALUES_INTO = new TriConsumer() { @@ -78,14 +75,16 @@ public void accept(final String key, final Object value, final StringBuilder str private final KeyValuePair[] additionalFields; private final Set topLevelLabels; + private final boolean stackTraceAsArray; private String serviceName; private boolean includeMarkers; - private EcsLayout(Configuration config, String serviceName, boolean includeMarkers, KeyValuePair[] additionalFields, Collection topLevelLabels) { + private EcsLayout(Configuration config, String serviceName, boolean includeMarkers, KeyValuePair[] additionalFields, Collection topLevelLabels, boolean stackTraceAsArray) { super(config, UTF_8, null, null); this.serviceName = serviceName; this.includeMarkers = includeMarkers; this.topLevelLabels = new HashSet(topLevelLabels); + this.stackTraceAsArray = stackTraceAsArray; this.topLevelLabels.add("trace.id"); this.topLevelLabels.add("transaction.id"); this.additionalFields = additionalFields; @@ -96,16 +95,6 @@ public static EcsLayout.Builder newBuilder() { return new EcsLayout.Builder().asBuilder(); } - private static StringBuilder getMessageStringBuilder() { - StringBuilder result = messageStringBuilder.get(); - if (result == null) { - result = new StringBuilder(DEFAULT_STRING_BUILDER_SIZE); - messageStringBuilder.set(result); - } - result.setLength(0); - return result; - } - private static boolean valueNeedsLookup(final String value) { return value != null && value.contains("${"); } @@ -132,6 +121,7 @@ private StringBuilder toText(LogEvent event, StringBuilder builder, boolean gcFr EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName()); serializeLabels(event, builder); serializeTags(event, builder); + EcsJsonSerializer.serializeException(builder, event.getThrown(), stackTraceAsArray); EcsJsonSerializer.serializeObjectEnd(builder); return builder; } @@ -143,7 +133,7 @@ private void serializeLabels(LogEvent event, StringBuilder builder) { for (KeyValuePair additionalField : additionalFields) { CharSequence value = null; if (valueNeedsLookup(additionalField.getValue())) { - StringBuilder lookupValue = getMessageStringBuilder(); + StringBuilder lookupValue = EcsJsonSerializer.getMessageStringBuilder(); lookupValue.append(additionalField.getValue()); if (strSubstitutor.replaceIn(event, lookupValue)) { value = lookupValue; @@ -205,7 +195,7 @@ private void serializeMessage(StringBuilder builder, boolean gcFree, Message mes if (message instanceof CharSequence) { JsonUtils.quoteAsString(((CharSequence) message), builder); } else if (gcFree && message instanceof StringBuilderFormattable) { - final StringBuilder messageBuffer = getMessageStringBuilder(); + final StringBuilder messageBuffer = EcsJsonSerializer.getMessageStringBuilder(); try { ((StringBuilderFormattable) message).formatTo(messageBuffer); JsonUtils.quoteAsString(messageBuffer, builder); @@ -215,10 +205,6 @@ private void serializeMessage(StringBuilder builder, boolean gcFree, Message mes } else { JsonUtils.quoteAsString(EcsJsonSerializer.toNullSafeString(message.getFormattedMessage()), builder); } - if (thrown != null) { - builder.append("\\n"); - JsonUtils.quoteAsString(formatThrowable(thrown), builder); - } builder.append("\", "); if (message instanceof MapMessage) { MapMessage mapMessage = (MapMessage) message; @@ -226,14 +212,6 @@ private void serializeMessage(StringBuilder builder, boolean gcFree, Message mes } } - private static CharSequence formatThrowable(final Throwable throwable) { - StringBuilderWriter sw = new StringBuilderWriter(getMessageStringBuilder()); - final PrintWriter pw = new PrintWriter(sw); - throwable.printStackTrace(pw); - pw.flush(); - return sw.getBuilder(); - } - public static class Builder extends AbstractStringLayout.Builder implements org.apache.logging.log4j.core.util.Builder { @@ -241,6 +219,8 @@ public static class Builder extends AbstractStringLayout.BuilderemptyList() : Arrays.asList(topLevelLabels)); + return new EcsLayout(getConfiguration(), serviceName, includeMarkers, additionalFields, topLevelLabels == null ? Collections.emptyList() : Arrays.asList(topLevelLabels), stackTraceAsArray); + } + + public boolean isStackTraceAsArray() { + return stackTraceAsArray; } } } diff --git a/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java b/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java index 6043111d..8fbc03b0 100644 --- a/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java +++ b/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java @@ -68,6 +68,7 @@ void setUp() { .setConfiguration(ctx.getConfiguration()) .setServiceName("test") .setIncludeMarkers(true) + .setStackTraceAsArray(true) .setAdditionalFields(new KeyValuePair[]{ new KeyValuePair("cluster.uuid", "9fe9134b-20b0-465e-acf9-8cc09ac9053b"), new KeyValuePair("node.id", "${node.id}"), diff --git a/log4j2-ecs-layout/src/test/resources/log4j2-test.xml b/log4j2-ecs-layout/src/test/resources/log4j2-test.xml index cc9f56c1..dc009041 100644 --- a/log4j2-ecs-layout/src/test/resources/log4j2-test.xml +++ b/log4j2-ecs-layout/src/test/resources/log4j2-test.xml @@ -5,7 +5,7 @@ - + diff --git a/logback-ecs-encoder/README.md b/logback-ecs-encoder/README.md index 39b13089..35723f76 100644 --- a/logback-ecs-encoder/README.md +++ b/logback-ecs-encoder/README.md @@ -21,8 +21,6 @@ All you have to do is to use the `co.elastic.logging.logback.EcsEncoder` instead ```xml my-application - - true ``` @@ -58,4 +56,12 @@ All you have to do is to use the `co.elastic.logging.logback.EcsEncoder` instead --> -``` \ No newline at end of file +``` + +## Encoder Parameters + +|Parameter name |Type |Default|Description| +|-----------------|-------|-------|-----------| +|serviceName |String | |Sets the `service.name` field so you can filter your logs by a particular service | +|includeMarkers |boolean|`false`|Log [Markers](https://www.slf4j.org/api/org/slf4j/Marker.html) as `tags` | +|stackTraceAsArray|boolean|`false`|Serializes the `error.stack_trace` as a JSON array where each element is in a new line to improve readability. Note that this requires a slightly more complex Filebeat setup. See also https://github.com/elastic/java-ecs-logging/blob/master/README.md#TODO| diff --git a/logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java b/logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java index fe1899da..a9c09082 100644 --- a/logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java +++ b/logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java @@ -26,9 +26,10 @@ import ch.qos.logback.classic.pattern.ThrowableProxyConverter; import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.IThrowableProxy; +import ch.qos.logback.classic.spi.ThrowableProxy; import ch.qos.logback.core.encoder.EncoderBase; import co.elastic.logging.EcsJsonSerializer; -import co.elastic.logging.JsonUtils; import org.slf4j.Marker; import java.nio.charset.Charset; @@ -39,6 +40,7 @@ public class EcsEncoder extends EncoderBase { private static final Charset UTF_8 = Charset.forName("UTF-8"); + private boolean stackTraceAsArray = false; private String serviceName; private boolean includeMarkers = false; private ThrowableProxyConverter throwableProxyConverter; @@ -61,13 +63,18 @@ public byte[] encode(ILoggingEvent event) { StringBuilder builder = new StringBuilder(); EcsJsonSerializer.serializeObjectStart(builder, event.getTimeStamp()); EcsJsonSerializer.serializeLogLevel(builder, event.getLevel().toString()); - EcsJsonSerializer.serializeFormattedMessage(builder, event.getFormattedMessage(), null); - serializeException(event, builder); + EcsJsonSerializer.serializeFormattedMessage(builder, event.getFormattedMessage()); serializeMarkers(event, builder); EcsJsonSerializer.serializeServiceName(builder, serviceName); EcsJsonSerializer.serializeThreadName(builder, event.getThreadName()); EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName()); EcsJsonSerializer.serializeLabels(builder, event.getMDCPropertyMap(), topLevelLabels); + IThrowableProxy throwableProxy = event.getThrowableProxy(); + if (throwableProxy instanceof ThrowableProxy) { + EcsJsonSerializer.serializeException(builder, ((ThrowableProxy) throwableProxy).getThrowable(), stackTraceAsArray); + } else if (throwableProxy != null) { + EcsJsonSerializer.serializeException(builder, throwableProxy.getClassName(), throwableProxy.getMessage(), throwableProxyConverter.convert(event), stackTraceAsArray); + } EcsJsonSerializer.serializeObjectEnd(builder); // all these allocations kinda hurt return builder.toString().getBytes(UTF_8); @@ -92,16 +99,6 @@ private void serializeMarker(StringBuilder builder, Marker marker) { } } - private void serializeException(ILoggingEvent event, StringBuilder builder) { - if (event.getThrowableProxy() != null) { - // remove `", ` - builder.setLength(builder.length() - 3); - builder.append("\\n"); - JsonUtils.quoteAsString(throwableProxyConverter.convert(event), builder); - builder.append("\","); - } - } - @Override public byte[] footerBytes() { return null; @@ -114,4 +111,8 @@ public void setServiceName(String serviceName) { public void setIncludeMarkers(boolean includeMarkers) { this.includeMarkers = includeMarkers; } + + public void setStackTraceAsArray(boolean stackTraceAsArray) { + this.stackTraceAsArray = stackTraceAsArray; + } } diff --git a/logback-ecs-encoder/src/test/java/co/elastic/logging/logback/EcsEncoderTest.java b/logback-ecs-encoder/src/test/java/co/elastic/logging/logback/EcsEncoderTest.java index 3a01da3a..250c4f5d 100644 --- a/logback-ecs-encoder/src/test/java/co/elastic/logging/logback/EcsEncoderTest.java +++ b/logback-ecs-encoder/src/test/java/co/elastic/logging/logback/EcsEncoderTest.java @@ -48,6 +48,7 @@ void setUp() { ecsEncoder = new EcsEncoder(); ecsEncoder.setServiceName("test"); ecsEncoder.setIncludeMarkers(true); + ecsEncoder.setStackTraceAsArray(true); ecsEncoder.start(); } diff --git a/logback-ecs-encoder/src/test/resources/logback-config.xml b/logback-ecs-encoder/src/test/resources/logback-config.xml index 9477481a..710d742e 100644 --- a/logback-ecs-encoder/src/test/resources/logback-config.xml +++ b/logback-ecs-encoder/src/test/resources/logback-config.xml @@ -4,6 +4,7 @@ test true + true