Add support for logging markers as tags

ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java

@@ -103,6 +103,21 @@ public static void serializeTag(StringBuilder builder, String tag) {
         }
     }
 
+    public static void serializeTagStart(StringBuilder builder) {
+        builder.append("\"tags\":[");
+    }
+
+    public static void serializeSingleTag(StringBuilder builder, String tag) {
+        if (tag != null) {
+            builder.append("\"").append(tag).append("\",");
+        }
+    }
+
+    public static void serializeTagEnd(StringBuilder builder) {
+        builder.setLength(builder.length() - 1);
+        builder.append("],");
+    }
+
     public static void serializeLabels(StringBuilder builder, Map<String, ?> labels, Set<String> topLevelLabels) {
         if (!labels.isEmpty()) {
             for (Map.Entry<String, ?> entry : labels.entrySet()) {

log4j2-ecs-layout/README.md

@@ -17,7 +17,14 @@ Add a dependency to your application
 
 ## Step 2: use the `EcsLayout`
 
-Instead of the usual `<PatternLayout/>`, use `<EcsLayout serviceName="my-app"/>`
+Instead of the usual `<PatternLayout/>`, use `<EcsLayout serviceName="my-app"/>`.
+
+If you want to include [Markers](https://logging.apache.org/log4j/2.0/manual/markers.html) as tags,
+set the `includeMarkers` attribute to `true` (default: `false`).
+
+```
+<EcsLayout serviceName="my-app" includeMarkers="true"/>
+```
 
 ## Example
 ```xml

log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java

@@ -26,6 +26,7 @@
 
 
 import co.elastic.logging.EcsJsonSerializer;
+import org.apache.logging.log4j.Marker;
 import org.apache.logging.log4j.core.Layout;
 import org.apache.logging.log4j.core.LogEvent;
 import org.apache.logging.log4j.core.config.Configuration;
@@ -58,6 +59,7 @@
 public class EcsLayout extends AbstractStringLayout {
 
     private static final ThreadLocal<StringBuilder> messageStringBuilder = new ThreadLocal<StringBuilder>();
+    public static final Charset UTF_8 = Charset.forName("UTF-8");
 
     private final TriConsumer<String, Object, StringBuilder> WRITE_KEY_VALUES_INTO = new TriConsumer<String, Object, StringBuilder>() {
         @Override
@@ -76,10 +78,12 @@ public void accept(final String key, final Object value, final StringBuilder str
     private final KeyValuePair[] additionalFields;
     private final Set<String> topLevelLabels;
     private String serviceName;
+    private boolean includeMarkers;
 
-    private EcsLayout(Configuration config, String serviceName, KeyValuePair[] additionalFields, Collection<String> topLevelLabels) {
-        super(config, Charset.forName("UTF-8"), null, null);
+    private EcsLayout(Configuration config, String serviceName, boolean includeMarkers, KeyValuePair[] additionalFields, Collection<String> topLevelLabels) {
+        super(config, UTF_8, null, null);
         this.serviceName = serviceName;
+        this.includeMarkers = includeMarkers;
         this.topLevelLabels = new HashSet<String>(topLevelLabels);
         this.topLevelLabels.add("trace.id");
         this.topLevelLabels.add("transaction.id");
@@ -152,16 +156,35 @@ private void serializeLabels(LogEvent event, StringBuilder builder) {
 
     private void serializeTags(LogEvent event, StringBuilder builder) {
         List<String> contextStack = event.getContextStack().asList();
+        Marker marker = event.getMarker();
+        if (!contextStack.isEmpty() || (includeMarkers && marker != null)) {
+            EcsJsonSerializer.serializeTagStart(builder);
+        }
+
         if (!contextStack.isEmpty()) {
-            builder.append("\"tags\":[");
             for (int i = 0; i < contextStack.size(); i++) {
                 builder.append('\"');
                 JsonUtils.quoteAsString(contextStack.get(i), builder);
                 builder.append("\",");
             }
-            // removes last comma
-            builder.setLength(builder.length() - 1);
-            builder.append("],");
+        }
+
+        if (includeMarkers && marker != null) {
+            serializeMarker(builder, marker);
+        }
+
+        if (!contextStack.isEmpty() || (includeMarkers && marker != null)) {
+            EcsJsonSerializer.serializeTagEnd(builder);
+        }
+    }
+
+    private void serializeMarker(StringBuilder builder, Marker marker) {
+        EcsJsonSerializer.serializeSingleTag(builder, marker.getName());
+        if (marker.hasParents()) {
+            Marker[] parents = marker.getParents();
+            for (int i = 0; i < parents.length; i++) {
+                serializeMarker(builder, parents[i]);
+            }
         }
     }
 
@@ -201,14 +224,16 @@ public static class Builder extends AbstractStringLayout.Builder<EcsLayout.Build
 
         @PluginBuilderAttribute("serviceName")
         private String serviceName;
+        @PluginBuilderAttribute("includeMarkers")
+        private boolean includeMarkers = false;
         @PluginElement("AdditionalField")
         private KeyValuePair[] additionalFields;
         @PluginElement("TopLevelLabels")
         private String[] topLevelLabels;
 
         Builder() {
             super();
-            setCharset(Charset.forName("UTF-8"));
+            setCharset(UTF_8);
         }
 
         public KeyValuePair[] getAdditionalFields() {
@@ -219,6 +244,10 @@ public String getServiceName() {
             return serviceName;
         }
 
+        public boolean isIncludeMarkers() {
+            return includeMarkers;
+        }
+
         public String[] getTopLevelLabels() {
             return topLevelLabels;
         }
@@ -243,9 +272,14 @@ public EcsLayout.Builder setServiceName(final String serviceName) {
             return asBuilder();
         }
 
+        public EcsLayout.Builder setIncludeMarkers(final boolean includeMarkers) {
+            this.includeMarkers = includeMarkers;
+            return asBuilder();
+        }
+
         @Override
         public EcsLayout build() {
-            return new EcsLayout(getConfiguration(), serviceName, additionalFields, topLevelLabels == null ? Collections.<String>emptyList() : Arrays.<String>asList(topLevelLabels));
+            return new EcsLayout(getConfiguration(), serviceName, includeMarkers, additionalFields, topLevelLabels == null ? Collections.<String>emptyList() : Arrays.<String>asList(topLevelLabels));
         }
     }
 }

log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutIntegrationTest.java

@@ -0,0 +1,130 @@
+/*-
+ * #%L
+ * Java ECS logging
+ * %%
+ * Copyright (C) 2019 Elastic and contributors
+ * %%
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * #L%
+ */
+package co.elastic.logging.log4j2;
+
+import co.elastic.logging.AbstractEcsLoggingTest;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.MarkerManager;
+import org.apache.logging.log4j.ThreadContext;
+import org.apache.logging.log4j.core.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.ConfigurationFactory;
+import org.apache.logging.log4j.core.config.xml.XmlConfigurationFactory;
+import org.apache.logging.log4j.test.appender.ListAppender;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class Log4j2EcsLayoutIntegrationTest extends AbstractEcsLoggingTest {
+
+    private static ConfigurationFactory configFactory = new XmlConfigurationFactory();
+    private LoggerContext ctx = LoggerContext.getContext();
+    private Logger root = ctx.getRootLogger();
+    private ListAppender listAppender;
+
+    @AfterAll
+    static void cleanupClass() {
+        ConfigurationFactory.removeConfigurationFactory(configFactory);
+    }
+
+    @BeforeAll
+    static void setupClass() throws URISyntaxException {
+        ConfigurationFactory.setConfigurationFactory(configFactory);
+        final ClassLoader classLoader = Log4j2EcsLayoutIntegrationTest.class.getClassLoader();
+        final LoggerContext ctx = LoggerContext.getContext(classLoader,
+                true,
+                classLoader.getResource("log4j2-config.xml").toURI());
+        ctx.reconfigure();
+    }
+
+    @BeforeEach
+    void setUp() throws Exception {
+        listAppender = (ListAppender) root.getAppenders().get("TestAppender");
+        ctx.getConfiguration().getProperties().put("node.id", "foo");
+    }
+
+    @AfterEach
+    void tearDown() throws Exception {
+        listAppender.clear();
+        ThreadContext.clearAll();
+    }
+
+    @Test
+    void globalLabels() throws Exception {
+        putMdc("trace.id", "foo");
+        debug("test");
+        assertThat(getLastLogLine().get("cluster.uuid").textValue()).isEqualTo("9fe9134b-20b0-465e-acf9-8cc09ac9053b");
+        assertThat(getLastLogLine().get("node.id").textValue()).isEqualTo("foo");
+        assertThat(getLastLogLine().get("404")).isNull();
+    }
+
+    @Test
+    void testMarker() throws Exception {
+        Marker parent = MarkerManager.getMarker("parent");
+        Marker child = MarkerManager.getMarker("child").setParents(parent);
+        Marker grandchild = MarkerManager.getMarker("grandchild").setParents(child);
+        root.debug(grandchild, "test");
+
+        assertThat(getLastLogLine().get("tags")).contains(
+                TextNode.valueOf("parent"),
+                TextNode.valueOf("child"),
+                TextNode.valueOf("grandchild"));
+    }
+
+    @Override
+    public void putMdc(String key, String value) {
+        ThreadContext.put(key, value);
+    }
+
+    @Override
+    public boolean putNdc(String message) {
+        ThreadContext.push(message);
+        return true;
+    }
+
+    @Override
+    public void debug(String message) {
+        root.debug(message);
+    }
+
+    @Override
+    public void error(String message, Throwable t) {
+        root.error(message, t);
+    }
+
+    @Override
+    public JsonNode getLastLogLine() throws IOException {
+        return objectMapper.readTree(listAppender.getMessages().get(0));
+    }
+}

log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java

@@ -26,8 +26,10 @@
 
 import co.elastic.logging.AbstractEcsLoggingTest;
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.TextNode;
 import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.MarkerManager;
 import org.apache.logging.log4j.ThreadContext;
 import org.apache.logging.log4j.core.Appender;
 import org.apache.logging.log4j.core.BasicConfigurationFactory;
@@ -51,29 +53,29 @@ class Log4j2EcsLayoutTest extends AbstractEcsLoggingTest {
     private static ConfigurationFactory configFactory = new BasicConfigurationFactory();
     private LoggerContext ctx = LoggerContext.getContext();
     private Logger root = ctx.getRootLogger();
-    private ObjectMapper objectMapper = new ObjectMapper();
     private ListAppender listAppender;
 
     @AfterAll
-    public static void cleanupClass() {
+    static void cleanupClass() {
         ConfigurationFactory.removeConfigurationFactory(configFactory);
     }
 
     @BeforeAll
-    public static void setupClass() {
+    static void setupClass() {
         ConfigurationFactory.setConfigurationFactory(configFactory);
         final LoggerContext ctx = LoggerContext.getContext();
         ctx.reconfigure();
     }
 
     @BeforeEach
-    public void setUp() throws Exception {
+    void setUp() throws Exception {
         for (final Appender appender : root.getAppenders().values()) {
             root.removeAppender(appender);
         }
         EcsLayout ecsLayout = EcsLayout.newBuilder()
                 .setConfiguration(ctx.getConfiguration())
                 .setServiceName("test")
+                .setIncludeMarkers(true)
                 .setAdditionalFields(new KeyValuePair[]{
                         new KeyValuePair("cluster.uuid", "9fe9134b-20b0-465e-acf9-8cc09ac9053b"),
                         new KeyValuePair("node.id", "${node.id}"),
@@ -88,7 +90,7 @@ public void setUp() throws Exception {
     }
 
     @AfterEach
-    public void tearDown() throws Exception {
+    void tearDown() throws Exception {
         ThreadContext.clearAll();
     }
 
@@ -101,6 +103,19 @@ void globalLabels() throws Exception {
         assertThat(getLastLogLine().get("404")).isNull();
     }
 
+    @Test
+    void testMarker() throws Exception {
+        Marker parent = MarkerManager.getMarker("parent");
+        Marker child = MarkerManager.getMarker("child").setParents(parent);
+        Marker grandchild = MarkerManager.getMarker("grandchild").setParents(child);
+        root.debug(grandchild, "test");
+
+        assertThat(getLastLogLine().get("tags")).contains(
+                TextNode.valueOf("parent"),
+                TextNode.valueOf("child"),
+                TextNode.valueOf("grandchild"));
+    }
+
     @Override
     public void putMdc(String key, String value) {
         ThreadContext.put(key, value);

log4j2-ecs-layout/src/test/resources/log4j2-config.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration packages="co.elastic.logging.log4j2">
+    <Appenders>
+        <List name="TestAppender" newLine="false" raw="false" level="debug">
+            <EcsLayout serviceName="test" includeMarkers="true">
+                <properties>
+                    <property name="node.id">foo</property>
+                </properties>
+                <KeyValuePair key="cluster.uuid" value="9fe9134b-20b0-465e-acf9-8cc09ac9053b"/>
+                <KeyValuePair key="node.id" value="${node.id}"/>
+            </EcsLayout>
+        </List>
+    </Appenders>
+    <Loggers>
+        <Root level="debug">
+            <AppenderRef ref="TestAppender"/>
+        </Root>
+    </Loggers>
+</Configuration>

logback-ecs-encoder/README.md

@@ -21,6 +21,8 @@ All you have to do is to use the `co.elastic.logging.logback.EcsEncoder` instead
 ```xml
 <encoder class="co.elastic.logging.logback.EcsEncoder">
     <serviceName>my-application</serviceName>
+    <!-- Log markers as tags, default: false. See also https://www.slf4j.org/api/org/slf4j/Marker.html -->
+    <includeMarkers>true</includeMarkers>
 </encoder>
 ```
 
@@ -56,5 +58,4 @@ All you have to do is to use the `co.elastic.logging.logback.EcsEncoder` instead
         -->
     </root>
 </configuration>
-
 ```