Skip to content

Commit

Permalink
Add support for logging markers as tags (elastic#14)
Browse files Browse the repository at this point in the history
Logback (SLF4J) and Log4j 2 support Markers, which can contain useful information.

This change adds support for optionally logging these markers as tags in the created/encoded log messages.

By default, this feature is disabled and can be enabled by setting the `includeMarkers` configuration setting to `true` in the Log4j 2 `EcsLayout` and Logback `EcsEncoder`.

See also:
* https://logging.apache.org/log4j/2.0/manual/markers.html
* https://www.slf4j.org/api/org/slf4j/Marker.html
  • Loading branch information
joschi authored and felixbarny committed Sep 12, 2019
1 parent 283f2b7 commit d95ad5d
Show file tree
Hide file tree
Showing 14 changed files with 461 additions and 98 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,21 @@ public static void serializeTag(StringBuilder builder, String tag) {
}
}

public static void serializeTagStart(StringBuilder builder) {
builder.append("\"tags\":[");
}

public static void serializeSingleTag(StringBuilder builder, String tag) {
if (tag != null) {
builder.append("\"").append(tag).append("\",");
}
}

public static void serializeTagEnd(StringBuilder builder) {
builder.setLength(builder.length() - 1);
builder.append("],");
}

public static void serializeLabels(StringBuilder builder, Map<String, ?> labels, Set<String> topLevelLabels) {
if (!labels.isEmpty()) {
for (Map.Entry<String, ?> entry : labels.entrySet()) {
Expand Down
9 changes: 8 additions & 1 deletion log4j2-ecs-layout/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,14 @@ Add a dependency to your application

## Step 2: use the `EcsLayout`

Instead of the usual `<PatternLayout/>`, use `<EcsLayout serviceName="my-app"/>`
Instead of the usual `<PatternLayout/>`, use `<EcsLayout serviceName="my-app"/>`.

If you want to include [Markers](https://logging.apache.org/log4j/2.0/manual/markers.html) as tags,
set the `includeMarkers` attribute to `true` (default: `false`).

```
<EcsLayout serviceName="my-app" includeMarkers="true"/>
```

## Example
```xml
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

import co.elastic.logging.EcsJsonSerializer;
import co.elastic.logging.JsonUtils;
import org.apache.logging.log4j.Marker;
import org.apache.logging.log4j.core.Layout;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.config.Configuration;
Expand Down Expand Up @@ -59,6 +60,7 @@
public class EcsLayout extends AbstractStringLayout {

private static final ThreadLocal<StringBuilder> messageStringBuilder = new ThreadLocal<StringBuilder>();
public static final Charset UTF_8 = Charset.forName("UTF-8");

private final TriConsumer<String, Object, StringBuilder> WRITE_KEY_VALUES_INTO = new TriConsumer<String, Object, StringBuilder>() {
@Override
Expand All @@ -77,10 +79,12 @@ public void accept(final String key, final Object value, final StringBuilder str
private final KeyValuePair[] additionalFields;
private final Set<String> topLevelLabels;
private String serviceName;
private boolean includeMarkers;

private EcsLayout(Configuration config, String serviceName, KeyValuePair[] additionalFields, Collection<String> topLevelLabels) {
super(config, Charset.forName("UTF-8"), null, null);
private EcsLayout(Configuration config, String serviceName, boolean includeMarkers, KeyValuePair[] additionalFields, Collection<String> topLevelLabels) {
super(config, UTF_8, null, null);
this.serviceName = serviceName;
this.includeMarkers = includeMarkers;
this.topLevelLabels = new HashSet<String>(topLevelLabels);
this.topLevelLabels.add("trace.id");
this.topLevelLabels.add("transaction.id");
Expand Down Expand Up @@ -163,16 +167,36 @@ private void serializeLabels(LogEvent event, StringBuilder builder) {

private void serializeTags(LogEvent event, StringBuilder builder) {
List<String> contextStack = event.getContextStack().asList();
Marker marker = event.getMarker();
boolean hasTags = !contextStack.isEmpty() || (includeMarkers && marker != null);
if (hasTags) {
EcsJsonSerializer.serializeTagStart(builder);
}

if (!contextStack.isEmpty()) {
builder.append("\"tags\":[");
for (int i = 0; i < contextStack.size(); i++) {
builder.append('\"');
JsonUtils.quoteAsString(contextStack.get(i), builder);
builder.append("\",");
}
// removes last comma
builder.setLength(builder.length() - 1);
builder.append("],");
}

if (includeMarkers && marker != null) {
serializeMarker(builder, marker);
}

if (hasTags) {
EcsJsonSerializer.serializeTagEnd(builder);
}
}

private void serializeMarker(StringBuilder builder, Marker marker) {
EcsJsonSerializer.serializeSingleTag(builder, marker.getName());
if (marker.hasParents()) {
Marker[] parents = marker.getParents();
for (int i = 0; i < parents.length; i++) {
serializeMarker(builder, parents[i]);
}
}
}

Expand Down Expand Up @@ -215,14 +239,16 @@ public static class Builder extends AbstractStringLayout.Builder<EcsLayout.Build

@PluginBuilderAttribute("serviceName")
private String serviceName;
@PluginBuilderAttribute("includeMarkers")
private boolean includeMarkers = false;
@PluginElement("AdditionalField")
private KeyValuePair[] additionalFields;
@PluginElement("TopLevelLabels")
private String[] topLevelLabels;

Builder() {
super();
setCharset(Charset.forName("UTF-8"));
setCharset(UTF_8);
}

public KeyValuePair[] getAdditionalFields() {
Expand All @@ -233,6 +259,10 @@ public String getServiceName() {
return serviceName;
}

public boolean isIncludeMarkers() {
return includeMarkers;
}

public String[] getTopLevelLabels() {
return topLevelLabels;
}
Expand All @@ -257,9 +287,14 @@ public EcsLayout.Builder setServiceName(final String serviceName) {
return asBuilder();
}

public EcsLayout.Builder setIncludeMarkers(final boolean includeMarkers) {
this.includeMarkers = includeMarkers;
return asBuilder();
}

@Override
public EcsLayout build() {
return new EcsLayout(getConfiguration(), serviceName, additionalFields, topLevelLabels == null ? Collections.<String>emptyList() : Arrays.<String>asList(topLevelLabels));
return new EcsLayout(getConfiguration(), serviceName, includeMarkers, additionalFields, topLevelLabels == null ? Collections.<String>emptyList() : Arrays.<String>asList(topLevelLabels));
}
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/*-
* #%L
* Java ECS logging
* %%
* Copyright (C) 2019 Elastic and contributors
* %%
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* #L%
*/
package co.elastic.logging.log4j2;

import co.elastic.logging.AbstractEcsLoggingTest;
import com.fasterxml.jackson.databind.node.TextNode;
import org.apache.logging.log4j.Marker;
import org.apache.logging.log4j.MarkerManager;
import org.apache.logging.log4j.ThreadContext;
import org.apache.logging.log4j.core.Logger;
import org.apache.logging.log4j.message.StringMapMessage;
import org.apache.logging.log4j.test.appender.ListAppender;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;

import java.util.Map;

import static org.assertj.core.api.Assertions.assertThat;

abstract class AbstractLog4j2EcsLayoutTest extends AbstractEcsLoggingTest {
protected Logger root;
protected ListAppender listAppender;

@AfterEach
void tearDown() throws Exception {
ThreadContext.clearAll();
}

@Test
void globalLabels() throws Exception {
putMdc("trace.id", "foo");
debug("test");
assertThat(getLastLogLine().get("cluster.uuid").textValue()).isEqualTo("9fe9134b-20b0-465e-acf9-8cc09ac9053b");
assertThat(getLastLogLine().get("node.id").textValue()).isEqualTo("foo");
assertThat(getLastLogLine().get("404")).isNull();
}

@Test
void testMarker() throws Exception {
Marker parent = MarkerManager.getMarker("parent");
Marker child = MarkerManager.getMarker("child").setParents(parent);
Marker grandchild = MarkerManager.getMarker("grandchild").setParents(child);
root.debug(grandchild, "test");

assertThat(getLastLogLine().get("tags")).contains(
TextNode.valueOf("parent"),
TextNode.valueOf("child"),
TextNode.valueOf("grandchild"));
}

@Test
void testMapMessage() throws Exception {
root.info(new StringMapMessage(Map.of("foo", "bar")));
assertThat(getLastLogLine().get("labels.foo").textValue()).isEqualTo("bar");
}

@Override
public void putMdc(String key, String value) {
ThreadContext.put(key, value);
}

@Override
public boolean putNdc(String message) {
ThreadContext.push(message);
return true;
}

@Override
public void debug(String message) {
root.debug(message);
}

@Override
public void error(String message, Throwable t) {
root.error(message, t);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*-
* #%L
* Java ECS logging
* %%
* Copyright (C) 2019 Elastic and contributors
* %%
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* #L%
*/
package co.elastic.logging.log4j2;

import com.fasterxml.jackson.databind.JsonNode;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.test.appender.ListAppender;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;

import java.io.IOException;

class Log4j2EcsLayoutIntegrationTest extends AbstractLog4j2EcsLayoutTest {
@BeforeEach
void setUp() {
root = LoggerContext.getContext().getRootLogger();
listAppender = (ListAppender) root.getAppenders().get("TestAppender");
}

@AfterEach
void tearDown() throws Exception {
super.tearDown();
listAppender.clear();
}

@Override
public JsonNode getLastLogLine() throws IOException {
return objectMapper.readTree(listAppender.getMessages().get(0));
}
}
Loading

0 comments on commit d95ad5d

Please sign in to comment.