diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7cb6afab..65477b2f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,20 +41,6 @@ jobs: runs-on: ubuntu-latest container: maven:3-jdk-8 - services: - splunk: - image: splunk/splunk:${{matrix.splunk-version}} - env: - SPLUNK_START_ARGS: --accept-license - SPLUNK_PASSWORD: changed! - ports: - - 8089 - - 8088 - - 5555 - - 15000 - - 10667 - - 10668/udp - steps: - uses: actions/checkout@v2 - name: Set up JDK @@ -72,4 +58,5 @@ jobs: run: mvn -P AcceptanceTest -B verify --file pom.xml env: SPLUNK_PASSWORD: changed! - SPLUNK_HOST: splunk \ No newline at end of file + SPLUNK_HOST: splunk + USE_IMAGE_VERSION: ${{matrix.splunk-version}} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 07c10a86..25f67223 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ 2.22.2 - **/HttpEventCollector_*.class + **/TestSuiteAcceptanceTests.class @@ -160,7 +160,7 @@ 2.22.2 - **/HttpLoggerStressTest.class + **/TestSuiteStressTests.class @@ -198,6 +198,18 @@ 1.7.36 test + + org.testcontainers + testcontainers + 1.20.1 + test + + + net.logstash.logback + logstash-logback-encoder + 7.3 + test + ch.qos.logback logback-classic diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorLogbackAppender.java b/src/main/java/com/splunk/logging/HttpEventCollectorLogbackAppender.java index f77c164d..1e7bf88e 100644 --- a/src/main/java/com/splunk/logging/HttpEventCollectorLogbackAppender.java +++ b/src/main/java/com/splunk/logging/HttpEventCollectorLogbackAppender.java @@ -22,19 +22,27 @@ import ch.qos.logback.classic.spi.StackTraceElementProxy; import ch.qos.logback.core.AppenderBase; import ch.qos.logback.core.Layout; +import ch.qos.logback.core.encoder.Encoder; +import ch.qos.logback.core.encoder.LayoutWrappingEncoder; +import ch.qos.logback.core.spi.DeferredProcessingAware; import com.google.gson.Gson; import com.splunk.logging.hec.MetadataTags; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; /** * Logback Appender which writes its events to Splunk http event collector rest endpoint. + * Remarks: + * {@link ch.qos.logback.core.spi.DeferredProcessingAware} is the super-interface of all loggable events. */ -public class HttpEventCollectorLogbackAppender extends AppenderBase { +public class HttpEventCollectorLogbackAppender + extends AppenderBase { + private HttpEventCollectorSender sender = null; - private Layout _layout; + private Encoder _encoder; private boolean _includeLoggerName = true; private boolean _includeThreadName = true; private boolean _includeMDC = true; @@ -59,11 +67,17 @@ public class HttpEventCollectorLogbackAppender extends AppenderBase { private long _batchSize = 0; private String _sendMode; private long _retriesOnError = 0; - private Map _metadata = new HashMap<>(); + private final Map _metadata = new HashMap<>(); private boolean _batchingConfigured = false; - private HttpEventCollectorSender.TimeoutSettings timeoutSettings = new HttpEventCollectorSender.TimeoutSettings(); + private final HttpEventCollectorSender.TimeoutSettings timeoutSettings = new HttpEventCollectorSender.TimeoutSettings(); + + private static class EncodeFailException extends Exception { + EncodeFailException(Throwable cause) { + super(cause); + } + } @Override public void start() { @@ -92,6 +106,16 @@ public void start() { throw new IllegalArgumentException("Batching configuration and sending type of raw are incompatible."); } + // encoder + if (this._encoder == null) { + addError("No encoder set for the appender named [" + name + "]."); + } else { + _encoder.setContext(this.context); + if (!_encoder.isStarted()) { + _encoder.start(); + } + } + this.sender = new HttpEventCollectorSender( _url, _token, _channel, _type, _batchInterval, _batchCount, _batchSize, _sendMode, metadata, timeoutSettings); @@ -141,6 +165,7 @@ public void stop() { if (!started) return; this.sender.close(); + _encoder.stop(); super.stop(); } @@ -153,6 +178,19 @@ protected void append(E e) { } } + private String encodeMessage(E message) throws EncodeFailException { + byte[] encoded; + + try { + encoded = _encoder.encode(message); + } catch (Exception e) { + throw new EncodeFailException(e); + } + + return new String(encoded, StandardCharsets.UTF_8); + } + + @SuppressWarnings("unchecked") private void sendEvent(ILoggingEvent event) { event.prepareForDeferredProcessing(); if (event.hasCallerData()) { @@ -189,31 +227,32 @@ private void sendEvent(ILoggingEvent event) { // No actions here } - MarkerConverter c = new MarkerConverter(); if (this.started) { - this.sender.send( - event.getTimeStamp(), - event.getLevel().toString(), - _layout.doLayout((E) event), - _includeLoggerName ? event.getLoggerName() : null, - _includeThreadName ? event.getThreadName() : null, - _includeMDC ? event.getMDCPropertyMap() : null, - (_includeException && isExceptionOccured) ? exceptionDetail : null, - c.convert(event) - ); + MarkerConverter c = new MarkerConverter(); + try { + this.sender.send( + event.getTimeStamp(), + event.getLevel().toString(), + this.encodeMessage((E) event), // Fine. E is super of ILoggingEvent + _includeLoggerName ? event.getLoggerName() : null, + _includeThreadName ? event.getThreadName() : null, + _includeMDC ? event.getMDCPropertyMap() : null, + (_includeException && isExceptionOccured) ? exceptionDetail : null, + c.convert(event) + ); + } catch (EncodeFailException e) { + addWarn("Failed to encode event. Dropping event.", e.getCause()); + } } } // send non ILoggingEvent such as ch.qos.logback.access.spi.IAccessEvent private void sendEvent(E e) { - String message = _layout.doLayout(e); - if (message == null) { - throw new IllegalArgumentException(String.format( - "The logback layout %s is probably incorrect, " + - "and fails to format the message.", - _layout.toString())); + try { + this.sender.send(this.encodeMessage(e)); + } catch (EncodeFailException ex) { + addWarn("Failed to encode event. Dropping event.", ex.getCause()); } - this.sender.send(message); } public void setUrl(String url) { @@ -261,11 +300,22 @@ public String getType() { } public void setLayout(Layout layout) { - this._layout = layout; + this.addWarn("This appender no longer admits a layout as a sub-component, set an encoder instead."); + this.addWarn("To ensure compatibility, wrapping your layout in LayoutWrappingEncoder."); + this.addWarn("See also http://logback.qos.ch/codes.html#layoutInsteadOfEncoder for details"); + + LayoutWrappingEncoder layoutWrappingEncoder = new LayoutWrappingEncoder<>(); + layoutWrappingEncoder.setLayout(layout); + layoutWrappingEncoder.setContext(this.context); + this._encoder = layoutWrappingEncoder; + } + + public Encoder getEncoder() { + return this._encoder; } - public Layout getLayout() { - return this._layout; + public void setEncoder(Encoder encoder) { + this._encoder = encoder; } public boolean getIncludeLoggerName() { diff --git a/src/test/java/HttpEventCollector_Log4j2Test.java b/src/test/java/HttpEventCollector_Log4j2Test.java index e3be845f..991f3f68 100644 --- a/src/test/java/HttpEventCollector_Log4j2Test.java +++ b/src/test/java/HttpEventCollector_Log4j2Test.java @@ -30,6 +30,7 @@ import java.util.List; public final class HttpEventCollector_Log4j2Test { + private String httpEventCollectorName = "Log4j2Test"; final List> errors = new ArrayList<>(); final List logEx = new ArrayList<>(); diff --git a/src/test/java/HttpEventCollector_Test.java b/src/test/java/HttpEventCollector_Test.java index 7d76ead7..e2aeb8a5 100644 --- a/src/test/java/HttpEventCollector_Test.java +++ b/src/test/java/HttpEventCollector_Test.java @@ -17,22 +17,25 @@ */ import ch.qos.logback.core.joran.spi.JoranException; +import com.splunk.*; import com.splunk.logging.HttpEventCollectorErrorHandler; import com.splunk.logging.HttpEventCollectorEventInfo; import org.junit.Assert; import org.junit.Test; -import java.io.*; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Method; import java.net.URI; import java.net.URL; import java.net.URLClassLoader; -import java.util.*; -import java.lang.reflect.*; - -import com.splunk.*; -import org.slf4j.*; +import java.util.Date; +import java.util.HashMap; +import java.util.List; public class HttpEventCollector_Test { + public static void addPath(String s) throws Exception { File f = new File(s); URI u = f.toURI(); diff --git a/src/test/java/TestRuleSplunkContainer.java b/src/test/java/TestRuleSplunkContainer.java new file mode 100644 index 00000000..a47efd06 --- /dev/null +++ b/src/test/java/TestRuleSplunkContainer.java @@ -0,0 +1,70 @@ + +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.MultipleFailureException; +import org.junit.runners.model.Statement; +import org.testcontainers.containers.FixedHostPortGenericContainer; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.InternetProtocol; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +public class TestRuleSplunkContainer implements TestRule { + // Allow choosing image version from CI + private static String dockerImageName() { + String version = System.getenv().getOrDefault("USE_IMAGE_VERSION", "latest"); + return String.join( + ":", + "splunk/splunk", + version); + } + + @Override + public Statement apply(Statement base, Description description) { + try ( + GenericContainer splunk = new FixedHostPortGenericContainer<>(dockerImageName()) + .withFixedExposedPort(8000, 8000) + .withFixedExposedPort(5555, 5555) + .withFixedExposedPort(8088, 8088) + .withFixedExposedPort(8089, 8089) + .withFixedExposedPort(15000, 15000) + .withFixedExposedPort(10667, 10667) + .withFixedExposedPort(10668, 10668, InternetProtocol.UDP) + .withExposedPorts(9997) + .withEnv("SPLUNK_START_ARGS", "--accept-license") + .withEnv("SPLUNK_PASSWORD", "changed!") + .withStartupTimeout(Duration.ofMinutes(2L)) + .waitingFor(Wait.forListeningPorts(9997)) // must wait on an unfixed port + ) { + return wrapTestCase(base, splunk); + } + } + + private Statement wrapTestCase(final Statement base, final GenericContainer container) { + return new Statement() { + public void evaluate() throws Throwable { + List errors = new ArrayList<>(); + + try { + // Pre-Test + container.start(); + base.evaluate(); + } catch (Throwable preError) { + errors.add(preError); + } finally { + // Post-Test + try { + container.stop(); + } catch (Throwable postError) { + errors.add(postError); + } + } + + MultipleFailureException.assertEmpty(errors); + } + }; + } +} diff --git a/src/test/java/TestSuiteAcceptanceTests.java b/src/test/java/TestSuiteAcceptanceTests.java new file mode 100644 index 00000000..1cba0f82 --- /dev/null +++ b/src/test/java/TestSuiteAcceptanceTests.java @@ -0,0 +1,15 @@ +import org.junit.ClassRule; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + HttpEventCollector_JavaLoggingTest.class, + HttpEventCollector_Log4j2Test.class, + HttpEventCollector_LogbackTest.class, + HttpEventCollector_Test.class +}) +public class TestSuiteAcceptanceTests { + @ClassRule + public static final TestRuleSplunkContainer TEST_CONTAINER = new TestRuleSplunkContainer(); +} diff --git a/src/test/java/TestSuiteStressTests.java b/src/test/java/TestSuiteStressTests.java new file mode 100644 index 00000000..62a9c2e3 --- /dev/null +++ b/src/test/java/TestSuiteStressTests.java @@ -0,0 +1,12 @@ +import org.junit.ClassRule; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + HttpLoggerStressTest.class +}) +public class TestSuiteStressTests { + @ClassRule + public static final TestRuleSplunkContainer TEST_CONTAINER = new TestRuleSplunkContainer(); +} diff --git a/src/test/resources/logback.xml b/src/test/resources/logback.xml index 505d338c..0c1e9f61 100644 --- a/src/test/resources/logback.xml +++ b/src/test/resources/logback.xml @@ -57,13 +57,11 @@ under the License. 11111111-2222-3333-4444-555555555555 splunktest battlecat - text + json HttpEventCollectorUnitTestMiddleware 5000 2000 - - %msg - +