Elasticsearch 8 compatibility - set default mapping type to null (#81)

log4j2-elasticsearch-jest/README.md

@@ -1,5 +1,5 @@
 # log4j2-elasticsearch-jest
-This log4j2 appender plugin uses Jest HTTP client to push logs in batches to Elasticsearch 2.x, 5.x and 6.x clusters. By default, FasterXML is used generate output via [JacksonJsonLayout](https://github.com/rfoltyns/log4j2-elasticsearch/blob/master/log4j2-elasticsearch-core/src/main/java/org/appenders/log4j2/elasticsearch/JacksonJsonLayout.java).
+This log4j2 appender plugin uses Jest HTTP client to push logs in batches to Elasticsearch 2.x, 5.x, 6.x, 7.x and 8.x clusters. By default, FasterXML is used generate output via [JacksonJsonLayout](https://github.com/rfoltyns/log4j2-elasticsearch/blob/master/log4j2-elasticsearch-core/src/main/java/org/appenders/log4j2/elasticsearch/JacksonJsonLayout.java).
 
 ## Maven
 
@@ -35,7 +35,7 @@ Add this snippet to `log4j2.xml` configuration:
         <JacksonJsonLayout />
         <AsyncBatchDelivery batchSize="1000" deliveryInterval="5000" >
             <IndexTemplate name="log4j2" path="classpath:indexTemplate.json" />
-            <JestHttp serverUris="http://localhost:9200" mappingType="<see mappingType description>"/>
+            <JestHttp serverUris="http://localhost:9200" />
         </AsyncBatchDelivery>
     </Elasticsearch>
 </Appenders>
@@ -50,7 +50,7 @@ Add this snippet to `log4j2.xml` configuration:
 | defaultMaxTotalConnectionPerRoute | Attribute | no       | 4                           | Number of connections available per Apache CPool.                                                                                                                                            |
 | discoveryEnabled                  | Attribute | no       | false                       | If `true`, `io.searchbox.client.config.discovery.NodeChecker` will use `serverUris` to auto-discover Elasticsearch nodes. Otherwise, `serverUris` will be the final list of available nodes. |
 | ioThreadCount                     | Attribute | no       | No. of available processors | Number of `I/O Dispatcher` threads started by Apache HC `IOReactor`                                                                                                                          |
-| mappingType                       | Attribute | no       | `index`                     | Name of index mapping type to use in ES cluster. Use `_doc` for Elasticsearch 7.x.                                                                                                           |
+| mappingType                       | Attribute | no       | `null` since 1.6            | Name of index mapping type to use. Applicable to Elasticsearch 7.x and older. See [removal of types](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html).    |
 | auth                              | Element   | no       | None                        | Security config. [XPackAuth](#pem-cert-config)                                                                                                                                               |
 
 ### Buffered HTTP
@@ -73,7 +73,7 @@ Example:
         </JacksonJsonLayout>
         <AsyncBatchDelivery batchSize="1000" deliveryInterval="5000" >
             <IndexTemplate name="log4j2" path="classpath:indexTemplate.json" />
-            <JestBufferedHttp serverUris="http://localhost:9200" mappingType="<see mappingType description>">
+            <JestBufferedHttp serverUris="http://localhost:9200">
                 <PooledItemSourceFactory itemSizeInBytes="1024000" initialPoolSize="4" />
             </JestBufferedHttp>
         </AsyncBatchDelivery>
@@ -82,7 +82,7 @@ Example:
 ```
 
 ### Programmatic config
-See [programmatc config example](https://github.com/rfoltyns/log4j2-elasticsearch/blob/master/log4j2-elasticsearch-jest/src/test/java/org/appenders/log4j2/elasticsearch/jest/smoke/SmokeTest.java).
+See [programmatic config example](https://github.com/rfoltyns/log4j2-elasticsearch/blob/master/log4j2-elasticsearch-jest/src/test/java/org/appenders/log4j2/elasticsearch/jest/smoke/SmokeTest.java).
 
 ### Delivery frequency
 See [delivery frequency](../log4j2-elasticsearch-core#delivery-frequency)
@@ -97,6 +97,9 @@ See [failover options](../log4j2-elasticsearch-core#failover)
 See [index name](../log4j2-elasticsearch-core#index-name) or [index rollover](../log4j2-elasticsearch-core#index-rollover)
 
 ### Index template
+
+Since 1.6, this module is compatible with Elasticsearch 8.x by default. Use `apiVersion` for older clusters.
+
 See [index template docs](../log4j2-elasticsearch-core#index-template)
 
 ### SSL/TLS
@@ -130,12 +133,12 @@ Since 1.2, HTTPS can be configured using `XPackAuth` tag:
 
 ### Compatibility matrix
 
-| Feature/Version  | 2.x        | 5.x        | 6.x        | 7.x        |
-|------------------|------------|------------|------------|------------|
-| IndexTemplate    | Yes        | Yes        | Yes        | Yes        |
-| BasicCredentials | Yes        | Yes        | Yes        | Yes        |
-| JKS              | Yes        | Not tested | Not tested | Not tested |
-| PEM              | Not tested | Yes        | Yes        | Yes        |
+| Feature/Version  | 2.x        | 5.x        | 6.x        | 7.x        | 8.x        |
+|------------------|------------|------------|------------|------------|------------|
+| IndexTemplate    | Yes        | Yes        | Yes        | Yes        | Yes        |
+| BasicCredentials | Yes        | Yes        | Yes        | Yes        | Yes        |
+| JKS              | Yes        | Not tested | Not tested | Not tested | Not tested |
+| PEM              | Not tested | Yes        | Yes        | Yes        | Yes        |
 
 ## Pluggable JCTools
 

log4j2-elasticsearch-jest/pom.xml

@@ -8,7 +8,7 @@
     <modelVersion>4.0.0</modelVersion>
     <artifactId>log4j2-elasticsearch-jest</artifactId>
     <name>Log4j2 Elasticsearch Jest HTTP client</name>
-    <description>Log4j2 Appender plugin pushing logs in batches to  Elasticsearch (2.x/5.x/6.x) clusters</description>
+    <description>Log4j2 Appender plugin pushing logs in batches to Elasticsearch 2.x/5.x/6.x/7.x/8.x clusters</description>
 
     <dependencies>
         <dependency>

log4j2-elasticsearch-jest/src/main/java/org/appenders/log4j2/elasticsearch/jest/BufferedBulkOperations.java

@@ -44,12 +44,13 @@
  */
 public class BufferedBulkOperations implements BatchOperations<Bulk> {
 
-    public static final String DEFAULT_MAPPING_TYPE = "_doc";
+    public static final String DEFAULT_MAPPING_TYPE = null;
 
     private final PooledItemSourceFactory pooledItemSourceFactory;
 
     /**
      * "_doc" since 1.5
+     * {@code null} since 1.6
      */
     private final String mappingType;
     private final JacksonMixIn[] mixIns;

log4j2-elasticsearch-jest/src/main/java/org/appenders/log4j2/elasticsearch/jest/JestBulkOperations.java

@@ -31,10 +31,10 @@
 
 public class JestBulkOperations implements BatchOperations<Bulk> {
 
-    public static final String DEFAULT_MAPPING_TYPE = "_doc";
+    public static final String DEFAULT_MAPPING_TYPE = null;
 
     /**
-     * "_doc" since 1.5
+     * {@code null} since 1.6
      */
     private final String mappingType;
 

log4j2-elasticsearch-jest/src/test/java/org/appenders/log4j2/elasticsearch/jest/BufferedBulkOperationsTest.java

@@ -72,24 +72,6 @@ public void throwsOnStringSource() {
 
     }
 
-    @Test
-    public void defaultBufferedBulkOperationsSetsDefaultMappingType() {
-
-        // given
-        PooledItemSourceFactory bufferedSourceFactory = PooledItemSourceFactoryTest.createDefaultTestSourceFactoryConfig().build();
-        BufferedBulkOperations bulkOperations = new BufferedBulkOperations(bufferedSourceFactory);
-
-        ItemSource itemSource = mock(ItemSource.class);
-        BufferedIndex item = (BufferedIndex) bulkOperations.createBatchItem("testIndex", itemSource);
-
-        // when
-        String type = item.getType();
-
-        // then
-        assertEquals(DEFAULT_MAPPING_TYPE, type);
-
-    }
-
     @Test
     public void mappingTypeCanBeSet() {
 
@@ -110,7 +92,7 @@ public void mappingTypeCanBeSet() {
     }
 
     @Test
-    public void usesDefaultMappingTypeIfNotProvided() {
+    public void defaultMappingTypeIsNull() {
 
         // given
         PooledItemSourceFactory itemSourceFactory = PooledItemSourceFactoryTest.createDefaultTestSourceFactoryConfig().build();
@@ -123,7 +105,7 @@ public void usesDefaultMappingTypeIfNotProvided() {
         String type = item.getType();
 
         // then
-        assertEquals("_doc", type);
+        assertNull(type);
 
     }
 
@@ -176,8 +158,9 @@ public void defaultWriterCanSerializeBufferedBulk() throws IOException {
 
         // given
         PooledItemSourceFactory bufferedSourceFactory = PooledItemSourceFactoryTest.createDefaultTestSourceFactoryConfig().build();
+        final String expectedMappingType = UUID.randomUUID().toString();
 
-        BufferedBulkOperations bufferedBulkOperations = new BufferedBulkOperations(bufferedSourceFactory);
+        BufferedBulkOperations bufferedBulkOperations = new BufferedBulkOperations(bufferedSourceFactory, new JacksonMixIn[]{}, expectedMappingType);
 
         JacksonJsonLayout layout = createDefaultTestJacksonJsonLayout(bufferedSourceFactory);
 
@@ -205,7 +188,7 @@ public void defaultWriterCanSerializeBufferedBulk() throws IOException {
                 .addMixIn(TestIndex.class, BulkableActionMixIn.class)
                 .readValue(scanner.nextLine(), TestIndex.class);
         assertEquals(indexName, deserializedAction.index);
-        assertNotNull(deserializedAction.type);
+        assertEquals(expectedMappingType, deserializedAction.type);
 
         TestLogEvent deserializedDocument = new ObjectMapper().readValue(scanner.nextLine(), TestLogEvent.class);
         assertEquals(timeMillis, deserializedDocument.timeMillis);

log4j2-elasticsearch-jest/src/test/java/org/appenders/log4j2/elasticsearch/jest/JestBulkOperationsTest.java

@@ -38,6 +38,7 @@
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
@@ -106,7 +107,7 @@ public void bulkContainsAddedSourceItem() {
     }
 
     @Test
-    public void usesDefaultMappingTypeIfNotProvided() {
+    public void defaultMappingTypeIsNull() {
 
         // given
         BatchOperations<Bulk> bulkOperations = new JestBulkOperations();
@@ -119,7 +120,7 @@ public void usesDefaultMappingTypeIfNotProvided() {
         String type = item.getType();
 
         // then
-        assertEquals("_doc", type);
+        assertNull(type);
 
     }
 

log4j2-elasticsearch-jest/src/test/java/org/appenders/log4j2/elasticsearch/jest/PutIndexTemplateTest.java

@@ -172,7 +172,7 @@ public void createsGenericJestRequestIfNotComposable() {
     }
 
     @Test
-    public void defaultGenericJestRequestNotComposable() {
+    public void defaultGenericJestRequestIsComposable() {
 
         // given
         PutIndexTemplate setupStep = new PutIndexTemplate(TEST_TEMPLATE_NAME, TEST_SOURCE);
@@ -182,7 +182,7 @@ public void defaultGenericJestRequestNotComposable() {
 
         // then
         assertEquals("PUT", request.getRestMethodName());
-        assertEquals("_template/" + TEST_TEMPLATE_NAME, request.buildURI());
+        assertEquals("_index_template/" + TEST_TEMPLATE_NAME, request.buildURI());
 
     }
 

log4j2-elasticsearch-jest/src/test/java/org/appenders/log4j2/elasticsearch/jest/smoke/SmokeTest.java

@@ -269,8 +269,14 @@ private OpSource[] setupOpSources(final Version version, final String indexName,
         return result.toArray(new OpSource[0]);
     }
 
-    private String mappingType(Version version) {
-        return !version.lowerThan("7.0.0") ? "_doc" : "index";
+    private String mappingType(final Version version) {
+        if (version.lowerThan("7.0.0")) {
+            return "index";
+        }
+        if (version.lowerThan("8.0.0")) {
+            return "_doc";
+        }
+        return null;
     }
 
 }

log4j2-elasticsearch-jest/src/test/resources/log4j2.properties

@@ -25,7 +25,7 @@ appender.es.batchDelivery.indexTemplate.path = classpath:indexTemplate-7.json
 
 # with Buffered Jest HTTP client
 appender.es.batchDelivery.objectFactory.type = JestBufferedHttp
-appender.es.batchDelivery.objectFactory.serverUris = http://localhost:9200
+appender.es.batchDelivery.objectFactory.serverUris = https://localhost:9200
 appender.es.batchDelivery.objectFactory.connTimeout = 500
 appender.es.batchDelivery.objectFactory.readTimeout = 10000
 appender.es.batchDelivery.objectFactory.maxTotalConnection = 40
@@ -40,6 +40,20 @@ appender.es.batchDelivery.objectFactory.itemSourceFactory.monitored = true
 appender.es.batchDelivery.objectFactory.itemSourceFactory.monitorTaskInterval = 5000
 appender.es.batchDelivery.objectFactory.itemSourceFactory.resizeTimeout = 100
 
+# with Security
+appender.es.batchDelivery.objectFactory.auth.type = XPackAuth
+
+appender.es.batchDelivery.objectFactory.auth.credentials.type = BasicCredentials
+appender.es.batchDelivery.objectFactory.auth.credentials.username = admin
+appender.es.batchDelivery.objectFactory.auth.credentials.password = changeme
+
+appender.es.batchDelivery.objectFactory.auth.certInfo.type = PEM
+appender.es.batchDelivery.objectFactory.auth.certInfo.keyPath=${sys:pemCertInfo.keyPathWithPassphrase}
+appender.es.batchDelivery.objectFactory.auth.certInfo.keyPassphrase=${sys:pemCertInfo.keyPassphrase}
+appender.es.batchDelivery.objectFactory.auth.certInfo.clientCertPath=${sys:pemCertInfo.clientCertPath}
+appender.es.batchDelivery.objectFactory.auth.certInfo.caPath=${sys:pemCertInfo.caPath}
+
+
 # with JacksonJsonLayout
 appender.es.layout.type = JacksonJsonLayout