From 145226548416880edd0d22adb5fa06cd17c96a99 Mon Sep 17 00:00:00 2001
From: Louis Leseur <louis.leseur@be-ys.cloud>
Date: Thu, 13 Apr 2023 11:17:19 +0200
Subject: [PATCH] Add option to not replace "." with "_" in field names

Elasticsearch does not allow "." characters in field names since version 2.0.
Support has been restored since version 5.0.
For compatibility, graylog replaces "." with "_".
However, when Elasticsearch >= 5.0 is used, this is unnecessary.
For instance, Wazuh Indexer from the Wazuh project is forked from
Opensearch 1.3. The character replacement causes issues with Wazuh,
as the Dashboard expects dots as separator in the field name.

This adds the option `replace_dots_in_field_names` to revert this behavior and
allow the use of ".". The replacement is enabled by default for
compatibility with existing graylog configurations.

The extractor configuration in the web interface has been modified to show
a warning when the user inputs a "." in the "Key separator" field.

Closes: https://github.com/Graylog2/graylog2-server/issues/4583
Closes: https://github.com/Graylog2/graylog2-server/issues/6588
Closes: https://github.com/Graylog2/graylog2-server/issues/13043
Closes: https://github.com/Graylog2/graylog2-server/issues/14901
Bug: https://github.com/elastic/elasticsearch/issues/19443
---
 .../src/main/java/org/graylog2/plugin/Message.java       | 9 +++++++--
 .../JSONExtractorConfiguration.jsx                       | 8 ++++++++
 misc/graylog.conf                                        | 7 +++++++
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/Message.java b/graylog2-server/src/main/java/org/graylog2/plugin/Message.java
index 9a12cc6426aa5..60bab2e01b8c1 100644
--- a/graylog2-server/src/main/java/org/graylog2/plugin/Message.java
+++ b/graylog2-server/src/main/java/org/graylog2/plugin/Message.java
@@ -19,6 +19,7 @@
 import com.codahale.metrics.Meter;
 import com.eaio.uuid.UUID;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.joschi.jadconfig.Parameter;
 import com.google.common.base.Function;
 import com.google.common.base.Joiner;
 import com.google.common.base.Objects;
@@ -299,6 +300,9 @@ public class Message implements Messages, Indexable {
 
     private static final IdentityHashMap<Class<?>, Integer> classSizes = Maps.newIdentityHashMap();
 
+    @Parameter(value = "replace_dots_in_field_names")
+    private boolean replaceDotsInFieldNames = true;
+
     static {
         classSizes.put(byte.class, 1);
         classSizes.put(Byte.class, 1);
@@ -402,9 +406,10 @@ public Map<String, Object> toElasticSearchObject(ObjectMapper objectMapper, @Non
             }
 
             final Object value = entry.getValue();
-            // Elasticsearch does not allow "." characters in keys since version 2.0.
+            // Elasticsearch does not allow "." characters in keys from versions 2.0 to 5.0 (excluded).
             // See: https://www.elastic.co/guide/en/elasticsearch/reference/2.0/breaking_20_mapping_changes.html#_field_names_may_not_contain_dots
-            if (key.contains(".")) {
+            // See: https://www.elastic.co/guide/en/elasticsearch/reference/5.0/release-notes-5.0.0.html#enhancement-5.0.0
+            if (key.contains(".") && replaceDotsInFieldNames) {
                 final String newKey = key.replace('.', KEY_REPLACEMENT_CHAR);
 
                 // If the message already contains the transformed key, we skip the field and emit a warning.
diff --git a/graylog2-web-interface/src/components/extractors/extractors_configuration/JSONExtractorConfiguration.jsx b/graylog2-web-interface/src/components/extractors/extractors_configuration/JSONExtractorConfiguration.jsx
index 85299adda73ca..403e3e72f7be2 100644
--- a/graylog2-web-interface/src/components/extractors/extractors_configuration/JSONExtractorConfiguration.jsx
+++ b/graylog2-web-interface/src/components/extractors/extractors_configuration/JSONExtractorConfiguration.jsx
@@ -100,6 +100,13 @@ const JSONExtractorConfiguration = createReactClass({
     return this.state.trying || !this.props.exampleMessage;
   },
 
+  _keySeparatorError(separator) {
+    if (separator.includes("."))
+      return "Warning: Elasticsearch does not allow '.' in field names from version 2.0 up to 5.0 (excluded)"
+    else
+      return null
+  },
+
   render() {
     return (
       <div>
@@ -129,6 +136,7 @@ const JSONExtractorConfiguration = createReactClass({
                defaultValue={this.state.configuration.key_separator}
                required
                onChange={this._onChange('key_separator')}
+               error={this._keySeparatorError(this.state.configuration.key_separator)}
                help={<span>What string to use to concatenate different keys of a nested JSON object (only used if <em>not</em> flattened).</span>} />
 
         <Input type="text"
diff --git a/misc/graylog.conf b/misc/graylog.conf
index 2a2aa7c45bf79..259e4c3019e6f 100644
--- a/misc/graylog.conf
+++ b/misc/graylog.conf
@@ -406,6 +406,13 @@ allow_leading_wildcard_searches = false
 # should only be enabled after making sure your Elasticsearch cluster has enough memory.
 allow_highlighting = false
 
+# Replace "." characters in keys with "_". Dots are not allowed by Elasticsearch version 2.0 up to 5.0 (excluded).
+# Support have been restored in Elasticsearch 5.0.
+# See: https://www.elastic.co/guide/en/elasticsearch/reference/2.0/breaking_20_mapping_changes.html#_field_names_may_not_contain_dots
+# See: https://www.elastic.co/guide/en/elasticsearch/reference/5.0/release-notes-5.0.0.html#enhancement-5.0.0
+# Default: true
+#replace_dots_in_field_names = true
+
 # Global timeout for index optimization (force merge) requests.
 # Default: 1h
 #elasticsearch_index_optimization_timeout = 1h