Extend allowed characters for grok field names (#21745) (#31653)

elastic · Jun 29, 2018 · b7b413e · b7b413e
1 parent 5925611
commit b7b413e
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 5 deletions.
diff --git a/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java b/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java
@@ -52,7 +52,7 @@ public final class Grok {
             "%\\{" +
             "(?<name>" +
             "(?<pattern>[A-z0-9]+)" +
-            "(?::(?<subname>[A-z0-9_:.-]+))?" +
+            "(?::(?<subname>[[:alnum:]@\\[\\]_:.-]+))?" +
             ")" +
             "(?:=(?<definition>" +
             "(?:" +
@@ -81,11 +81,11 @@ public final class Grok {
     public Grok(Map<String, String> patternBank, String grokPattern) {
         this(patternBank, grokPattern, true, ThreadWatchdog.noop());
     }
-    
+
     public Grok(Map<String, String> patternBank, String grokPattern, ThreadWatchdog threadWatchdog) {
         this(patternBank, grokPattern, true, threadWatchdog);
     }
-    
+
     Grok(Map<String, String> patternBank, String grokPattern, boolean namedCaptures) {
         this(patternBank, grokPattern, namedCaptures, ThreadWatchdog.noop());
     }

diff --git a/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java b/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java
@@ -412,10 +412,10 @@ public void testMultipleNamedCapturesWithSameName() {
         expected.put("num", "1");
         assertThat(grok.captures("12"), equalTo(expected));
     }
-    
+
     public void testExponentialExpressions() {
         AtomicBoolean run = new AtomicBoolean(true); // to avoid a lingering thread when test has completed
-        
+
         String grokPattern = "Bonsuche mit folgender Anfrage: Belegart->\\[%{WORD:param2},(?<param5>(\\s*%{NOTSPACE})*)\\] " +
             "Zustand->ABGESCHLOSSEN Kassennummer->%{WORD:param9} Bonnummer->%{WORD:param10} Datum->%{DATESTAMP_OTHER:param11}";
         String logLine = "Bonsuche mit folgender Anfrage: Belegart->[EINGESCHRAENKTER_VERKAUF, VERKAUF, NACHERFASSUNG] " +
@@ -439,4 +439,50 @@ public void testExponentialExpressions() {
         run.set(false);
         assertThat(e.getMessage(), equalTo("grok pattern matching was interrupted after [200] ms"));
     }
+
+    public void testAtInFieldName() {
+        assertGrokedField("@metadata");
+    }
+
+    public void assertNonAsciiLetterInFieldName() {
+        assertGrokedField("metädata");
+    }
+
+    public void assertSquareBracketInFieldName() {
+        assertGrokedField("metadat[a]");
+        assertGrokedField("metad[a]ta");
+        assertGrokedField("[m]etadata");
+    }
+
+    public void testUnderscoreInFieldName() {
+        assertGrokedField("meta_data");
+    }
+
+    public void testDotInFieldName() {
+        assertGrokedField("meta.data");
+    }
+
+    public void testMinusInFieldName() {
+        assertGrokedField("meta-data");
+    }
+
+    public void testAlphanumericFieldName() {
+        assertGrokedField(randomAlphaOfLengthBetween(1, 5));
+        assertGrokedField(randomAlphaOfLengthBetween(1, 5) + randomIntBetween(0, 100));
+        assertGrokedField(randomIntBetween(0, 100) + randomAlphaOfLengthBetween(1, 5));
+        assertGrokedField(String.valueOf(randomIntBetween(0, 100)));
+    }
+
+    public void testUnsupportedBracketsInFieldName() {
+        Grok grok = new Grok(basePatterns, "%{WORD:unsuppo(r)ted}");
+        Map<String, Object> matches = grok.captures("line");
+        assertNull(matches);
+    }
+
+    private void assertGrokedField(String fieldName) {
+        String line = "foo";
+        Grok grok = new Grok(basePatterns, "%{WORD:" + fieldName + "}");
+        Map<String, Object> matches = grok.captures(line);
+        assertEquals(line, matches.get(fieldName));
+    }
 }