From 3c223ceea19ebe1a9bfbfdc0edd2c0abda669f60 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 27 Jan 2020 17:05:59 +0000 Subject: [PATCH] [ML] Fix 2 digit year regex in find_file_structure (#51469) The DATE and DATESTAMP Grok patterns match 2 digit years as well as 4 digit years. The pattern determination in find_file_structure worked correctly in this case, but the regex used to create a multi-line start pattern was assuming a 4 digit year. Also, the quick rule-out patterns did not always correctly consider 2 digit years, meaning that detection was inconsistent. This change fixes both problems, and also extends the tests for DATE and DATESTAMP to check both 2 and 4 digit years. --- .../TimestampFormatFinder.java | 17 +- .../TimestampFormatFinderTests.java | 190 ++++++++++++++++-- 2 files changed, 184 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java index fef24b2ac73a4..f02802b47abab 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java @@ -159,13 +159,18 @@ public final class TimestampFormatFinder { "%{MONTH} +%{MONTHDAY} %{YEAR} %{HOUR}:%{MINUTE}:(?:[0-5][0-9]|60)\\b", "CISCOTIMESTAMP", Arrays.asList(" 11 1111 11 11 11", " 1 1111 11 11 11"), 1, 0), new CandidateTimestampFormat(CandidateTimestampFormat::indeterminateDayMonthFormatFromExample, - "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "\\b%{DATESTAMP}\\b", "DATESTAMP", - // In DATESTAMP the month may be 1 or 2 digits, but the day must be 2 - Arrays.asList("11 11 1111 11 11 11", "1 11 1111 11 11 11", "11 1 1111 11 11 11"), 0, 10), + "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "\\b%{DATESTAMP}\\b", "DATESTAMP", + // In DATESTAMP the month may be 1 or 2 digits, the year 2 or 4, but the day must be 2 + // Also note the Grok pattern search space is set to start one character before a quick rule-out + // match because we don't want 11 11 11 matching into 1111 11 11 with this pattern + Arrays.asList("11 11 1111 11 11 11", "1 11 1111 11 11 11", "11 1 1111 11 11 11", "11 11 11 11 11 11", "1 11 11 11 11 11", + "11 1 11 11 11 11"), 1, 10), new CandidateTimestampFormat(CandidateTimestampFormat::indeterminateDayMonthFormatFromExample, - "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}\\b", "\\b%{DATE}\\b", "DATE", - // In DATE the month may be 1 or 2 digits, but the day must be 2 - Arrays.asList("11 11 1111", "11 1 1111", "1 11 1111"), 0, 0), + "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}\\b", "\\b%{DATE}\\b", "DATE", + // In DATE the month may be 1 or 2 digits, the year 2 or 4, but the day must be 2 + // Also note the Grok pattern search space is set to start one character before a quick rule-out + // match because we don't want 11 11 11 matching into 1111 11 11 with this pattern + Arrays.asList("11 11 1111", "11 1 1111", "1 11 1111", "11 11 11", "11 1 11", "1 11 11"), 1, 0), UNIX_MS_CANDIDATE_FORMAT, UNIX_CANDIDATE_FORMAT, TAI64N_CANDIDATE_FORMAT, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinderTests.java index a0327183a16bc..c9b04e0cce722 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinderTests.java @@ -237,6 +237,14 @@ public void testParseIndeterminateDateNumbers() { assertEquals(5, indeterminateDateNumbers[0]); assertEquals(15, indeterminateDateNumbers[1]); + // US with padding, 2 digit year + indeterminateDateNumbers = + TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("05/15/18 16:14:56", + Collections.singletonList("??/??/yy HH:mm:ss")); + assertEquals(2, indeterminateDateNumbers.length); + assertEquals(5, indeterminateDateNumbers[0]); + assertEquals(15, indeterminateDateNumbers[1]); + // US without padding indeterminateDateNumbers = TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("5/15/2018 16:14:56", @@ -245,6 +253,14 @@ public void testParseIndeterminateDateNumbers() { assertEquals(5, indeterminateDateNumbers[0]); assertEquals(15, indeterminateDateNumbers[1]); + // US without padding, 2 digit year + indeterminateDateNumbers = + TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("5/15/18 16:14:56", + Collections.singletonList("?/?/yy HH:mm:ss")); + assertEquals(2, indeterminateDateNumbers.length); + assertEquals(5, indeterminateDateNumbers[0]); + assertEquals(15, indeterminateDateNumbers[1]); + // EU with padding indeterminateDateNumbers = TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("15/05/2018 16:14:56", @@ -253,6 +269,14 @@ public void testParseIndeterminateDateNumbers() { assertEquals(15, indeterminateDateNumbers[0]); assertEquals(5, indeterminateDateNumbers[1]); + // EU with padding, 2 digit year + indeterminateDateNumbers = + TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("15/05/18 16:14:56", + Collections.singletonList("??/??/yy HH:mm:ss")); + assertEquals(2, indeterminateDateNumbers.length); + assertEquals(15, indeterminateDateNumbers[0]); + assertEquals(5, indeterminateDateNumbers[1]); + // EU without padding indeterminateDateNumbers = TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("15/5/2018 16:14:56", @@ -260,6 +284,14 @@ public void testParseIndeterminateDateNumbers() { assertEquals(2, indeterminateDateNumbers.length); assertEquals(15, indeterminateDateNumbers[0]); assertEquals(5, indeterminateDateNumbers[1]); + + // EU without padding, 2 digit year + indeterminateDateNumbers = + TimestampFormatFinder.TimestampMatch.parseIndeterminateDateNumbers("15/5/18 16:14:56", + Collections.singletonList("?/?/yy HH:mm:ss")); + assertEquals(2, indeterminateDateNumbers.length); + assertEquals(15, indeterminateDateNumbers[0]); + assertEquals(5, indeterminateDateNumbers[1]); } public void testDeterminiseJavaTimestampFormat() { @@ -293,6 +325,14 @@ public void testGuessIsDayFirstFromFormats() { timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/5/18 16:14:56"); + timestampFormatFinder.addSample("06/6/18 17:14:56"); + timestampFormatFinder.addSample("07/7/18 18:14:56"); + + assertTrue(timestampFormatFinder.guessIsDayFirstFromFormats(timestampFormatFinder.getRawJavaTimestampFormats())); + + timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("5/05/2018 16:14:56"); timestampFormatFinder.addSample("6/06/2018 17:14:56"); timestampFormatFinder.addSample("7/07/2018 18:14:56"); @@ -302,12 +342,28 @@ public void testGuessIsDayFirstFromFormats() { timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("5/05/18 16:14:56"); + timestampFormatFinder.addSample("6/06/18 17:14:56"); + timestampFormatFinder.addSample("7/07/18 18:14:56"); + + assertFalse(timestampFormatFinder.guessIsDayFirstFromFormats(timestampFormatFinder.getRawJavaTimestampFormats())); + + timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("5/05/2018 16:14:56"); timestampFormatFinder.addSample("06/6/2018 17:14:56"); timestampFormatFinder.addSample("7/07/2018 18:14:56"); // Inconsistent so no decision assertNull(timestampFormatFinder.guessIsDayFirstFromFormats(timestampFormatFinder.getRawJavaTimestampFormats())); + + timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); + + timestampFormatFinder.addSample("5/05/18 16:14:56"); + timestampFormatFinder.addSample("06/6/18 17:14:56"); + timestampFormatFinder.addSample("7/07/18 18:14:56"); + + assertNull(timestampFormatFinder.guessIsDayFirstFromFormats(timestampFormatFinder.getRawJavaTimestampFormats())); } public void testGuessIsDayFirstFromMatchesSingleFormat() { @@ -323,6 +379,15 @@ public void testGuessIsDayFirstFromMatchesSingleFormat() { timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/18 16:14:56"); + timestampFormatFinder.addSample("05/15/18 17:14:56"); + timestampFormatFinder.addSample("05/25/18 18:14:56"); + + assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null)); + assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null)); + + timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("15/05/2018 17:14:56"); timestampFormatFinder.addSample("25/05/2018 18:14:56"); @@ -332,6 +397,15 @@ public void testGuessIsDayFirstFromMatchesSingleFormat() { timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/18 16:14:56"); + timestampFormatFinder.addSample("15/05/18 17:14:56"); + timestampFormatFinder.addSample("25/05/18 18:14:56"); + + assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(null)); + assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(null)); + + timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("05/06/2018 17:14:56"); timestampFormatFinder.addSample("05/07/2018 18:14:56"); @@ -340,6 +414,13 @@ public void testGuessIsDayFirstFromMatchesSingleFormat() { assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null)); assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null)); + timestampFormatFinder.addSample("05/05/18 16:14:56"); + timestampFormatFinder.addSample("05/06/18 17:14:56"); + timestampFormatFinder.addSample("05/07/18 18:14:56"); + + assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null)); + assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(null)); + timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); timestampFormatFinder.addSample("05/05/2018 16:14:56"); @@ -352,6 +433,15 @@ public void testGuessIsDayFirstFromMatchesSingleFormat() { timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/18 16:14:56"); + timestampFormatFinder.addSample("06/05/18 17:14:56"); + timestampFormatFinder.addSample("07/05/18 18:14:56"); + + assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(null)); + assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(null)); + + timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("06/06/2018 17:14:56"); timestampFormatFinder.addSample("07/07/2018 18:14:56"); @@ -359,6 +449,15 @@ public void testGuessIsDayFirstFromMatchesSingleFormat() { // Insufficient evidence to decide assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(null)); assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(null)); + + timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, true, NOOP_TIMEOUT_CHECKER); + + timestampFormatFinder.addSample("05/05/18 16:14:56"); + timestampFormatFinder.addSample("06/06/18 17:14:56"); + timestampFormatFinder.addSample("07/07/18 18:14:56"); + + assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(null)); + assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(null)); } public void testGuessIsDayFirstFromMatchesMultipleFormats() { @@ -366,9 +465,11 @@ public void testGuessIsDayFirstFromMatchesMultipleFormats() { // Similar to the test above, but with the possibility that the secondary // ISO8601 formats cause confusion - this test proves that they don't + // DATESTAMP supports both 2 and 4 digit years, so each test is repeated for both lengths TimestampFormatFinder.TimestampFormat expectedPrimaryFormat = new TimestampFormatFinder.TimestampFormat(Collections.singletonList("??/??/yyyy HH:mm:ss"), - Pattern.compile("\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b"), "DATESTAMP", Collections.emptyMap(), ""); + Pattern.compile("\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b"), "DATESTAMP", + Collections.emptyMap(), ""); TimestampFormatFinder timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); @@ -383,6 +484,17 @@ public void testGuessIsDayFirstFromMatchesMultipleFormats() { timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/18 16:14:56"); + timestampFormatFinder.addSample("2018-05-15T17:14:56"); + timestampFormatFinder.addSample("05/15/18 17:14:56"); + timestampFormatFinder.addSample("2018-05-25T18:14:56"); + timestampFormatFinder.addSample("05/25/18 18:14:56"); + + assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); + assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); + + timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("2018-05-15T17:14:56"); timestampFormatFinder.addSample("15/05/2018 17:14:56"); @@ -394,6 +506,17 @@ public void testGuessIsDayFirstFromMatchesMultipleFormats() { timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/18 16:14:56"); + timestampFormatFinder.addSample("2018-05-15T17:14:56"); + timestampFormatFinder.addSample("15/05/18 17:14:56"); + timestampFormatFinder.addSample("2018-05-25T18:14:56"); + timestampFormatFinder.addSample("25/05/18 18:14:56"); + + assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); + assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); + + timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("2018-05-06T17:14:56"); timestampFormatFinder.addSample("05/06/2018 17:14:56"); @@ -406,6 +529,17 @@ public void testGuessIsDayFirstFromMatchesMultipleFormats() { timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/18 16:14:56"); + timestampFormatFinder.addSample("2018-05-06T17:14:56"); + timestampFormatFinder.addSample("05/06/18 17:14:56"); + timestampFormatFinder.addSample("2018-05-07T18:14:56"); + timestampFormatFinder.addSample("05/07/18 18:14:56"); + + assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); + assertFalse(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); + + timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("2018-05-06T17:14:56"); timestampFormatFinder.addSample("06/05/2018 17:14:56"); @@ -418,6 +552,17 @@ public void testGuessIsDayFirstFromMatchesMultipleFormats() { timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/18 16:14:56"); + timestampFormatFinder.addSample("2018-05-06T17:14:56"); + timestampFormatFinder.addSample("06/05/18 17:14:56"); + timestampFormatFinder.addSample("2018-05-07T18:14:56"); + timestampFormatFinder.addSample("07/05/18 18:14:56"); + + assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); + assertTrue(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); + + timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); + timestampFormatFinder.addSample("05/05/2018 16:14:56"); timestampFormatFinder.addSample("2018-06-06T17:14:56"); timestampFormatFinder.addSample("06/06/2018 17:14:56"); @@ -427,6 +572,17 @@ public void testGuessIsDayFirstFromMatchesMultipleFormats() { // Insufficient evidence to decide assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); + + timestampFormatFinder = new TimestampFormatFinder(explanation, true, true, false, NOOP_TIMEOUT_CHECKER); + + timestampFormatFinder.addSample("05/05/18 16:14:56"); + timestampFormatFinder.addSample("2018-06-06T17:14:56"); + timestampFormatFinder.addSample("06/06/18 17:14:56"); + timestampFormatFinder.addSample("2018-07-07T18:14:56"); + timestampFormatFinder.addSample("07/07/18 18:14:56"); + + assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); + assertNull(timestampFormatFinder.guessIsDayFirstFromMatches(expectedPrimaryFormat)); } public void testGuessIsDayFirstFromLocale() { @@ -674,31 +830,31 @@ public void testFindFormatGivenOnlyKnownTimestampFormat() { Arrays.asList("MMM dd yyyy HH:mm:ss", "MMM d yyyy HH:mm:ss", "MMM d yyyy HH:mm:ss"), 1526400896000L); validateTimestampMatch("05/15/2018 17:14:56,374", "DATESTAMP", - "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM/dd/yyyy HH:mm:ss,SSS", 1526400896374L); + "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM/dd/yyyy HH:mm:ss,SSS", 1526400896374L); validateTimestampMatch("05-15-2018-17:14:56.374", "DATESTAMP", - "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM-dd-yyyy-HH:mm:ss.SSS", 1526400896374L); + "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM-dd-yyyy-HH:mm:ss.SSS", 1526400896374L); validateTimestampMatch("15/05/2018 17:14:56.374", "DATESTAMP", - "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd/MM/yyyy HH:mm:ss.SSS", 1526400896374L); + "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd/MM/yyyy HH:mm:ss.SSS", 1526400896374L); validateTimestampMatch("15-05-2018-17:14:56,374", "DATESTAMP", - "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd-MM-yyyy-HH:mm:ss,SSS", 1526400896374L); + "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd-MM-yyyy-HH:mm:ss,SSS", 1526400896374L); validateTimestampMatch("15.05.2018 17:14:56.374", "DATESTAMP", - "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd.MM.yyyy HH:mm:ss.SSS", 1526400896374L); + "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd.MM.yyyy HH:mm:ss.SSS", 1526400896374L); validateTimestampMatch("05/15/2018 17:14:56", "DATESTAMP", - "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM/dd/yyyy HH:mm:ss", 1526400896000L); + "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM/dd/yyyy HH:mm:ss", 1526400896000L); validateTimestampMatch("05-15-2018-17:14:56", "DATESTAMP", - "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM-dd-yyyy-HH:mm:ss", 1526400896000L); + "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM-dd-yyyy-HH:mm:ss", 1526400896000L); validateTimestampMatch("15/05/2018 17:14:56", "DATESTAMP", - "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd/MM/yyyy HH:mm:ss", 1526400896000L); + "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd/MM/yyyy HH:mm:ss", 1526400896000L); validateTimestampMatch("15-05-2018-17:14:56", "DATESTAMP", - "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd-MM-yyyy-HH:mm:ss", 1526400896000L); + "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd-MM-yyyy-HH:mm:ss", 1526400896000L); validateTimestampMatch("15.05.2018 17:14:56", "DATESTAMP", - "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd.MM.yyyy HH:mm:ss", 1526400896000L); + "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "dd.MM.yyyy HH:mm:ss", 1526400896000L); - validateTimestampMatch("05/15/2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}\\b", "MM/dd/yyyy", 1526338800000L); - validateTimestampMatch("05-15-2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}\\b", "MM-dd-yyyy", 1526338800000L); - validateTimestampMatch("15/05/2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}\\b", "dd/MM/yyyy", 1526338800000L); - validateTimestampMatch("15-05-2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}\\b", "dd-MM-yyyy", 1526338800000L); - validateTimestampMatch("15.05.2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}\\b", "dd.MM.yyyy", 1526338800000L); + validateTimestampMatch("05/15/2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}\\b", "MM/dd/yyyy", 1526338800000L); + validateTimestampMatch("05-15-2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}\\b", "MM-dd-yyyy", 1526338800000L); + validateTimestampMatch("15/05/2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}\\b", "dd/MM/yyyy", 1526338800000L); + validateTimestampMatch("15-05-2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}\\b", "dd-MM-yyyy", 1526338800000L); + validateTimestampMatch("15.05.2018", "DATE", "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}\\b", "dd.MM.yyyy", 1526338800000L); } public void testFindFormatGivenOnlySystemDate() { @@ -811,7 +967,7 @@ public void testFindFormatGivenRealLogMessages() { validateFindInFullMessage("10-28-2016 16:22:47.636 +0200 ERROR Network - " + "Error encountered for connection from src=192.168.0.1:12345. Local side shutting down", "", "DATESTAMP", - "\\b\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM-dd-yyyy HH:mm:ss.SSS"); + "\\b\\d{1,2}[/.-]\\d{1,2}[/.-](?:\\d{2}){1,2}[- ]\\d{2}:\\d{2}:\\d{2}\\b", "MM-dd-yyyy HH:mm:ss.SSS"); validateFindInFullMessage("2018-01-06 19:22:20.106822|INFO |VirtualServer |1 |client " + " 'User1'(id:2) was added to channelgroup 'Channel Admin'(id:5) by client 'User1'(id:2) in channel '3er Instanz'(id:2)", "",