From cf06c02cf3de6d71c6130b572de15c964bd25980 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 25 Sep 2018 13:22:42 +0100 Subject: [PATCH] Adjustments to match the changes of #33900 --- .../xpack/ml/configcreator/LogConfigWriter.java | 16 ++++++++-------- .../ml/configcreator/LogConfigWriterTests.java | 10 +++++++--- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/ml/log-config-creator/src/main/java/org/elasticsearch/xpack/ml/configcreator/LogConfigWriter.java b/x-pack/plugin/ml/log-config-creator/src/main/java/org/elasticsearch/xpack/ml/configcreator/LogConfigWriter.java index 74efee2bc1422..a91b98bf1218f 100644 --- a/x-pack/plugin/ml/log-config-creator/src/main/java/org/elasticsearch/xpack/ml/configcreator/LogConfigWriter.java +++ b/x-pack/plugin/ml/log-config-creator/src/main/java/org/elasticsearch/xpack/ml/configcreator/LogConfigWriter.java @@ -570,14 +570,14 @@ private void createJsonConfigs(FileStructure structure) { String logstashFromFileDateFilter = ""; String ingestPipelineDateProcessor = ""; if (structure.getTimestampField() != null) { - logstashFromFilebeatDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getTimestampFormats(), + logstashFromFilebeatDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getJodaTimestampFormats(), structure.needClientTimezone(), true); - logstashFromFileDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getTimestampFormats(), + logstashFromFileDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getJodaTimestampFormats(), structure.needClientTimezone(), false); String jsonEscapedField = structure.getTimestampField().replaceAll("([\\\\\"])", "\\\\$1").replace("\t", "\\t"); ingestPipelineDateProcessor = String.format(Locale.ROOT, JSON_INGEST_PIPELINE_DATE_PROCESSOR_TEMPLATE, jsonEscapedField, makeIngestPipelineTimezoneSetting(structure.needClientTimezone()), - structure.getTimestampFormats().stream().collect(Collectors.joining("\", \"", "\"", "\""))); + structure.getJodaTimestampFormats().stream().collect(Collectors.joining("\", \"", "\"", "\""))); } String filebeatInputOptions = makeFilebeatInputOptions(null, null, structure.getCharset()); @@ -603,9 +603,9 @@ private void createXmlConfigs(FileStructure structure) { String logstashFromFileDateFilter = ""; if (structure.getTimestampField() != null) { String timeFieldPath = "[" + topLevelTag + "][" + structure.getTimestampField() + "]"; - logstashFromFilebeatDateFilter = makeLogstashDateFilter(timeFieldPath, structure.getTimestampFormats(), + logstashFromFilebeatDateFilter = makeLogstashDateFilter(timeFieldPath, structure.getJodaTimestampFormats(), structure.needClientTimezone(), true); - logstashFromFileDateFilter = makeLogstashDateFilter(timeFieldPath, structure.getTimestampFormats(), + logstashFromFileDateFilter = makeLogstashDateFilter(timeFieldPath, structure.getJodaTimestampFormats(), structure.needClientTimezone(), false); } @@ -626,9 +626,9 @@ private void createDelimitedConfigs(FileStructure structure) { String logstashFromFilebeatDateFilter = ""; String logstashFromFileDateFilter = ""; if (structure.getTimestampField() != null) { - logstashFromFilebeatDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getTimestampFormats(), + logstashFromFilebeatDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getJodaTimestampFormats(), structure.needClientTimezone(), true); - logstashFromFileDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getTimestampFormats(), + logstashFromFileDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getJodaTimestampFormats(), structure.needClientTimezone(), false); } @@ -660,7 +660,7 @@ private void createTextConfigs(FileStructure structure, List sampleMessa String grokPattern = structure.getGrokPattern(); String grokQuote = bestLogstashQuoteFor(grokPattern); String interimTimestampField = structure.getTimestampField(); - String dateFormatsStr = structure.getTimestampFormats().stream().collect(Collectors.joining("\", \"", "\"", "\"")); + String dateFormatsStr = structure.getJodaTimestampFormats().stream().collect(Collectors.joining("\", \"", "\"", "\"")); String filebeatInputOptions = makeFilebeatInputOptions(structure.getMultilineStartPattern(), null, structure.getCharset()); filebeatToLogstashConfig = String.format(Locale.ROOT, TEXT_FILEBEAT_TO_LOGSTASH_TEMPLATE, filebeatInputOptions, diff --git a/x-pack/plugin/ml/log-config-creator/src/test/java/org/elasticsearch/xpack/ml/configcreator/LogConfigWriterTests.java b/x-pack/plugin/ml/log-config-creator/src/test/java/org/elasticsearch/xpack/ml/configcreator/LogConfigWriterTests.java index 42e2f661ba8c6..50d245f667ae7 100644 --- a/x-pack/plugin/ml/log-config-creator/src/test/java/org/elasticsearch/xpack/ml/configcreator/LogConfigWriterTests.java +++ b/x-pack/plugin/ml/log-config-creator/src/test/java/org/elasticsearch/xpack/ml/configcreator/LogConfigWriterTests.java @@ -348,7 +348,7 @@ public void testCreateConfigsGivenElasticsearchLog() throws Exception { containsString("encoding: '" + charset.toLowerCase(Locale.ROOT) + "'")); } assertThat(logConfigWriter.getFilebeatToLogstashConfig(), - containsString("multiline.pattern: '^\\[\\b\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}'\n")); + containsString("multiline.pattern: '^\\[\\b\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2},\\d{3}'\n")); assertThat(logConfigWriter.getFilebeatToLogstashConfig(), containsString(logstashHost)); assertThat(logConfigWriter.getLogstashFromFilebeatConfig(), containsString("match => { \"message\" => \"\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:loglevel} \\]" + @@ -364,7 +364,11 @@ public void testCreateConfigsGivenElasticsearchLog() throws Exception { containsString("match => { \"message\" => \"\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:loglevel} \\]" + "\\[.*\" }\n")); assertThat(logConfigWriter.getLogstashFromFileConfig(), containsString("match => [ \"timestamp\", \"ISO8601\" ]\n")); - assertThat(logConfigWriter.getLogstashFromFileConfig(), not(containsString("timezone =>"))); + if (timezone == null) { + assertThat(logConfigWriter.getLogstashFromFileConfig(), not(containsString("timezone =>"))); + } else { + assertThat(logConfigWriter.getLogstashFromFileConfig(), containsString("timezone => \"" + timezone + "\"\n")); + } assertThat(logConfigWriter.getLogstashFromFileConfig(), containsString(elasticsearchHost)); if (charset.equals(StandardCharsets.UTF_8.name())) { assertThat(logConfigWriter.getFilebeatToIngestPipelineConfig(), not(containsString("encoding:"))); @@ -373,7 +377,7 @@ public void testCreateConfigsGivenElasticsearchLog() throws Exception { containsString("encoding: '" + charset.toLowerCase(Locale.ROOT) + "'")); } assertThat(logConfigWriter.getFilebeatToIngestPipelineConfig(), - containsString("multiline.pattern: '^\\[\\b\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}'\n")); + containsString("multiline.pattern: '^\\[\\b\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2},\\d{3}'\n")); assertThat(logConfigWriter.getFilebeatToIngestPipelineConfig(), containsString(elasticsearchHost)); assertThat(logConfigWriter.getIngestPipelineFromFilebeatConfig(), containsString("\"patterns\": [ \"\\\\[%{TIMESTAMP_ISO8601:timestamp}\\\\]\\\\[%{LOGLEVEL:loglevel} \\\\]" +