Skip to content

Commit

Permalink
Adjustments to match the changes of elastic#33900
Browse files Browse the repository at this point in the history
  • Loading branch information
@droberts195
droberts195 committed Sep 25, 2018
1 parent 1bd510a commit cf06c02
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 11 deletions.
16 changes: 8 additions & 8 deletions ...onfig-creator/src/main/java/org/elasticsearch/xpack/ml/configcreator/LogConfigWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -570,14 +570,14 @@ private void createJsonConfigs(FileStructure structure) {
String logstashFromFileDateFilter = "";
String ingestPipelineDateProcessor = "";
if (structure.getTimestampField() != null) {
logstashFromFilebeatDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getTimestampFormats(),
logstashFromFilebeatDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getJodaTimestampFormats(),
structure.needClientTimezone(), true);
logstashFromFileDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getTimestampFormats(),
logstashFromFileDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getJodaTimestampFormats(),
structure.needClientTimezone(), false);
String jsonEscapedField = structure.getTimestampField().replaceAll("([\\\\\"])", "\\\\$1").replace("\t", "\\t");
ingestPipelineDateProcessor = String.format(Locale.ROOT, JSON_INGEST_PIPELINE_DATE_PROCESSOR_TEMPLATE, jsonEscapedField,
makeIngestPipelineTimezoneSetting(structure.needClientTimezone()),
structure.getTimestampFormats().stream().collect(Collectors.joining("\", \"", "\"", "\"")));
structure.getJodaTimestampFormats().stream().collect(Collectors.joining("\", \"", "\"", "\"")));
}

String filebeatInputOptions = makeFilebeatInputOptions(null, null, structure.getCharset());
Expand All @@ -603,9 +603,9 @@ private void createXmlConfigs(FileStructure structure) {
String logstashFromFileDateFilter = "";
if (structure.getTimestampField() != null) {
String timeFieldPath = "[" + topLevelTag + "][" + structure.getTimestampField() + "]";
logstashFromFilebeatDateFilter = makeLogstashDateFilter(timeFieldPath, structure.getTimestampFormats(),
logstashFromFilebeatDateFilter = makeLogstashDateFilter(timeFieldPath, structure.getJodaTimestampFormats(),
structure.needClientTimezone(), true);
logstashFromFileDateFilter = makeLogstashDateFilter(timeFieldPath, structure.getTimestampFormats(),
logstashFromFileDateFilter = makeLogstashDateFilter(timeFieldPath, structure.getJodaTimestampFormats(),
structure.needClientTimezone(), false);
}

Expand All @@ -626,9 +626,9 @@ private void createDelimitedConfigs(FileStructure structure) {
String logstashFromFilebeatDateFilter = "";
String logstashFromFileDateFilter = "";
if (structure.getTimestampField() != null) {
logstashFromFilebeatDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getTimestampFormats(),
logstashFromFilebeatDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getJodaTimestampFormats(),
structure.needClientTimezone(), true);
logstashFromFileDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getTimestampFormats(),
logstashFromFileDateFilter = makeLogstashDateFilter(structure.getTimestampField(), structure.getJodaTimestampFormats(),
structure.needClientTimezone(), false);
}

Expand Down Expand Up @@ -660,7 +660,7 @@ private void createTextConfigs(FileStructure structure, List<String> sampleMessa
String grokPattern = structure.getGrokPattern();
String grokQuote = bestLogstashQuoteFor(grokPattern);
String interimTimestampField = structure.getTimestampField();
String dateFormatsStr = structure.getTimestampFormats().stream().collect(Collectors.joining("\", \"", "\"", "\""));
String dateFormatsStr = structure.getJodaTimestampFormats().stream().collect(Collectors.joining("\", \"", "\"", "\""));

String filebeatInputOptions = makeFilebeatInputOptions(structure.getMultilineStartPattern(), null, structure.getCharset());
filebeatToLogstashConfig = String.format(Locale.ROOT, TEXT_FILEBEAT_TO_LOGSTASH_TEMPLATE, filebeatInputOptions,
Expand Down
10 changes: 7 additions & 3 deletions ...-creator/src/test/java/org/elasticsearch/xpack/ml/configcreator/LogConfigWriterTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,7 @@ public void testCreateConfigsGivenElasticsearchLog() throws Exception {
containsString("encoding: '" + charset.toLowerCase(Locale.ROOT) + "'"));
}
assertThat(logConfigWriter.getFilebeatToLogstashConfig(),
containsString("multiline.pattern: '^\\[\\b\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}'\n"));
containsString("multiline.pattern: '^\\[\\b\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2},\\d{3}'\n"));
assertThat(logConfigWriter.getFilebeatToLogstashConfig(), containsString(logstashHost));
assertThat(logConfigWriter.getLogstashFromFilebeatConfig(),
containsString("match => { \"message\" => \"\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:loglevel} \\]" +
Expand All @@ -364,7 +364,11 @@ public void testCreateConfigsGivenElasticsearchLog() throws Exception {
containsString("match => { \"message\" => \"\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:loglevel} \\]" +
"\\[.*\" }\n"));
assertThat(logConfigWriter.getLogstashFromFileConfig(), containsString("match => [ \"timestamp\", \"ISO8601\" ]\n"));
assertThat(logConfigWriter.getLogstashFromFileConfig(), not(containsString("timezone =>")));
if (timezone == null) {
assertThat(logConfigWriter.getLogstashFromFileConfig(), not(containsString("timezone =>")));
} else {
assertThat(logConfigWriter.getLogstashFromFileConfig(), containsString("timezone => \"" + timezone + "\"\n"));
}
assertThat(logConfigWriter.getLogstashFromFileConfig(), containsString(elasticsearchHost));
if (charset.equals(StandardCharsets.UTF_8.name())) {
assertThat(logConfigWriter.getFilebeatToIngestPipelineConfig(), not(containsString("encoding:")));
Expand All @@ -373,7 +377,7 @@ public void testCreateConfigsGivenElasticsearchLog() throws Exception {
containsString("encoding: '" + charset.toLowerCase(Locale.ROOT) + "'"));
}
assertThat(logConfigWriter.getFilebeatToIngestPipelineConfig(),
containsString("multiline.pattern: '^\\[\\b\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}'\n"));
containsString("multiline.pattern: '^\\[\\b\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2},\\d{3}'\n"));
assertThat(logConfigWriter.getFilebeatToIngestPipelineConfig(), containsString(elasticsearchHost));
assertThat(logConfigWriter.getIngestPipelineFromFilebeatConfig(),
containsString("\"patterns\": [ \"\\\\[%{TIMESTAMP_ISO8601:timestamp}\\\\]\\\\[%{LOGLEVEL:loglevel} \\\\]" +
Expand Down

0 comments on commit cf06c02

Please sign in to comment.