From e032adfb97d66a637256ed64161673df965006fe Mon Sep 17 00:00:00 2001 From: Asif Sohail Mohammed Date: Fri, 12 Nov 2021 14:06:25 -0600 Subject: [PATCH] Added configConverter which converts conf to yaml Signed-off-by: Asif Sohail Mohammed --- .../parser/LogstashConfigConverter.java | 56 +++++++++++++++++++ .../parser/LogstashConfigConverterTest.java | 29 ++++++++++ .../dataprepper/logstash/parser/logstash.conf | 14 +++++ 3 files changed, 99 insertions(+) create mode 100644 data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/parser/LogstashConfigConverter.java create mode 100644 data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/LogstashConfigConverterTest.java create mode 100644 data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/parser/logstash.conf diff --git a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/parser/LogstashConfigConverter.java b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/parser/LogstashConfigConverter.java new file mode 100644 index 0000000000..7a677c9f62 --- /dev/null +++ b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/parser/LogstashConfigConverter.java @@ -0,0 +1,56 @@ +package org.opensearch.dataprepper.logstash.parser; + +import com.amazon.dataprepper.model.configuration.PipelineModel; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.tree.ParseTree; +import org.opensearch.dataprepper.logstash.LogstashLexer; +import org.opensearch.dataprepper.logstash.LogstashParser; +import org.opensearch.dataprepper.logstash.mapping.LogstashMapper; +import org.opensearch.dataprepper.logstash.model.LogstashConfiguration; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Map; + +/** + * Converts Logstash configuration file and returns YAML file location + * + * @since 1.2 + */ +public class LogstashConfigConverter { + public String convertLogstashConfigurationToPipeline(String logstashConfigurationPath, String outputDirectory) throws IOException { + final String logstashConfigAsString = new String(Files.readAllBytes(Paths.get(logstashConfigurationPath))); + + LogstashLexer lexer = new LogstashLexer(CharStreams.fromString(logstashConfigAsString)); + CommonTokenStream tokens = new CommonTokenStream(lexer); + LogstashParser parser = new LogstashParser(tokens); + ParseTree tree = parser.config(); + + LogstashVisitor visitor = new LogstashVisitor(); + LogstashConfiguration logstashConfiguration = (LogstashConfiguration) visitor.visit(tree); + + LogstashMapper logstashMapper = new LogstashMapper(); + PipelineModel pipelineModel = logstashMapper.mapPipeline(logstashConfiguration); + Map pipeline = Collections.singletonMap("log-pipeline", pipelineModel); + + YAMLFactory factory = new YAMLFactory(); + factory.disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER); + factory.enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR); + factory.disable(YAMLGenerator.Feature.SPLIT_LINES); + factory.enable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE); + + ObjectMapper mapper = new ObjectMapper(factory); + + String yamlFilePath = outputDirectory + "logstash.yaml"; + mapper.writeValue(new File(yamlFilePath), pipeline); + + return yamlFilePath; + } +} diff --git a/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/LogstashConfigConverterTest.java b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/LogstashConfigConverterTest.java new file mode 100644 index 0000000000..45e4ca41e8 --- /dev/null +++ b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/LogstashConfigConverterTest.java @@ -0,0 +1,29 @@ +package org.opensearch.dataprepper.logstash.parser; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +class LogstashConfigConverterTest { + private LogstashConfigConverter logstashConfigConverter; + + @BeforeEach + void createObjectUnderTest() { + logstashConfigConverter = new LogstashConfigConverter(); + } + + @Test + void convertLogstashConfigurationToPipeline() throws IOException { + + String outputDirectory = "build/resources/test/org/opensearch/dataprepper/logstash/parser/"; + String confPath = "src/test/resources/org/opensearch/dataprepper/logstash/parser/logstash.conf"; + + String actualYamlPath = logstashConfigConverter.convertLogstashConfigurationToPipeline(confPath, outputDirectory); + String expectedYamlPath = "build/resources/test/org/opensearch/dataprepper/logstash/parser/logstash.yaml"; + + assertThat(actualYamlPath, equalTo(expectedYamlPath)); + } +} \ No newline at end of file diff --git a/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/parser/logstash.conf b/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/parser/logstash.conf new file mode 100644 index 0000000000..ce98d4989d --- /dev/null +++ b/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/parser/logstash.conf @@ -0,0 +1,14 @@ +input { + http { + port => 8000 + threads => 100 + max_pending_requests => 128 + } +} +output { + elasticsearch { + hosts => ["https://localhost:9200"] + user => admin + password => admin + } +} \ No newline at end of file