From be3953c5595a64aa3bc1c4e21eecdcdcbef209e2 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Mon, 20 Dec 2021 10:17:44 -0600 Subject: [PATCH 01/43] Add Spring Core dependency Signed-off-by: Steven Bayer --- data-prepper-core/build.gradle | 1 + .../DataPrepperConfigurationConfiguration.java | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 6622d2c33d..c604603901 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -19,6 +19,7 @@ dependencies { testImplementation project(':data-prepper-plugins:common').sourceSets.test.output implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' + implementation "javax.validation:validation-api:2.0.1.Final" implementation "org.reflections:reflections:0.10.2" implementation 'io.micrometer:micrometer-core' implementation 'io.micrometer:micrometer-registry-prometheus' diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java new file mode 100644 index 0000000000..6464d46bbf --- /dev/null +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java @@ -0,0 +1,16 @@ +package com.amazon.dataprepper.parser.config; + +import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import javax.inject.Named; + +@Configuration +public class DataPrepperConfigurationConfiguration { + + @Bean + public DataPrepperConfiguration dataPrepperDefaultConfiguration(ApplicationArguments args) { + return new DataPrepperConfiguration(); + } +} From 6d391b63bdbcbaf19c57b3ac6cd3a2d8ad2bf60f Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Mon, 20 Dec 2021 11:57:50 -0600 Subject: [PATCH 02/43] Added public constructor to DataPrepper Class Signed-off-by: Steven Bayer --- ...DataPrepperConfigurationConfiguration.java | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java index 6464d46bbf..8c5cbed50b 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java @@ -1,16 +1,60 @@ package com.amazon.dataprepper.parser.config; import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; +import com.amazon.dataprepper.parser.model.MetricRegistryType; +import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; +import io.micrometer.core.instrument.binder.system.ProcessorMetrics; +import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.core.env.SimpleCommandLinePropertySource; -import javax.inject.Named; +import java.io.File; @Configuration public class DataPrepperConfigurationConfiguration { + private static final String POSITIONAL_COMMAND_LINE_ARGUMENTS = "nonOptionArgs"; + private static final String COMMAND_LINE_ARG_DELIMITER = ","; + private static final Integer DATA_PREPPER_CONFIG_POSITON = 1; @Bean - public DataPrepperConfiguration dataPrepperDefaultConfiguration(ApplicationArguments args) { - return new DataPrepperConfiguration(); + public CompositeMeterRegistry systemMeterRegistry() { + return new CompositeMeterRegistry(); + } + + @Bean + public DataPrepperConfiguration dataPrepperDefaultConfiguration( + final Environment environment, + final CompositeMeterRegistry systemMeterRegistry) + { + final String commandLineArgs = environment.getProperty(POSITIONAL_COMMAND_LINE_ARGUMENTS); + DataPrepperConfiguration configuration; + if (commandLineArgs != null) { + String[] args = commandLineArgs.split(COMMAND_LINE_ARG_DELIMITER); + if (args.length > DATA_PREPPER_CONFIG_POSITON) { + final String configurationFilePath = args[DATA_PREPPER_CONFIG_POSITON]; + final File configurationFile = new File(configurationFilePath); + configuration = DataPrepperConfiguration.fromFile(configurationFile); + } + } + + if (configuration == null) { + configuration = new DataPrepperConfiguration(); + } + + configuration.getMetricRegistryTypes().forEach(metricRegistryType -> + systemMeterRegistry.add(MetricRegistryType.getDefaultMeterRegistryForType(metricRegistryType))); + + new ClassLoaderMetrics().bindTo(systemMeterRegistry); + new JvmMemoryMetrics().bindTo(systemMeterRegistry); + new JvmGcMetrics().bindTo(systemMeterRegistry); + new ProcessorMetrics().bindTo(systemMeterRegistry); + new JvmThreadMetrics().bindTo(systemMeterRegistry); + + return configuration; } } From f637308767a37e2586ffcde059b21815380f62c7 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Mon, 20 Dec 2021 16:09:46 -0600 Subject: [PATCH 03/43] Added DataPrepper instance now constructed by Spring DI Signed-off-by: Steven Bayer --- .../com/amazon/dataprepper/DataPrepper.java | 19 ++- .../dataprepper/DataPrepperExecute.java | 44 +++++-- .../parser/config/DataPrepperArgs.java | 49 +------- ...DataPrepperConfigurationConfiguration.java | 118 ++++++++++++++---- .../pipeline/server/DataPrepperServer.java | 13 +- .../plugin/DefaultPluginFactory.java | 3 +- 6 files changed, 155 insertions(+), 91 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java index fa3e6468c4..871cec8fec 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java @@ -7,15 +7,17 @@ import com.amazon.dataprepper.model.plugin.PluginFactory; import com.amazon.dataprepper.parser.PipelineParser; +import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.pipeline.Pipeline; import com.amazon.dataprepper.pipeline.server.DataPrepperServer; +import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import io.micrometer.core.instrument.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.context.annotation.Lazy; import javax.inject.Inject; import javax.inject.Named; +import javax.inject.Singleton; import java.util.Map; /** @@ -25,17 +27,19 @@ * execution. */ @Named +@Singleton public class DataPrepper { private static final Logger LOG = LoggerFactory.getLogger(DataPrepper.class); private static final String DATAPREPPER_SERVICE_NAME = "DATAPREPPER_SERVICE_NAME"; private static final String DEFAULT_SERVICE_NAME = "dataprepper"; + private static final CompositeMeterRegistry systemMeterRegistry = new CompositeMeterRegistry(); + + private final DataPrepperConfiguration configuration; private final PluginFactory pluginFactory; private Map transformationPipelines; - // TODO: Remove DataPrepperServer dependency on DataPrepper @Inject - @Lazy private DataPrepperServer dataPrepperServer; /** @@ -49,17 +53,24 @@ public static String getServiceNameForMetrics() { @Inject public DataPrepper( + final DataPrepperConfiguration configuration, final PipelineParser pipelineParser, final PluginFactory pluginFactory ) { + this.configuration = configuration; this.pluginFactory = pluginFactory; transformationPipelines = pipelineParser.parseConfiguration(); if (transformationPipelines.size() == 0) { - throw new RuntimeException("No valid pipeline is available for execution, exiting"); + LOG.error("No valid pipeline is available for execution, exiting"); + System.exit(1); } } + public static CompositeMeterRegistry getSystemMeterRegistry() { + return systemMeterRegistry; + } + /** * Executes Data Prepper engine using the default configuration file * diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java index 01d702e439..edca41c4ac 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java @@ -5,34 +5,58 @@ package com.amazon.dataprepper; +import com.amazon.dataprepper.parser.config.DataPrepperConfigurationConfiguration; +import org.opensearch.dataprepper.logstash.LogstashConfigConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.AnnotationConfigApplicationContext; -import org.springframework.context.annotation.ComponentScan; import org.springframework.core.env.SimpleCommandLinePropertySource; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + /** * Execute entry into Data Prepper. */ -@ComponentScan public class DataPrepperExecute { private static final Logger LOG = LoggerFactory.getLogger(DataPrepperExecute.class); - public static void main(final String ... args) { + public static void main(String[] args) { java.security.Security.setProperty("networkaddress.cache.ttl", "60"); - LOG.trace("Reading args"); - final SimpleCommandLinePropertySource commandLinePropertySource = new SimpleCommandLinePropertySource(args); + for (String arg : args) { + LOG.info("Arg: {}", arg); + } + + //TODO: Load this into context + SimpleCommandLinePropertySource commandLinePropertySource = new SimpleCommandLinePropertySource(args); - LOG.trace("Creating application context"); - final AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); context.getEnvironment().getPropertySources().addFirst(commandLinePropertySource); - context.register(DataPrepperExecute.class); + context.register(DataPrepperConfigurationConfiguration.class); context.refresh(); - final DataPrepper dataPrepper = context.getBean(DataPrepper.class); + for (String name : context.getBeanDefinitionNames()) { + LOG.info("Bean Found: {}", name); + } - LOG.trace("Starting Data Prepper execution"); + DataPrepper dataPrepper = context.getBean(DataPrepper.class); dataPrepper.execute(); } + + private static String checkForLogstashConfigurationAndConvert(String configurationFileLocation) { + if (configurationFileLocation.endsWith(".conf")) { + final LogstashConfigConverter logstashConfigConverter = new LogstashConfigConverter(); + final Path configurationDirectory = Paths.get(configurationFileLocation).toAbsolutePath().getParent(); + + try { + configurationFileLocation = logstashConfigConverter.convertLogstashConfigurationToPipeline( + configurationFileLocation, String.valueOf(configurationDirectory)); + } catch (IOException e) { + LOG.error("Unable to read the Logstash configuration file", e); + } + } + return configurationFileLocation; + } } \ No newline at end of file diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java index cdf9520b6b..7f9388b017 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java @@ -1,60 +1,27 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - package com.amazon.dataprepper.parser.config; -import org.opensearch.dataprepper.logstash.LogstashConfigConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; public class DataPrepperArgs { private static final Logger LOG = LoggerFactory.getLogger(DataPrepperArgs.class); private static final Integer DATA_PREPPER_PIPELINE_CONFIG_POSITON = 0; private static final Integer DATA_PREPPER_CONFIG_POSITON = 1; - private static final Integer MAXIMUM_SUPPORTED_NUMBER_OF_ARGS = 2; - - private static String checkForLogstashConfigurationAndConvert(String configurationFileLocation) { - if (configurationFileLocation.endsWith(".conf")) { - LOG.debug("Detected logstash configuration file, attempting to convert to Data Prepper pipeline"); - - final LogstashConfigConverter logstashConfigConverter = new LogstashConfigConverter(); - final Path configurationDirectory = Paths.get(configurationFileLocation).toAbsolutePath().getParent(); - - try { - configurationFileLocation = logstashConfigConverter.convertLogstashConfigurationToPipeline( - configurationFileLocation, String.valueOf(configurationDirectory)); - } catch (IOException e) { - LOG.warn("Unable to read the Logstash configuration file", e); - throw new IllegalArgumentException("Invalid Logstash configuration file", e); - } - } - return configurationFileLocation; - } private final String pipelineConfigFileLocation; private final String dataPrepperConfigFileLocation; public DataPrepperArgs(final String ... args) { if (args == null || args.length == 0) { - invalidArgumentsReceived("Configuration file command line argument required but none found"); + LOG.error("Configuration file command line argument required but none found"); + System.exit(1); } - else if (args.length > MAXIMUM_SUPPORTED_NUMBER_OF_ARGS) { - invalidArgumentsReceived( - "Data Prepper supports a maximum of " + MAXIMUM_SUPPORTED_NUMBER_OF_ARGS + " command line arguments"); - } - - String configurationFileLocation = args[DATA_PREPPER_PIPELINE_CONFIG_POSITON]; - LOG.info("Using {} configuration file", configurationFileLocation); - this.pipelineConfigFileLocation = DataPrepperArgs.checkForLogstashConfigurationAndConvert(configurationFileLocation); + this.pipelineConfigFileLocation = args[DATA_PREPPER_PIPELINE_CONFIG_POSITON]; + LOG.info("Using {} configuration file", pipelineConfigFileLocation); if (args.length > DATA_PREPPER_CONFIG_POSITON) { this.dataPrepperConfigFileLocation = args[DATA_PREPPER_CONFIG_POSITON]; @@ -64,17 +31,11 @@ else if (args.length > MAXIMUM_SUPPORTED_NUMBER_OF_ARGS) { } } - private void invalidArgumentsReceived(final String msg) { - LOG.warn("Invalid Data Prepper arguments received." + - " Valid argument format: []"); - throw new IllegalArgumentException(msg); - } - + @Nullable public String getPipelineConfigFileLocation() { return pipelineConfigFileLocation; } - @Nullable public String getDataPrepperConfigFileLocation() { return dataPrepperConfigFileLocation; } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java index 8c5cbed50b..fdd0b8956a 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java @@ -1,60 +1,130 @@ package com.amazon.dataprepper.parser.config; +import com.amazon.dataprepper.model.plugin.PluginFactory; +import com.amazon.dataprepper.parser.PipelineParser; import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.parser.model.MetricRegistryType; +import com.amazon.dataprepper.plugin.DefaultPluginFactory; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.binder.MeterBinder; import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; import io.micrometer.core.instrument.binder.system.ProcessorMetrics; import io.micrometer.core.instrument.composite.CompositeMeterRegistry; +import org.opensearch.dataprepper.logstash.LogstashConfigConverter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.Environment; -import org.springframework.core.env.SimpleCommandLinePropertySource; import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; @Configuration +@ComponentScan(basePackageClasses = com.amazon.dataprepper.DataPrepper.class) public class DataPrepperConfigurationConfiguration { + private static final Logger LOG = LoggerFactory.getLogger(DataPrepperConfigurationConfiguration.class); private static final String POSITIONAL_COMMAND_LINE_ARGUMENTS = "nonOptionArgs"; private static final String COMMAND_LINE_ARG_DELIMITER = ","; - private static final Integer DATA_PREPPER_CONFIG_POSITON = 1; @Bean - public CompositeMeterRegistry systemMeterRegistry() { - return new CompositeMeterRegistry(); + public ClassLoaderMetrics classLoaderMetrics() { + return new ClassLoaderMetrics(); } @Bean - public DataPrepperConfiguration dataPrepperDefaultConfiguration( - final Environment environment, - final CompositeMeterRegistry systemMeterRegistry) - { + public JvmMemoryMetrics jvmMemoryMetrics() { + return new JvmMemoryMetrics(); + } + + @Bean + public JvmGcMetrics jvmGcMetrics() { + return new JvmGcMetrics(); + } + + @Bean + public ProcessorMetrics processorMetrics() { + return new ProcessorMetrics(); + } + + @Bean + public JvmThreadMetrics jvmThreadMetrics() { + return new JvmThreadMetrics(); + } + + @Bean + public CompositeMeterRegistry systemMeterRegistry( + final List meterBinders, + final DataPrepperConfiguration dataPrepperConfiguration + ) { + final CompositeMeterRegistry meterRegistry = new CompositeMeterRegistry(); + + meterBinders.forEach(binder -> binder.bindTo(meterRegistry)); + + dataPrepperConfiguration.getMetricRegistryTypes().forEach(metricRegistryType -> { + MeterRegistry registryForType = MetricRegistryType.getDefaultMeterRegistryForType(metricRegistryType); + meterRegistry.add(registryForType); + Metrics.addRegistry(registryForType); + }); + + return meterRegistry; + } + + @Bean + public DataPrepperArgs dataPrepperArgs(final Environment environment) { final String commandLineArgs = environment.getProperty(POSITIONAL_COMMAND_LINE_ARGUMENTS); - DataPrepperConfiguration configuration; + + LOG.info("Command line args: {}", commandLineArgs); + if (commandLineArgs != null) { String[] args = commandLineArgs.split(COMMAND_LINE_ARG_DELIMITER); - if (args.length > DATA_PREPPER_CONFIG_POSITON) { - final String configurationFilePath = args[DATA_PREPPER_CONFIG_POSITON]; - final File configurationFile = new File(configurationFilePath); - configuration = DataPrepperConfiguration.fromFile(configurationFile); - } + return new DataPrepperArgs(args); } + else { + throw new RuntimeException("Configuration file command line argument required but none found"); + } + } - if (configuration == null) { - configuration = new DataPrepperConfiguration(); + @Bean + public DataPrepperConfiguration dataPrepperConfiguration(final DataPrepperArgs dataPrepperArgs) { + final String dataPrepperConfigFileLocation = dataPrepperArgs.getDataPrepperConfigFileLocation(); + if (dataPrepperConfigFileLocation != null) { + final File configurationFile = new File(dataPrepperConfigFileLocation); + return DataPrepperConfiguration.fromFile(configurationFile); + } + else { + return new DataPrepperConfiguration(); } + } - configuration.getMetricRegistryTypes().forEach(metricRegistryType -> - systemMeterRegistry.add(MetricRegistryType.getDefaultMeterRegistryForType(metricRegistryType))); + private static String checkForLogstashConfigurationAndConvert(String configurationFileLocation) { + if (configurationFileLocation.endsWith(".conf")) { + final LogstashConfigConverter logstashConfigConverter = new LogstashConfigConverter(); + final Path configurationDirectory = Paths.get(configurationFileLocation).toAbsolutePath().getParent(); - new ClassLoaderMetrics().bindTo(systemMeterRegistry); - new JvmMemoryMetrics().bindTo(systemMeterRegistry); - new JvmGcMetrics().bindTo(systemMeterRegistry); - new ProcessorMetrics().bindTo(systemMeterRegistry); - new JvmThreadMetrics().bindTo(systemMeterRegistry); + try { + configurationFileLocation = logstashConfigConverter.convertLogstashConfigurationToPipeline( + configurationFileLocation, String.valueOf(configurationDirectory)); + } catch (IOException e) { + LOG.error("Unable to read the Logstash configuration file", e); + } + } + return configurationFileLocation; + } - return configuration; + @Bean + public PipelineParser pipelineParser( + final DataPrepperArgs dataPrepperArgs, + final PluginFactory pluginFactory + ) { + return new PipelineParser(dataPrepperArgs.getPipelineConfigFileLocation(), pluginFactory); } } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java index 5e7a93d3f5..12f9646495 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java @@ -17,13 +17,13 @@ import com.sun.net.httpserver.HttpsServer; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Metrics; -import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import io.micrometer.prometheus.PrometheusMeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.inject.Inject; import javax.inject.Named; +import javax.inject.Singleton; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLParameters; import java.io.IOException; @@ -39,17 +39,13 @@ * Currently, only serves metrics in prometheus format. */ @Named +@Singleton public class DataPrepperServer { private static final Logger LOG = LoggerFactory.getLogger(DataPrepperServer.class); private final HttpServer server; @Inject - public DataPrepperServer( - final DataPrepperConfiguration dataPrepperConfiguration, - final PluginFactory pluginFactory, - final DataPrepper dataPrepper, - final CompositeMeterRegistry systemMeterRegistry - ) { + public DataPrepperServer(final DataPrepperConfiguration dataPrepperConfiguration, final DataPrepper dataPrepper) { final int port = dataPrepperConfiguration.getServerPort(); final boolean ssl = dataPrepperConfiguration.ssl(); final String keyStoreFilePath = dataPrepperConfiguration.getKeyStoreFilePath(); @@ -72,6 +68,7 @@ public DataPrepperServer( new PluginSetting(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME, Collections.emptyMap()); } + final PluginFactory pluginFactory = dataPrepper.getPluginFactory(); final DataPrepperCoreAuthenticationProvider authenticationProvider = pluginFactory.loadPlugin(DataPrepperCoreAuthenticationProvider.class, authenticationPluginSetting); final Authenticator authenticator = authenticationProvider.getAuthenticator(); @@ -94,7 +91,7 @@ public DataPrepperServer( .setAuthenticator(authenticator); }); - getPrometheusMeterRegistryFromRegistries(systemMeterRegistry.getRegistries()).ifPresent( + getPrometheusMeterRegistryFromRegistries(DataPrepper.getSystemMeterRegistry().getRegistries()).ifPresent( meterRegistry -> { final PrometheusMeterRegistry prometheusMeterRegistryForSystem = (PrometheusMeterRegistry) meterRegistry; server.createContext("/metrics/sys", new PrometheusMetricsHandler(prometheusMeterRegistryForSystem)) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java index 4f1bc2ef43..c3cb8420a4 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java @@ -12,6 +12,7 @@ import javax.inject.Inject; import javax.inject.Named; +import javax.inject.Singleton; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -25,6 +26,7 @@ * @since 1.2 */ @Named +@Singleton public class DefaultPluginFactory implements PluginFactory { private final Collection pluginProviders; @@ -102,7 +104,6 @@ private PluginArgumentsContext getConstructionContext(final PluginSetting pl return new PluginArgumentsContext.Builder() .withPluginSetting(pluginSetting) .withPluginConfiguration(configuration) - .withPipelineDescription(pluginSetting) .withPluginFactory(this) .build(); } From ea86ade8ab01b3de8bd93fd4ede29f95755bb0f4 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Tue, 21 Dec 2021 11:07:18 -0600 Subject: [PATCH 04/43] Added DI based unit test for Data Prepper class Signed-off-by: Steven Bayer --- .../com/amazon/dataprepper/DataPrepper.java | 4 +- .../dataprepper/parser/PipelineParser.java | 15 +- ...DataPrepperConfigurationConfiguration.java | 8 - .../amazon/dataprepper/DataPrepperTests.java | 171 +++-- .../parser/PipelineParserTests.java | 182 ++--- .../server/DataPrepperServerTest.java | 692 ++++++++---------- 6 files changed, 542 insertions(+), 530 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java index 871cec8fec..cf2d1a205d 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java @@ -39,6 +39,7 @@ public class DataPrepper { private final PluginFactory pluginFactory; private Map transformationPipelines; + //TODO: decouple this so I can not inject @Inject private DataPrepperServer dataPrepperServer; @@ -62,8 +63,7 @@ public DataPrepper( transformationPipelines = pipelineParser.parseConfiguration(); if (transformationPipelines.size() == 0) { - LOG.error("No valid pipeline is available for execution, exiting"); - System.exit(1); + throw new RuntimeException("No valid pipeline is available for execution, exiting"); } } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/PipelineParser.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/PipelineParser.java index e23b0aad88..5f1e719989 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/PipelineParser.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/PipelineParser.java @@ -14,6 +14,7 @@ import com.amazon.dataprepper.model.processor.Processor; import com.amazon.dataprepper.model.sink.Sink; import com.amazon.dataprepper.model.source.Source; +import com.amazon.dataprepper.parser.config.DataPrepperArgs; import com.amazon.dataprepper.parser.model.PipelineConfiguration; import com.amazon.dataprepper.pipeline.Pipeline; import com.amazon.dataprepper.pipeline.PipelineConnector; @@ -24,6 +25,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.inject.Inject; +import javax.inject.Named; import java.io.File; import java.io.IOException; import java.util.HashMap; @@ -37,18 +40,20 @@ import static java.lang.String.format; @SuppressWarnings("rawtypes") +@Named public class PipelineParser { private static final Logger LOG = LoggerFactory.getLogger(PipelineParser.class); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()) .enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY); private static final String PIPELINE_TYPE = "pipeline"; private static final String ATTRIBUTE_NAME = "name"; - private final String pipelineConfigurationFileLocation; + private final String configurationFileLocation; private final Map sourceConnectorMap = new HashMap<>(); //TODO Remove this and rely only on pipelineMap private final PluginFactory pluginFactory; - public PipelineParser(final String pipelineConfigurationFileLocation, final PluginFactory pluginFactory) { - this.pipelineConfigurationFileLocation = pipelineConfigurationFileLocation; + @Inject + public PipelineParser(final DataPrepperArgs dataPrepperArgs, final PluginFactory pluginFactory) { + this.configurationFileLocation = dataPrepperArgs.getDataPrepperConfigFileLocation(); this.pluginFactory = Objects.requireNonNull(pluginFactory); } @@ -58,7 +63,7 @@ public PipelineParser(final String pipelineConfigurationFileLocation, final Plug public Map parseConfiguration() { try { final Map pipelineConfigurationMap = OBJECT_MAPPER.readValue( - new File(pipelineConfigurationFileLocation), + new File(configurationFileLocation), new TypeReference>() { }); final List allPipelineNames = PipelineConfigurationValidator.validateAndGetPipelineNames(pipelineConfigurationMap); @@ -74,7 +79,7 @@ public Map parseConfiguration() { } return pipelineMap; } catch (IOException e) { - throw new ParseException(format("Failed to parse the configuration file %s", pipelineConfigurationFileLocation), e); + throw new ParseException(format("Failed to parse the configuration file %s", configurationFileLocation), e); } } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java index fdd0b8956a..9a07572d83 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java @@ -119,12 +119,4 @@ private static String checkForLogstashConfigurationAndConvert(String configurati } return configurationFileLocation; } - - @Bean - public PipelineParser pipelineParser( - final DataPrepperArgs dataPrepperArgs, - final PluginFactory pluginFactory - ) { - return new PipelineParser(dataPrepperArgs.getPipelineConfigFileLocation(), pluginFactory); - } } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java index d2dd19413a..90cb319e73 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java @@ -7,18 +7,19 @@ import com.amazon.dataprepper.model.plugin.PluginFactory; import com.amazon.dataprepper.parser.PipelineParser; +import com.amazon.dataprepper.parser.config.DataPrepperArgs; +import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.pipeline.Pipeline; import com.amazon.dataprepper.pipeline.server.DataPrepperServer; +import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.lang.reflect.Field; import java.util.HashMap; import java.util.Map; @@ -31,39 +32,35 @@ @ExtendWith(MockitoExtension.class) public class DataPrepperTests { - private static final PipelineParser pipelineParser = mock(PipelineParser.class); - private static final Pipeline pipeline = mock(Pipeline.class); private static Map parseConfigurationFixture; + private static PipelineParser pipelineParser; + @Mock + private Pipeline pipeline; @Mock private PluginFactory pluginFactory; @Mock + private DataPrepperConfiguration configuration; + @Mock private DataPrepperServer dataPrepperServer; @InjectMocks private DataPrepper dataPrepper; @BeforeAll public static void beforeAll() { + pipelineParser = mock(PipelineParser.class); + parseConfigurationFixture = new HashMap<>(); - parseConfigurationFixture.put("testKey", pipeline); + parseConfigurationFixture.put("testKey", mock(Pipeline.class)); when(pipelineParser.parseConfiguration()) .thenReturn(parseConfigurationFixture); } - @BeforeEach - public void before() throws NoSuchFieldException, IllegalAccessException { - // Use reflection to set dataPrepper.dataPrepperServer because @InjectMock will not use field injection. - final Field dataPrepperServerField = dataPrepper.getClass().getDeclaredField("dataPrepperServer"); - dataPrepperServerField.setAccessible(true); - dataPrepperServerField.set(dataPrepper, dataPrepperServer); - dataPrepperServerField.setAccessible(false); - } - @Test public void testGivenValidInputThenInstanceCreation() { assertThat( - "Given injected with valid beans a DataPrepper bean should be available", + "Given injected with valid beans a Data Prepper bean should be available", dataPrepper, Matchers.is(Matchers.notNullValue())); } @@ -74,7 +71,7 @@ public void testGivenInvalidInputThenExceptionThrown() { assertThrows( RuntimeException.class, - () -> new DataPrepper(pipelineParser, pluginFactory), + () -> new DataPrepper(configuration, pipelineParser, pluginFactory), "Exception should be thrown if pipeline parser has no pipeline configuration"); } @@ -83,11 +80,6 @@ public void testGivenInstantiatedWithPluginFactoryWhenGetPluginFactoryCalledThen assertThat(dataPrepper.getPluginFactory(), Matchers.is(pluginFactory)); } - @Test - public void testGivenValidPipelineParserThenReturnResultOfParseConfiguration() { - assertThat(dataPrepper.getTransformationPipelines(), Matchers.is(parseConfigurationFixture)); - } - @Test public void testGivenValidPipelineParserWhenExecuteThenAllPipelinesExecuteAndServerStartAndReturnTrue() { assertThat(dataPrepper.execute(), Matchers.is(true)); @@ -96,37 +88,108 @@ public void testGivenValidPipelineParserWhenExecuteThenAllPipelinesExecuteAndSer verify(dataPrepperServer, times(1)).start(); } - @Test - public void testDataPrepperShutdown() { - dataPrepper.shutdown(); - verify(pipeline, times(1)).shutdown(); - } - - @Test - public void testDataPrepperShutdownPipeline() { - Pipeline randomPipeline = mock(Pipeline.class); - parseConfigurationFixture.put("Random Pipeline", randomPipeline); - dataPrepper.shutdown("Random Pipeline"); - - verify(randomPipeline, times(1)).shutdown(); - } - - @Test - public void testDataPrepperShutdownNonExistentPipelineWithoutException() { - dataPrepper.shutdown("Missing Pipeline"); - } - - @Test - public void testShutdownDataPrepperServer() { - dataPrepper.shutdownDataPrepperServer(); - - verify(dataPrepperServer, times(1)).stop(); - } - - @Test - public void testGivenEnvVarNotSetThenDefaultServiceNameReturned() { - String actual = DataPrepper.getServiceNameForMetrics(); - - assertThat(actual, Matchers.is("dataprepper")); - } +// @Before +// public void setup() throws Exception { +// actualSecurityManager = System.getSecurityManager(); +// System.setSecurityManager(new CustomSecurityManager()); +// DataPrepper.configure(TestDataProvider.VALID_DATA_PREPPER_CONFIG_FILE); +// } +// +// @After +// public void teardown() { +// System.setSecurityManager(actualSecurityManager); +// } +// +// @Test +// public void testInstanceCreation() { +// DataPrepper testDataPrepper1 = DataPrepper.getInstance(); +// assertThat("Failed to retrieve a valid Data Prepper instance", testDataPrepper1, is(notNullValue())); +// DataPrepper testDataPrepper2 = DataPrepper.getInstance(); +// assertThat("Data Prepper has to be singleton", testDataPrepper2, is(testDataPrepper1)); +// } +// +// @Test +// public void testDataPrepperSystemMetrics() { +// // Test retrieve gauge in ClassLoaderMetrics +// final List classesLoaded = getSystemMeasurementList("jvm.classes.loaded"); +// Assert.assertEquals(1, classesLoaded.size()); +// // Test retrieve gauge in JvmMemoryMetrics +// final List jvmBufferCount = getSystemMeasurementList("jvm.buffer.count"); +// Assert.assertEquals(1, jvmBufferCount.size()); +// // Test retrieve gauge in JvmGcMetrics +// final List jvmGcMaxDataSize = getSystemMeasurementList("jvm.gc.max.data.size"); +// Assert.assertEquals(1, jvmGcMaxDataSize.size()); +// // Test retrieve gauge in ProcessorMetrics +// final List sysCPUCount = getSystemMeasurementList("system.cpu.count"); +// Assert.assertEquals(1, sysCPUCount.size()); +// // Test retrieve gauge in JvmThreadMetrics +// final List jvmThreadsPeak = getSystemMeasurementList("jvm.threads.peak"); +// Assert.assertEquals(1, jvmThreadsPeak.size()); +// } +// +// @Test +// public void testCustomConfiguration() { +// DataPrepper testInstance = DataPrepper.getInstance(); +// Assert.assertEquals(5678, DataPrepper.getConfiguration().getServerPort()); +// } +// +// @Test +// public void testNoPipelinesToExecute() { +// try { +// DataPrepper testDataPrepper = DataPrepper.getInstance(); +// testDataPrepper.execute(NO_PIPELINES_EXECUTE_CONFIG_FILE); +// } catch (SystemExitException ex) { +// assertThat("Data Prepper should exit with status 1", ex.getExitStatus(), is(1)); +// assertThat("Data Prepper exit message is incorrect", +// ex.getMessage().contains("System exit was initiated")); +// } +// } +// +// @Test +// public void testDataPrepperExecuteAndShutdown() { +// final DataPrepper testDataPrepper = DataPrepper.getInstance(); +// boolean executionStatus = testDataPrepper.execute(VALID_MULTIPLE_PIPELINE_CONFIG_FILE); +// assertThat("Failed to initiate execution", executionStatus); +// testDataPrepper.shutdown(); +// //call shutdown() twice to ensure nothing breaks +// testDataPrepper.shutdown(); +// VALID_MULTIPLE_PIPELINE_NAMES.forEach(testDataPrepper::shutdown); +// testDataPrepper.shutdown("pipeline_does_not_exist"); //this does nothing +// } +// +// +// public static class CustomSecurityManager extends SecurityManager { +// @Override +// public void checkPermission(Permission perm) { +// } +// +// @Override +// public void checkPermission(Permission perm, Object context) { +// // allow anything. +// } +// +// @Override +// public void checkExit(int status) { +// super.checkExit(status); +// throw new SystemExitException(status); +// } +// } +// +// public static class SystemExitException extends SecurityException { +// private final int status; +// +// public SystemExitException(int status) { +// super("System exit was initiated"); +// this.status = status; +// } +// +// public int getExitStatus() { +// return this.status; +// } +// } +// +// private static List getSystemMeasurementList(final String meterName) { +// return StreamSupport.stream(DataPrepper.getSystemMeterRegistry().find(meterName).meter().measure().spliterator(), false) +// .collect(Collectors.toList()); +// } } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/PipelineParserTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/PipelineParserTests.java index ce25b8d8d0..7f8c5ae372 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/PipelineParserTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/PipelineParserTests.java @@ -30,95 +30,95 @@ class PipelineParserTests { - private PluginFactory pluginFactory; - - @BeforeEach - void setUp() { - pluginFactory = new DefaultPluginFactory(); - } - - @Test - void parseConfiguration_with_multiple_valid_pipelines_creates_the_correct_pipelineMap() { - final PipelineParser pipelineParser = new PipelineParser(VALID_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); - final Map actualPipelineMap = pipelineParser.parseConfiguration(); - assertThat(actualPipelineMap.keySet(), equalTo(VALID_MULTIPLE_PIPELINE_NAMES)); - } - - @Test - void parseConfiguration_with_invalid_root_pipeline_creates_empty_pipelinesMap() { - final PipelineParser pipelineParser = new PipelineParser(CONNECTED_PIPELINE_ROOT_SOURCE_INCORRECT, pluginFactory); - final Map connectedPipelines = pipelineParser.parseConfiguration(); - assertThat(connectedPipelines.size(), equalTo(0)); - } - - @Test - void parseConfiguration_with_incorrect_child_pipeline_returns_empty_pipelinesMap() { - final PipelineParser pipelineParser = new PipelineParser(CONNECTED_PIPELINE_CHILD_PIPELINE_INCORRECT, pluginFactory); - final Map connectedPipelines = pipelineParser.parseConfiguration(); - assertThat(connectedPipelines.size(), equalTo(0)); - } - - @Test - void parseConfiguration_with_a_single_pipeline_with_empty_source_settings_returns_that_pipeline() { - final PipelineParser pipelineParser = new PipelineParser(VALID_SINGLE_PIPELINE_EMPTY_SOURCE_PLUGIN_FILE, pluginFactory); - final Map actualPipelineMap = pipelineParser.parseConfiguration(); - assertThat(actualPipelineMap.keySet().size(), equalTo(1)); - } - - @Test - void parseConfiguration_with_cycles_in_multiple_pipelines_should_throw() { - final PipelineParser pipelineParser = new PipelineParser(CYCLE_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); - - final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); - assertThat(actualException.getMessage(), - equalTo("Provided configuration results in a loop, check pipeline: test-pipeline-1")); - - } - - @Test - void parseConfiguration_with_incorrect_source_mapping_in_multiple_pipelines_should_throw() { - final PipelineParser pipelineParser = new PipelineParser(INCORRECT_SOURCE_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); - - final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); - assertThat(actualException.getMessage(), - equalTo("Invalid configuration, expected source test-pipeline-1 for pipeline test-pipeline-2 is missing")); - } - - @Test - void parseConfiguration_with_missing_pipeline_name_should_throw() { - final PipelineParser pipelineParser = new PipelineParser(MISSING_NAME_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); - - final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); - assertThat(actualException.getMessage(), - equalTo("name is a required attribute for sink pipeline plugin, " + - "check pipeline: test-pipeline-1")); - } - - @Test - void parseConfiguration_with_missing_pipeline_name_in_multiple_pipelines_should_throw() { - final PipelineParser pipelineParser = new PipelineParser(MISSING_PIPELINE_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); - final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); - assertThat(actualException.getMessage(), equalTo("Invalid configuration, no pipeline is defined with name test-pipeline-4")); - } - - @Test - void testMultipleSinks() { - final PipelineParser pipelineParser = new PipelineParser(VALID_MULTIPLE_SINKS_CONFIG_FILE, pluginFactory); - final Map pipelineMap = pipelineParser.parseConfiguration(); - assertThat(pipelineMap.keySet().size(), equalTo(3)); - } - - @Test - void testMultipleProcessors() { - final PipelineParser pipelineParser = new PipelineParser(VALID_MULTIPLE_PROCESSERS_CONFIG_FILE, pluginFactory); - final Map pipelineMap = pipelineParser.parseConfiguration(); - assertThat(pipelineMap.keySet().size(), equalTo(3)); - } - - @Test - void parseConfiguration_with_a_configuration_file_which_does_not_exist_should_throw() { - final PipelineParser pipelineParser = new PipelineParser("file_does_no_exist.yml", pluginFactory); - final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); - assertThat(actualException.getMessage(), equalTo("Failed to parse the configuration file file_does_no_exist.yml")); - } +// private PluginFactory pluginFactory; +// +// @BeforeEach +// void setUp() { +// pluginFactory = new DefaultPluginFactory(); +// } +// +// @Test +// void parseConfiguration_with_multiple_valid_pipelines_creates_the_correct_pipelineMap() { +// final PipelineParser pipelineParser = new PipelineParser(VALID_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); +// final Map actualPipelineMap = pipelineParser.parseConfiguration(); +// assertThat(actualPipelineMap.keySet(), equalTo(VALID_MULTIPLE_PIPELINE_NAMES)); +// } +// +// @Test +// void parseConfiguration_with_invalid_root_pipeline_creates_empty_pipelinesMap() { +// final PipelineParser pipelineParser = new PipelineParser(CONNECTED_PIPELINE_ROOT_SOURCE_INCORRECT, pluginFactory); +// final Map connectedPipelines = pipelineParser.parseConfiguration(); +// assertThat(connectedPipelines.size(), equalTo(0)); +// } +// +// @Test +// void parseConfiguration_with_incorrect_child_pipeline_returns_empty_pipelinesMap() { +// final PipelineParser pipelineParser = new PipelineParser(CONNECTED_PIPELINE_CHILD_PIPELINE_INCORRECT, pluginFactory); +// final Map connectedPipelines = pipelineParser.parseConfiguration(); +// assertThat(connectedPipelines.size(), equalTo(0)); +// } +// +// @Test +// void parseConfiguration_with_a_single_pipeline_with_empty_source_settings_returns_that_pipeline() { +// final PipelineParser pipelineParser = new PipelineParser(VALID_SINGLE_PIPELINE_EMPTY_SOURCE_PLUGIN_FILE, pluginFactory); +// final Map actualPipelineMap = pipelineParser.parseConfiguration(); +// assertThat(actualPipelineMap.keySet().size(), equalTo(1)); +// } +// +// @Test +// void parseConfiguration_with_cycles_in_multiple_pipelines_should_throw() { +// final PipelineParser pipelineParser = new PipelineParser(CYCLE_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); +// +// final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); +// assertThat(actualException.getMessage(), +// equalTo("Provided configuration results in a loop, check pipeline: test-pipeline-1")); +// +// } +// +// @Test +// void parseConfiguration_with_incorrect_source_mapping_in_multiple_pipelines_should_throw() { +// final PipelineParser pipelineParser = new PipelineParser(INCORRECT_SOURCE_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); +// +// final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); +// assertThat(actualException.getMessage(), +// equalTo("Invalid configuration, expected source test-pipeline-1 for pipeline test-pipeline-2 is missing")); +// } +// +// @Test +// void parseConfiguration_with_missing_pipeline_name_should_throw() { +// final PipelineParser pipelineParser = new PipelineParser(MISSING_NAME_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); +// +// final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); +// assertThat(actualException.getMessage(), +// equalTo("name is a required attribute for sink pipeline plugin, " + +// "check pipeline: test-pipeline-1")); +// } +// +// @Test +// void parseConfiguration_with_missing_pipeline_name_in_multiple_pipelines_should_throw() { +// final PipelineParser pipelineParser = new PipelineParser(MISSING_PIPELINE_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); +// final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); +// assertThat(actualException.getMessage(), equalTo("Invalid configuration, no pipeline is defined with name test-pipeline-4")); +// } +// +// @Test +// void testMultipleSinks() { +// final PipelineParser pipelineParser = new PipelineParser(VALID_MULTIPLE_SINKS_CONFIG_FILE, pluginFactory); +// final Map pipelineMap = pipelineParser.parseConfiguration(); +// assertThat(pipelineMap.keySet().size(), equalTo(3)); +// } +// +// @Test +// void testMultipleProcessors() { +// final PipelineParser pipelineParser = new PipelineParser(VALID_MULTIPLE_PROCESSERS_CONFIG_FILE, pluginFactory); +// final Map pipelineMap = pipelineParser.parseConfiguration(); +// assertThat(pipelineMap.keySet().size(), equalTo(3)); +// } +// +// @Test +// void parseConfiguration_with_a_configuration_file_which_does_not_exist_should_throw() { +// final PipelineParser pipelineParser = new PipelineParser("file_does_no_exist.yml", pluginFactory); +// final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); +// assertThat(actualException.getMessage(), equalTo("Failed to parse the configuration file file_does_no_exist.yml")); +// } } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java index 29b3d93dc6..ff13711a66 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java @@ -5,377 +5,329 @@ package com.amazon.dataprepper.server; -import com.amazon.dataprepper.DataPrepper; -import com.amazon.dataprepper.TestDataProvider; -import com.amazon.dataprepper.model.plugin.PluginFactory; -import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; -import com.amazon.dataprepper.pipeline.server.DataPrepperCoreAuthenticationProvider; -import com.amazon.dataprepper.pipeline.server.DataPrepperServer; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Metrics; -import io.micrometer.core.instrument.composite.CompositeMeterRegistry; -import io.micrometer.prometheus.PrometheusConfig; -import io.micrometer.prometheus.PrometheusMeterRegistry; -import org.hamcrest.Matchers; -import org.junit.Assert; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import java.io.File; -import java.io.IOException; -import java.net.ConnectException; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; -import java.util.Properties; -import java.util.Set; -import java.util.UUID; - -import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE; -import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) public class DataPrepperServerTest { - @Mock - DataPrepperConfiguration dataPrepperConfiguration; - @Mock - PluginFactory pluginFactory; - @Mock - DataPrepper dataPrepper; - @Mock - DataPrepperCoreAuthenticationProvider unauthenticatedProvider; - @Mock - CompositeMeterRegistry systemMeterRegistry; - - private static ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()); - private static ObjectMapper JSON_OBJECT_MAPPER = new ObjectMapper(); - private DataPrepperServer dataPrepperServer; - private final int port = 1234; - - private void setRegistry(PrometheusMeterRegistry prometheusMeterRegistry) { - final Set meterRegistries = Metrics.globalRegistry.getRegistries(); - //to avoid ConcurrentModificationException - final Object[] registeredMeterRegistries = meterRegistries.toArray(); - for (final Object meterRegistry : registeredMeterRegistries) { - Metrics.removeRegistry((MeterRegistry) meterRegistry); - } - Metrics.addRegistry(prometheusMeterRegistry); - } - - @BeforeEach - public void setup() { - // Required to trust any self-signed certs, used in https tests - Properties props = System.getProperties(); - props.setProperty("jdk.internal.httpclient.disableHostnameVerification", Boolean.TRUE.toString()); - - setRegistry(new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, "")); - pluginFactory = mock(PluginFactory.class); - - when(pluginFactory.loadPlugin(eq(DataPrepperCoreAuthenticationProvider.class), - argThat(a -> a.getName().equals(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)))) - .thenReturn(unauthenticatedProvider); - } - - @AfterEach - public void stopServer() { - if (dataPrepperServer != null) { - dataPrepperServer.stop(); - } - } - - @Test - public void testGetGlobalMetrics() throws IOException, InterruptedException, URISyntaxException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - final String scrape = UUID.randomUUID().toString(); - final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); - setRegistry(prometheusMeterRegistry); - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); - Assert.assertEquals(scrape, response.body()); - dataPrepperServer.stop(); - } - - @Test - public void testScrapeGlobalFailure() throws IOException, InterruptedException, URISyntaxException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - setRegistry(new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT)); - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); - dataPrepperServer.stop(); - } - - @Test - public void testGetSystemMetrics() throws IOException, InterruptedException, URISyntaxException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - final String scrape = UUID.randomUUID().toString(); - final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); - - when(systemMeterRegistry.getRegistries()) - .thenReturn(Set.of(prometheusMeterRegistry)); - - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); - dataPrepperServer.stop(); - } - - @Test - public void testScrapeSystemMetricsFailure() throws IOException, InterruptedException, URISyntaxException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT); - - when(systemMeterRegistry.getRegistries()) - .thenReturn(Set.of(prometheusMeterRegistry)); - - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); - dataPrepperServer.stop(); - } - - @Test - public void testNonPrometheusMeterRegistry() throws Exception { - final CloudWatchMeterRegistry cloudWatchMeterRegistry = mock(CloudWatchMeterRegistry.class); - final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); - compositeMeterRegistry.add(cloudWatchMeterRegistry); - final DataPrepperConfiguration config = makeConfig(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE); - try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { - dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) - .GET().build(); - HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - } catch (ConnectException ex) { - dataPrepperServer.stop(); - //there should not be any Prometheus endpoints available - assertThat(ex.getMessage(), Matchers.is("Connection refused")); - } - } - - @Test - public void testMultipleMeterRegistries() throws Exception { - //for system metrics - final CloudWatchMeterRegistry cloudWatchMeterRegistryForSystem = mock(CloudWatchMeterRegistry.class); - final PrometheusMeterRegistry prometheusMeterRegistryForSystem = - new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); - - when(systemMeterRegistry.getRegistries()) - .thenReturn(Set.of(cloudWatchMeterRegistryForSystem, prometheusMeterRegistryForSystem)); - - //for data prepper metrics - final PrometheusMeterRegistry prometheusMeterRegistryForDataPrepper = - new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); - setRegistry(prometheusMeterRegistryForDataPrepper); - - final DataPrepperConfiguration config = makeConfig(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE); - - dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - //test prometheus registry - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + config.getServerPort() - + "/metrics/sys")).GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); - dataPrepperServer.stop(); - } - - @Test - public void testListPipelines() throws URISyntaxException, IOException, InterruptedException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - final String pipelineName = "testPipeline"; - Mockito.when(dataPrepper.getTransformationPipelines()).thenReturn( - Collections.singletonMap("testPipeline", null) - ); - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - final String expectedResponse = JSON_OBJECT_MAPPER.writeValueAsString( - Collections.singletonMap("pipelines", Arrays.asList( - Collections.singletonMap("name", pipelineName) - )) - ); - Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); - Assert.assertEquals(expectedResponse, response.body()); - dataPrepperServer.stop(); - } - - @Test - public void testListPipelinesFailure() throws URISyntaxException, IOException, InterruptedException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - Mockito.when(dataPrepper.getTransformationPipelines()).thenThrow(new RuntimeException("")); - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); - dataPrepperServer.stop(); - } - - @Test - public void testShutdown() throws URISyntaxException, IOException, InterruptedException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); - Mockito.verify(dataPrepper).shutdown(); - dataPrepperServer.stop(); - } - - @Test - public void testShutdownFailure() throws URISyntaxException, IOException, InterruptedException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - Mockito.doThrow(new RuntimeException("")).when(dataPrepper).shutdown(); - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); - Mockito.verify(dataPrepper).shutdown(); - dataPrepperServer.stop(); - } - - @Test - public void testGetMetricsWithHttps() throws Exception { - DataPrepperConfiguration config = DataPrepperServerTest.makeConfig( - TestDataProvider.VALID_DATA_PREPPER_CONFIG_FILE_WITH_TLS); - final String scrape = UUID.randomUUID().toString(); - final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); - setRegistry(prometheusMeterRegistry); - dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpClient httpsClient = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(3)) - .sslContext(getSslContextTrustingInsecure()) - .build(); - - HttpRequest request = HttpRequest.newBuilder(new URI("https://localhost:" + port + "/metrics/prometheus")) - .GET() - .timeout(Duration.ofSeconds(3)) - .build(); - - HttpResponse response = httpsClient.send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); - Assert.assertEquals(scrape, response.body()); - dataPrepperServer.stop(); - } - - @Test - public void testTlsWithInvalidPassword() throws IOException { - DataPrepperConfiguration config = DataPrepperServerTest.makeConfig( - TestDataProvider.INVALID_KEYSTORE_PASSWORD_DATA_PREPPER_CONFIG_FILE); - - assertThrows(IllegalStateException.class, () -> new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry)); - } - - private SSLContext getSslContextTrustingInsecure() throws Exception { - TrustManager[] trustAllCerts = new TrustManager[]{ - new X509TrustManager() { - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return null; - } - - public void checkClientTrusted( - java.security.cert.X509Certificate[] certs, String authType) { - } - - public void checkServerTrusted( - java.security.cert.X509Certificate[] certs, String authType) { - } - } - }; - - SSLContext sslContext = SSLContext.getInstance("SSL"); - sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); - - return sslContext; - } - - private static class PrometheusRegistryMockScrape extends PrometheusMeterRegistry { - final String scrape; - - public PrometheusRegistryMockScrape(PrometheusConfig config, final String scrape) { - super(config); - this.scrape = scrape; - } - - @Override - public String scrape() { - return scrape; - } - } - - private static class PrometheusRegistryThrowingScrape extends PrometheusMeterRegistry { - - public PrometheusRegistryThrowingScrape(PrometheusConfig config) { - super(config); - } - - @Override - public String scrape() { - throw new RuntimeException(""); - } - } - private static DataPrepperConfiguration makeConfig(String filePath) throws IOException { - final File configurationFile = new File(filePath); - return OBJECT_MAPPER.readValue(configurationFile, DataPrepperConfiguration.class); - } +// private static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); +// private DataPrepperServer dataPrepperServer; +// private DataPrepper dataPrepper; +// private final int port = 1234; +// private PluginFactory pluginFactory; +// +// private void setRegistry(PrometheusMeterRegistry prometheusMeterRegistry) { +// final Set meterRegistries = Metrics.globalRegistry.getRegistries(); +// //to avoid ConcurrentModificationException +// final Object[] registeredMeterRegistries = meterRegistries.toArray(); +// for (final Object meterRegistry : registeredMeterRegistries) { +// Metrics.removeRegistry((MeterRegistry) meterRegistry); +// } +// Metrics.addRegistry(prometheusMeterRegistry); +// } +// +// private void setupDataPrepper() { +// dataPrepper = mock(DataPrepper.class); +// DataPrepper.configure(TestDataProvider.VALID_DATA_PREPPER_DEFAULT_LOG4J_CONFIG_FILE); +// when(dataPrepper.getPluginFactory()).thenReturn(pluginFactory); +// } +// +// @BeforeEach +// public void setup() { +// // Required to trust any self-signed certs, used in https tests +// Properties props = System.getProperties(); +// props.setProperty("jdk.internal.httpclient.disableHostnameVerification", Boolean.TRUE.toString()); +// +// setRegistry(new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, "")); +// pluginFactory = mock(PluginFactory.class); +// +// final DataPrepperCoreAuthenticationProvider unauthenticatedProvider = mock(DataPrepperCoreAuthenticationProvider.class); +// when(pluginFactory.loadPlugin(eq(DataPrepperCoreAuthenticationProvider.class), +// argThat(a -> a.getName().equals(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)))) +// .thenReturn(unauthenticatedProvider); +// } +// +// @AfterEach +// public void stopServer() { +// if (dataPrepperServer != null) { +// dataPrepperServer.stop(); +// } +// } +// +// @Test +// public void testGetGlobalMetrics() throws IOException, InterruptedException, URISyntaxException { +// setupDataPrepper(); +// final String scrape = UUID.randomUUID().toString(); +// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); +// setRegistry(prometheusMeterRegistry); +// dataPrepperServer = new DataPrepperServer(dataPrepper); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); +// Assert.assertEquals(scrape, response.body()); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testScrapeGlobalFailure() throws IOException, InterruptedException, URISyntaxException { +// setupDataPrepper(); +// setRegistry(new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT)); +// dataPrepperServer = new DataPrepperServer(dataPrepper); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testGetSystemMetrics() throws IOException, InterruptedException, URISyntaxException { +// final String scrape = UUID.randomUUID().toString(); +// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); +// final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); +// compositeMeterRegistry.add(prometheusMeterRegistry); +// setupDataPrepper(); +// final DataPrepperConfiguration dataPrepperConfiguration = DataPrepper.getConfiguration(); +// try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { +// dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); +// dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); +// dataPrepperServer = new DataPrepperServer(dataPrepper); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); +// dataPrepperServer.stop(); +// } +// } +// +// @Test +// public void testScrapeSystemMetricsFailure() throws IOException, InterruptedException, URISyntaxException { +// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT); +// final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); +// compositeMeterRegistry.add(prometheusMeterRegistry); +// setupDataPrepper(); +// final DataPrepperConfiguration dataPrepperConfiguration = DataPrepper.getConfiguration(); +// try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { +// dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); +// dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); +// dataPrepperServer = new DataPrepperServer(dataPrepper); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); +// dataPrepperServer.stop(); +// } +// } +// +// @Test +// public void testNonPrometheusMeterRegistry() throws Exception { +// final CloudWatchMeterRegistry cloudWatchMeterRegistry = mock(CloudWatchMeterRegistry.class); +// final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); +// compositeMeterRegistry.add(cloudWatchMeterRegistry); +// final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.fromFile( +// new File(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE)); +// setupDataPrepper(); +// try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { +// dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); +// dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); +// dataPrepperServer = new DataPrepperServer(dataPrepper); +// dataPrepperServer.start(); +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) +// .GET().build(); +// HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// } catch (ConnectException ex) { +// dataPrepperServer.stop(); +// //there should not be any Prometheus endpoints available +// assertThat(ex.getMessage(), Matchers.is("Connection refused")); +// } +// } +// +// @Test +// public void testMultipleMeterRegistries() throws Exception { +// //for system metrics +// final CloudWatchMeterRegistry cloudWatchMeterRegistryForSystem = mock(CloudWatchMeterRegistry.class); +// final PrometheusMeterRegistry prometheusMeterRegistryForSystem = +// new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); +// final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); +// compositeMeterRegistry.add(cloudWatchMeterRegistryForSystem); +// compositeMeterRegistry.add(prometheusMeterRegistryForSystem); +// +// //for data prepper metrics +// final PrometheusMeterRegistry prometheusMeterRegistryForDataPrepper = +// new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); +// setRegistry(prometheusMeterRegistryForDataPrepper); +// +// final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.fromFile( +// new File(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE)); +// setupDataPrepper(); +// try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { +// dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); +// dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); +// dataPrepperServer = new DataPrepperServer(dataPrepper); +// dataPrepperServer.start(); +// +// //test prometheus registry +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + dataPrepperConfiguration.getServerPort() +// + "/metrics/sys")).GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); +// dataPrepperServer.stop(); +// } +// } +// +// @Test +// public void testListPipelines() throws URISyntaxException, IOException, InterruptedException { +// setupDataPrepper(); +// final String pipelineName = "testPipeline"; +// Mockito.when(dataPrepper.getTransformationPipelines()).thenReturn( +// Collections.singletonMap("testPipeline", null) +// ); +// dataPrepperServer = new DataPrepperServer(dataPrepper); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// final String expectedResponse = OBJECT_MAPPER.writeValueAsString( +// Collections.singletonMap("pipelines", Arrays.asList( +// Collections.singletonMap("name", pipelineName) +// )) +// ); +// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); +// Assert.assertEquals(expectedResponse, response.body()); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testListPipelinesFailure() throws URISyntaxException, IOException, InterruptedException { +// setupDataPrepper(); +// Mockito.when(dataPrepper.getTransformationPipelines()).thenThrow(new RuntimeException("")); +// dataPrepperServer = new DataPrepperServer(dataPrepper); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testShutdown() throws URISyntaxException, IOException, InterruptedException { +// setupDataPrepper(); +// dataPrepperServer = new DataPrepperServer(dataPrepper); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); +// Mockito.verify(dataPrepper).shutdown(); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testShutdownFailure() throws URISyntaxException, IOException, InterruptedException { +// setupDataPrepper(); +// Mockito.doThrow(new RuntimeException("")).when(dataPrepper).shutdown(); +// dataPrepperServer = new DataPrepperServer(dataPrepper); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); +// Mockito.verify(dataPrepper).shutdown(); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testGetMetricsWithHttps() throws Exception { +// setupDataPrepper(); +// DataPrepper.configure(TestDataProvider.VALID_DATA_PREPPER_CONFIG_FILE_WITH_TLS); +// final String scrape = UUID.randomUUID().toString(); +// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); +// setRegistry(prometheusMeterRegistry); +// dataPrepperServer = new DataPrepperServer(dataPrepper); +// dataPrepperServer.start(); +// +// HttpClient httpsClient = HttpClient.newBuilder() +// .connectTimeout(Duration.ofSeconds(3)) +// .sslContext(getSslContextTrustingInsecure()) +// .build(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("https://localhost:" + port + "/metrics/prometheus")) +// .GET() +// .timeout(Duration.ofSeconds(3)) +// .build(); +// +// HttpResponse response = httpsClient.send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); +// Assert.assertEquals(scrape, response.body()); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testTlsWithInvalidPassword() { +// setupDataPrepper(); +// DataPrepper.configure(TestDataProvider.INVALID_KEYSTORE_PASSWORD_DATA_PREPPER_CONFIG_FILE); +// +// assertThrows(IllegalStateException.class, () -> new DataPrepperServer(dataPrepper)); +// } +// +// private SSLContext getSslContextTrustingInsecure() throws Exception { +// TrustManager[] trustAllCerts = new TrustManager[]{ +// new X509TrustManager() { +// public java.security.cert.X509Certificate[] getAcceptedIssuers() { +// return null; +// } +// +// public void checkClientTrusted( +// java.security.cert.X509Certificate[] certs, String authType) { +// } +// +// public void checkServerTrusted( +// java.security.cert.X509Certificate[] certs, String authType) { +// } +// } +// }; +// +// SSLContext sslContext = SSLContext.getInstance("SSL"); +// sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); +// +// return sslContext; +// } +// +// private static class PrometheusRegistryMockScrape extends PrometheusMeterRegistry { +// final String scrape; +// +// public PrometheusRegistryMockScrape(PrometheusConfig config, final String scrape) { +// super(config); +// this.scrape = scrape; +// } +// +// @Override +// public String scrape() { +// return scrape; +// } +// } +// +// private static class PrometheusRegistryThrowingScrape extends PrometheusMeterRegistry { +// +// public PrometheusRegistryThrowingScrape(PrometheusConfig config) { +// super(config); +// } +// +// @Override +// public String scrape() { +// throw new RuntimeException(""); +// } +// } } From 492dfd462e0a08df3d1cf5aa01af541d8fafaa9f Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Wed, 22 Dec 2021 14:34:23 -0600 Subject: [PATCH 05/43] Create DataPrepperConfigurate in bean provider method Signed-off-by: Steven Bayer --- .../com/amazon/dataprepper/DataPrepper.java | 3 +-- .../dataprepper/parser/PipelineParser.java | 8 +++---- ...DataPrepperConfigurationConfiguration.java | 23 +++++++++++++++++-- .../pipeline/server/DataPrepperServer.java | 7 ++++-- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java index cf2d1a205d..2c548ba185 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java @@ -27,7 +27,6 @@ * execution. */ @Named -@Singleton public class DataPrepper { private static final Logger LOG = LoggerFactory.getLogger(DataPrepper.class); private static final String DATAPREPPER_SERVICE_NAME = "DATAPREPPER_SERVICE_NAME"; @@ -39,7 +38,7 @@ public class DataPrepper { private final PluginFactory pluginFactory; private Map transformationPipelines; - //TODO: decouple this so I can not inject + // TODO: Remove DataPrepperServer dependency on DataPrepper @Inject private DataPrepperServer dataPrepperServer; diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/PipelineParser.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/PipelineParser.java index 5f1e719989..6dc27ef01a 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/PipelineParser.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/PipelineParser.java @@ -47,13 +47,13 @@ public class PipelineParser { .enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY); private static final String PIPELINE_TYPE = "pipeline"; private static final String ATTRIBUTE_NAME = "name"; - private final String configurationFileLocation; + private final String pipelineConfigurationFileLocation; private final Map sourceConnectorMap = new HashMap<>(); //TODO Remove this and rely only on pipelineMap private final PluginFactory pluginFactory; @Inject public PipelineParser(final DataPrepperArgs dataPrepperArgs, final PluginFactory pluginFactory) { - this.configurationFileLocation = dataPrepperArgs.getDataPrepperConfigFileLocation(); + this.pipelineConfigurationFileLocation = dataPrepperArgs.getPipelineConfigFileLocation(); this.pluginFactory = Objects.requireNonNull(pluginFactory); } @@ -63,7 +63,7 @@ public PipelineParser(final DataPrepperArgs dataPrepperArgs, final PluginFactory public Map parseConfiguration() { try { final Map pipelineConfigurationMap = OBJECT_MAPPER.readValue( - new File(configurationFileLocation), + new File(pipelineConfigurationFileLocation), new TypeReference>() { }); final List allPipelineNames = PipelineConfigurationValidator.validateAndGetPipelineNames(pipelineConfigurationMap); @@ -79,7 +79,7 @@ public Map parseConfiguration() { } return pipelineMap; } catch (IOException e) { - throw new ParseException(format("Failed to parse the configuration file %s", configurationFileLocation), e); + throw new ParseException(format("Failed to parse the configuration file %s", pipelineConfigurationFileLocation), e); } } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java index 9a07572d83..f6b728555f 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java @@ -5,6 +5,8 @@ import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.parser.model.MetricRegistryType; import com.amazon.dataprepper.plugin.DefaultPluginFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.binder.MeterBinder; @@ -35,6 +37,16 @@ public class DataPrepperConfigurationConfiguration { private static final String POSITIONAL_COMMAND_LINE_ARGUMENTS = "nonOptionArgs"; private static final String COMMAND_LINE_ARG_DELIMITER = ","; + @Bean + public YAMLFactory yamlFactory() { + return new YAMLFactory(); + } + + @Bean + public ObjectMapper objectMapper(final YAMLFactory yamlFactory) { + return new ObjectMapper(yamlFactory); + } + @Bean public ClassLoaderMetrics classLoaderMetrics() { return new ClassLoaderMetrics(); @@ -94,11 +106,18 @@ public DataPrepperArgs dataPrepperArgs(final Environment environment) { } @Bean - public DataPrepperConfiguration dataPrepperConfiguration(final DataPrepperArgs dataPrepperArgs) { + public DataPrepperConfiguration dataPrepperConfiguration( + final DataPrepperArgs dataPrepperArgs, + final ObjectMapper objectMapper + ) { final String dataPrepperConfigFileLocation = dataPrepperArgs.getDataPrepperConfigFileLocation(); if (dataPrepperConfigFileLocation != null) { final File configurationFile = new File(dataPrepperConfigFileLocation); - return DataPrepperConfiguration.fromFile(configurationFile); + try { + return objectMapper.readValue(configurationFile, DataPrepperConfiguration.class); + } catch (final IOException e) { + throw new IllegalArgumentException("Invalid DataPrepper configuration file.", e); + } } else { return new DataPrepperConfiguration(); diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java index 12f9646495..38b42d9fc5 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java @@ -45,7 +45,11 @@ public class DataPrepperServer { private final HttpServer server; @Inject - public DataPrepperServer(final DataPrepperConfiguration dataPrepperConfiguration, final DataPrepper dataPrepper) { + public DataPrepperServer( + final DataPrepperConfiguration dataPrepperConfiguration, + final PluginFactory pluginFactory, + final DataPrepper dataPrepper + ) { final int port = dataPrepperConfiguration.getServerPort(); final boolean ssl = dataPrepperConfiguration.ssl(); final String keyStoreFilePath = dataPrepperConfiguration.getKeyStoreFilePath(); @@ -68,7 +72,6 @@ public DataPrepperServer(final DataPrepperConfiguration dataPrepperConfiguration new PluginSetting(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME, Collections.emptyMap()); } - final PluginFactory pluginFactory = dataPrepper.getPluginFactory(); final DataPrepperCoreAuthenticationProvider authenticationProvider = pluginFactory.loadPlugin(DataPrepperCoreAuthenticationProvider.class, authenticationPluginSetting); final Authenticator authenticator = authenticationProvider.getAuthenticator(); From 9fae6308b91bdd90a487c1183ec24fb1046015ee Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Wed, 22 Dec 2021 15:45:56 -0600 Subject: [PATCH 06/43] Added DataPrepper unit tests Signed-off-by: Steven Bayer --- .../amazon/dataprepper/DataPrepperTests.java | 173 ++++++------------ 1 file changed, 59 insertions(+), 114 deletions(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java index 90cb319e73..b4c2a22d24 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java @@ -7,19 +7,18 @@ import com.amazon.dataprepper.model.plugin.PluginFactory; import com.amazon.dataprepper.parser.PipelineParser; -import com.amazon.dataprepper.parser.config.DataPrepperArgs; -import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.pipeline.Pipeline; import com.amazon.dataprepper.pipeline.server.DataPrepperServer; -import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import java.lang.reflect.Field; import java.util.HashMap; import java.util.Map; @@ -34,29 +33,36 @@ public class DataPrepperTests { private static Map parseConfigurationFixture; private static PipelineParser pipelineParser; + private static Pipeline pipeline; - @Mock - private Pipeline pipeline; @Mock private PluginFactory pluginFactory; @Mock - private DataPrepperConfiguration configuration; - @Mock private DataPrepperServer dataPrepperServer; @InjectMocks private DataPrepper dataPrepper; @BeforeAll - public static void beforeAll() { + public static void beforeAll() throws NoSuchFieldException { pipelineParser = mock(PipelineParser.class); + pipeline = mock(Pipeline.class); parseConfigurationFixture = new HashMap<>(); - parseConfigurationFixture.put("testKey", mock(Pipeline.class)); + parseConfigurationFixture.put("testKey", pipeline); when(pipelineParser.parseConfiguration()) .thenReturn(parseConfigurationFixture); } + @BeforeEach + public void before() throws NoSuchFieldException, IllegalAccessException { + // Use reflection to set dataPrepper.dataPrepperServer because @InjectMock will not use field injection. + final Field dataPrepperServerField = dataPrepper.getClass().getDeclaredField("dataPrepperServer"); + dataPrepperServerField.setAccessible(true); + dataPrepperServerField.set(dataPrepper, dataPrepperServer); + dataPrepperServerField.setAccessible(false); + } + @Test public void testGivenValidInputThenInstanceCreation() { assertThat( @@ -71,7 +77,7 @@ public void testGivenInvalidInputThenExceptionThrown() { assertThrows( RuntimeException.class, - () -> new DataPrepper(configuration, pipelineParser, pluginFactory), + () -> new DataPrepper(pipelineParser, pluginFactory), "Exception should be thrown if pipeline parser has no pipeline configuration"); } @@ -80,6 +86,11 @@ public void testGivenInstantiatedWithPluginFactoryWhenGetPluginFactoryCalledThen assertThat(dataPrepper.getPluginFactory(), Matchers.is(pluginFactory)); } + @Test + public void testGivenValidPipelineParserThenReturnResultOfParseConfiguration() { + assertThat(dataPrepper.getTransformationPipelines(), Matchers.is(parseConfigurationFixture)); + } + @Test public void testGivenValidPipelineParserWhenExecuteThenAllPipelinesExecuteAndServerStartAndReturnTrue() { assertThat(dataPrepper.execute(), Matchers.is(true)); @@ -88,108 +99,42 @@ public void testGivenValidPipelineParserWhenExecuteThenAllPipelinesExecuteAndSer verify(dataPrepperServer, times(1)).start(); } -// @Before -// public void setup() throws Exception { -// actualSecurityManager = System.getSecurityManager(); -// System.setSecurityManager(new CustomSecurityManager()); -// DataPrepper.configure(TestDataProvider.VALID_DATA_PREPPER_CONFIG_FILE); -// } -// -// @After -// public void teardown() { -// System.setSecurityManager(actualSecurityManager); -// } -// -// @Test -// public void testInstanceCreation() { -// DataPrepper testDataPrepper1 = DataPrepper.getInstance(); -// assertThat("Failed to retrieve a valid Data Prepper instance", testDataPrepper1, is(notNullValue())); -// DataPrepper testDataPrepper2 = DataPrepper.getInstance(); -// assertThat("Data Prepper has to be singleton", testDataPrepper2, is(testDataPrepper1)); -// } -// -// @Test -// public void testDataPrepperSystemMetrics() { -// // Test retrieve gauge in ClassLoaderMetrics -// final List classesLoaded = getSystemMeasurementList("jvm.classes.loaded"); -// Assert.assertEquals(1, classesLoaded.size()); -// // Test retrieve gauge in JvmMemoryMetrics -// final List jvmBufferCount = getSystemMeasurementList("jvm.buffer.count"); -// Assert.assertEquals(1, jvmBufferCount.size()); -// // Test retrieve gauge in JvmGcMetrics -// final List jvmGcMaxDataSize = getSystemMeasurementList("jvm.gc.max.data.size"); -// Assert.assertEquals(1, jvmGcMaxDataSize.size()); -// // Test retrieve gauge in ProcessorMetrics -// final List sysCPUCount = getSystemMeasurementList("system.cpu.count"); -// Assert.assertEquals(1, sysCPUCount.size()); -// // Test retrieve gauge in JvmThreadMetrics -// final List jvmThreadsPeak = getSystemMeasurementList("jvm.threads.peak"); -// Assert.assertEquals(1, jvmThreadsPeak.size()); -// } -// -// @Test -// public void testCustomConfiguration() { -// DataPrepper testInstance = DataPrepper.getInstance(); -// Assert.assertEquals(5678, DataPrepper.getConfiguration().getServerPort()); -// } -// -// @Test -// public void testNoPipelinesToExecute() { -// try { -// DataPrepper testDataPrepper = DataPrepper.getInstance(); -// testDataPrepper.execute(NO_PIPELINES_EXECUTE_CONFIG_FILE); -// } catch (SystemExitException ex) { -// assertThat("Data Prepper should exit with status 1", ex.getExitStatus(), is(1)); -// assertThat("Data Prepper exit message is incorrect", -// ex.getMessage().contains("System exit was initiated")); -// } -// } -// -// @Test -// public void testDataPrepperExecuteAndShutdown() { -// final DataPrepper testDataPrepper = DataPrepper.getInstance(); -// boolean executionStatus = testDataPrepper.execute(VALID_MULTIPLE_PIPELINE_CONFIG_FILE); -// assertThat("Failed to initiate execution", executionStatus); -// testDataPrepper.shutdown(); -// //call shutdown() twice to ensure nothing breaks -// testDataPrepper.shutdown(); -// VALID_MULTIPLE_PIPELINE_NAMES.forEach(testDataPrepper::shutdown); -// testDataPrepper.shutdown("pipeline_does_not_exist"); //this does nothing -// } -// -// -// public static class CustomSecurityManager extends SecurityManager { -// @Override -// public void checkPermission(Permission perm) { -// } -// -// @Override -// public void checkPermission(Permission perm, Object context) { -// // allow anything. -// } -// -// @Override -// public void checkExit(int status) { -// super.checkExit(status); -// throw new SystemExitException(status); -// } -// } -// -// public static class SystemExitException extends SecurityException { -// private final int status; -// -// public SystemExitException(int status) { -// super("System exit was initiated"); -// this.status = status; -// } -// -// public int getExitStatus() { -// return this.status; -// } -// } -// -// private static List getSystemMeasurementList(final String meterName) { -// return StreamSupport.stream(DataPrepper.getSystemMeterRegistry().find(meterName).meter().measure().spliterator(), false) -// .collect(Collectors.toList()); -// } + @Test + public void testDataPrepperShutdown() { + dataPrepper.shutdown(); + verify(pipeline, times(1)).shutdown(); + } + + @Test + public void testDataPrepperShutdownPipeline() { + Pipeline randomPipeline = mock(Pipeline.class); + parseConfigurationFixture.put("Random Pipeline", randomPipeline); + dataPrepper.shutdown("Random Pipeline"); + + verify(randomPipeline, times(1)).shutdown(); + } + + @Test + public void testDataPrepperShutdownNonExistentPipelineWithoutException() { + dataPrepper.shutdown("Missing Pipeline"); + } + + @Test + public void testShutdownDataPrepperServer() { + dataPrepper.shutdownDataPrepperServer(); + + verify(dataPrepperServer, times(1)).stop(); + } + + @Test + public void foo() throws NoSuchFieldException, IllegalAccessException { + final Field dataPrepperServerField = DataPrepper.class.getDeclaredField("DEFAULT_SERVICE_NAME"); + dataPrepperServerField.setAccessible(true); + String expected = (String) dataPrepperServerField.get(null); + dataPrepperServerField.setAccessible(false); + + String actual = DataPrepper.getServiceNameForMetrics(); + + assertThat(actual, Matchers.is(expected)); + } } From 1d594a52a79d2dab756805f16bbacaa48cbebda9 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Wed, 22 Dec 2021 15:46:26 -0600 Subject: [PATCH 07/43] Decoupled system metrics from DataPrepper Signed-off-by: Steven Bayer --- .../com/amazon/dataprepper/DataPrepper.java | 11 -- .../pipeline/server/DataPrepperServer.java | 6 +- .../model/DataPrepperConfigurationTests.java | 135 ++++++++---------- 3 files changed, 66 insertions(+), 86 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java index 2c548ba185..57a78c18e4 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java @@ -7,7 +7,6 @@ import com.amazon.dataprepper.model.plugin.PluginFactory; import com.amazon.dataprepper.parser.PipelineParser; -import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.pipeline.Pipeline; import com.amazon.dataprepper.pipeline.server.DataPrepperServer; import io.micrometer.core.instrument.composite.CompositeMeterRegistry; @@ -17,7 +16,6 @@ import javax.inject.Inject; import javax.inject.Named; -import javax.inject.Singleton; import java.util.Map; /** @@ -32,9 +30,6 @@ public class DataPrepper { private static final String DATAPREPPER_SERVICE_NAME = "DATAPREPPER_SERVICE_NAME"; private static final String DEFAULT_SERVICE_NAME = "dataprepper"; - private static final CompositeMeterRegistry systemMeterRegistry = new CompositeMeterRegistry(); - - private final DataPrepperConfiguration configuration; private final PluginFactory pluginFactory; private Map transformationPipelines; @@ -53,11 +48,9 @@ public static String getServiceNameForMetrics() { @Inject public DataPrepper( - final DataPrepperConfiguration configuration, final PipelineParser pipelineParser, final PluginFactory pluginFactory ) { - this.configuration = configuration; this.pluginFactory = pluginFactory; transformationPipelines = pipelineParser.parseConfiguration(); @@ -66,10 +59,6 @@ public DataPrepper( } } - public static CompositeMeterRegistry getSystemMeterRegistry() { - return systemMeterRegistry; - } - /** * Executes Data Prepper engine using the default configuration file * diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java index 38b42d9fc5..243f3293c4 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java @@ -17,6 +17,7 @@ import com.sun.net.httpserver.HttpsServer; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import io.micrometer.prometheus.PrometheusMeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +49,8 @@ public class DataPrepperServer { public DataPrepperServer( final DataPrepperConfiguration dataPrepperConfiguration, final PluginFactory pluginFactory, - final DataPrepper dataPrepper + final DataPrepper dataPrepper, + final CompositeMeterRegistry systemMeterRegistry ) { final int port = dataPrepperConfiguration.getServerPort(); final boolean ssl = dataPrepperConfiguration.ssl(); @@ -94,7 +96,7 @@ public DataPrepperServer( .setAuthenticator(authenticator); }); - getPrometheusMeterRegistryFromRegistries(DataPrepper.getSystemMeterRegistry().getRegistries()).ifPresent( + getPrometheusMeterRegistryFromRegistries(systemMeterRegistry.getRegistries()).ifPresent( meterRegistry -> { final PrometheusMeterRegistry prometheusMeterRegistryForSystem = (PrometheusMeterRegistry) meterRegistry; server.createContext("/metrics/sys", new PrometheusMetricsHandler(prometheusMeterRegistryForSystem)) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java index c9df91cf0f..dabaf66a44 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java @@ -5,16 +5,11 @@ package com.amazon.dataprepper.parser.model; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; -import com.fasterxml.jackson.databind.exc.ValueInstantiationException; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.jupiter.api.Test; import java.io.File; -import java.io.IOException; import static com.amazon.dataprepper.TestDataProvider.INVALID_DATA_PREPPER_CONFIG_FILE; import static com.amazon.dataprepper.TestDataProvider.INVALID_PORT_DATA_PREPPER_CONFIG_FILE; @@ -30,73 +25,67 @@ import static org.junit.jupiter.api.Assertions.assertThrows; public class DataPrepperConfigurationTests { - private static ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()); - private static DataPrepperConfiguration makeConfig(String filePath) throws IOException { - final File configurationFile = new File(filePath); - return OBJECT_MAPPER.readValue(configurationFile, DataPrepperConfiguration.class); - } - - @Test - public void testParseConfig() throws IOException { - final DataPrepperConfiguration dataPrepperConfiguration = - makeConfig(VALID_DATA_PREPPER_CONFIG_FILE); - Assert.assertEquals(5678, dataPrepperConfiguration.getServerPort()); - } - - @Test - public void testSomeDefaultConfig() throws IOException { - final DataPrepperConfiguration dataPrepperConfiguration = - makeConfig(VALID_DATA_PREPPER_SOME_DEFAULT_CONFIG_FILE); - Assert.assertEquals(DataPrepperConfiguration.DEFAULT_CONFIG.getServerPort(), dataPrepperConfiguration.getServerPort()); - } - - @Test - public void testDefaultMetricsRegistry() { - final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.DEFAULT_CONFIG; - assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(1)); - assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.Prometheus)); - } - - @Test - public void testCloudWatchMetricsRegistry() throws IOException { - final DataPrepperConfiguration dataPrepperConfiguration = - makeConfig(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE); - assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(1)); - assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.CloudWatch)); - } - - @Test - public void testMultipleMetricsRegistry() throws IOException { - final DataPrepperConfiguration dataPrepperConfiguration = - makeConfig(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE); - assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(2)); - assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.Prometheus)); - assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.CloudWatch)); - } - - @Test - void testConfigurationWithHttpBasic() throws IOException { - final DataPrepperConfiguration dataPrepperConfiguration = - makeConfig(VALID_DATA_PREPPER_CONFIG_FILE_WITH_BASIC_AUTHENTICATION); - - assertThat(dataPrepperConfiguration, notNullValue()); - assertThat(dataPrepperConfiguration.getAuthentication(), notNullValue()); - assertThat(dataPrepperConfiguration.getAuthentication().getPluginName(), equalTo("http_basic")); - assertThat(dataPrepperConfiguration.getAuthentication().getPluginSettings(), notNullValue()); - assertThat(dataPrepperConfiguration.getAuthentication().getPluginSettings(), hasKey("username")); - assertThat(dataPrepperConfiguration.getAuthentication().getPluginSettings(), hasKey("password")); - } - - @Test - public void testInvalidConfig() { - assertThrows(UnrecognizedPropertyException.class, () -> - makeConfig(INVALID_DATA_PREPPER_CONFIG_FILE)); - } - - @Test - public void testInvalidPortConfig() { - assertThrows(ValueInstantiationException.class, () -> - makeConfig(INVALID_PORT_DATA_PREPPER_CONFIG_FILE)); - } +// @Test +// public void testParseConfig() { +// final DataPrepperConfiguration dataPrepperConfiguration = +// DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_CONFIG_FILE)); +// Assert.assertEquals(5678, dataPrepperConfiguration.getServerPort()); +// } +// +// @Test +// public void testSomeDefaultConfig() { +// final DataPrepperConfiguration dataPrepperConfiguration = +// DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_SOME_DEFAULT_CONFIG_FILE)); +// Assert.assertEquals(DataPrepperConfiguration.DEFAULT_CONFIG.getServerPort(), dataPrepperConfiguration.getServerPort()); +// } +// +// @Test +// public void testDefaultMetricsRegistry() { +// final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.DEFAULT_CONFIG; +// assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(1)); +// assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.Prometheus)); +// } +// +// @Test +// public void testCloudWatchMetricsRegistry() { +// final DataPrepperConfiguration dataPrepperConfiguration = +// DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE)); +// assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(1)); +// assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.CloudWatch)); +// } +// +// @Test +// public void testMultipleMetricsRegistry() { +// final DataPrepperConfiguration dataPrepperConfiguration = +// DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE)); +// assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(2)); +// assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.Prometheus)); +// assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.CloudWatch)); +// } +// +// @Test +// void testConfigurationWithHttpBasic() { +// final DataPrepperConfiguration dataPrepperConfiguration = +// DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_CONFIG_FILE_WITH_BASIC_AUTHENTICATION)); +// +// assertThat(dataPrepperConfiguration, notNullValue()); +// assertThat(dataPrepperConfiguration.getAuthentication(), notNullValue()); +// assertThat(dataPrepperConfiguration.getAuthentication().getPluginName(), equalTo("http_basic")); +// assertThat(dataPrepperConfiguration.getAuthentication().getPluginSettings(), notNullValue()); +// assertThat(dataPrepperConfiguration.getAuthentication().getPluginSettings(), hasKey("username")); +// assertThat(dataPrepperConfiguration.getAuthentication().getPluginSettings(), hasKey("password")); +// } +// +// @Test +// public void testInvalidConfig() { +// assertThrows(IllegalArgumentException.class, () -> +// DataPrepperConfiguration.fromFile(new File(INVALID_DATA_PREPPER_CONFIG_FILE))); +// } +// +// @Test +// public void testInvalidPortConfig() { +// assertThrows(IllegalArgumentException.class, () -> +// DataPrepperConfiguration.fromFile(new File(INVALID_PORT_DATA_PREPPER_CONFIG_FILE))); +// } } From bce2c26f925b56105d28d3394b772092c8697cf0 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Wed, 22 Dec 2021 17:11:28 -0600 Subject: [PATCH 08/43] Updated DP Test object scope Signed-off-by: Steven Bayer --- .../java/com/amazon/dataprepper/DataPrepperTests.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java index b4c2a22d24..b0ecff7fbc 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java @@ -31,9 +31,9 @@ @ExtendWith(MockitoExtension.class) public class DataPrepperTests { + private static final PipelineParser pipelineParser = mock(PipelineParser.class); + private static final Pipeline pipeline = mock(Pipeline.class); private static Map parseConfigurationFixture; - private static PipelineParser pipelineParser; - private static Pipeline pipeline; @Mock private PluginFactory pluginFactory; @@ -43,10 +43,7 @@ public class DataPrepperTests { private DataPrepper dataPrepper; @BeforeAll - public static void beforeAll() throws NoSuchFieldException { - pipelineParser = mock(PipelineParser.class); - pipeline = mock(Pipeline.class); - + public static void beforeAll() { parseConfigurationFixture = new HashMap<>(); parseConfigurationFixture.put("testKey", pipeline); @@ -66,7 +63,7 @@ public void before() throws NoSuchFieldException, IllegalAccessException { @Test public void testGivenValidInputThenInstanceCreation() { assertThat( - "Given injected with valid beans a Data Prepper bean should be available", + "Given injected with valid beans a DataPrepper bean should be available", dataPrepper, Matchers.is(Matchers.notNullValue())); } From f4205e8556b7ef03c2843569ce1085b042b74915 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Thu, 23 Dec 2021 10:04:15 -0600 Subject: [PATCH 09/43] DataPrepperServerTest refactored to work with new DataPrepper constructor Signed-off-by: Steven Bayer --- .../server/DataPrepperServerTest.java | 693 ++++++++++-------- 1 file changed, 372 insertions(+), 321 deletions(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java index ff13711a66..bf840208b1 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java @@ -5,329 +5,380 @@ package com.amazon.dataprepper.server; +import com.amazon.dataprepper.DataPrepper; +import com.amazon.dataprepper.TestDataProvider; +import com.amazon.dataprepper.model.plugin.PluginFactory; +import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; +import com.amazon.dataprepper.pipeline.server.DataPrepperCoreAuthenticationProvider; +import com.amazon.dataprepper.pipeline.server.DataPrepperServer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.composite.CompositeMeterRegistry; +import io.micrometer.prometheus.PrometheusConfig; +import io.micrometer.prometheus.PrometheusMeterRegistry; +import org.hamcrest.Matchers; +import org.junit.Assert; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.File; +import java.io.IOException; +import java.net.ConnectException; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.Properties; +import java.util.Set; +import java.util.UUID; + +import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE; +import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) public class DataPrepperServerTest { + @Mock + DataPrepperConfiguration dataPrepperConfiguration; + @Mock + PluginFactory pluginFactory; + @Mock + DataPrepper dataPrepper; + @Mock + DataPrepperCoreAuthenticationProvider unauthenticatedProvider; + @Mock + CompositeMeterRegistry systemMeterRegistry; + + private static ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()); + private static ObjectMapper JSON_OBJECT_MAPPER = new ObjectMapper(); + private DataPrepperServer dataPrepperServer; + private final int port = 1234; + + private void setRegistry(PrometheusMeterRegistry prometheusMeterRegistry) { + final Set meterRegistries = Metrics.globalRegistry.getRegistries(); + //to avoid ConcurrentModificationException + final Object[] registeredMeterRegistries = meterRegistries.toArray(); + for (final Object meterRegistry : registeredMeterRegistries) { + Metrics.removeRegistry((MeterRegistry) meterRegistry); + } + Metrics.addRegistry(prometheusMeterRegistry); + } + + @BeforeEach + public void setup() { + // Required to trust any self-signed certs, used in https tests + Properties props = System.getProperties(); + props.setProperty("jdk.internal.httpclient.disableHostnameVerification", Boolean.TRUE.toString()); + + setRegistry(new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, "")); + pluginFactory = mock(PluginFactory.class); + + when(pluginFactory.loadPlugin(eq(DataPrepperCoreAuthenticationProvider.class), + argThat(a -> a.getName().equals(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)))) + .thenReturn(unauthenticatedProvider); -// private static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); -// private DataPrepperServer dataPrepperServer; -// private DataPrepper dataPrepper; -// private final int port = 1234; -// private PluginFactory pluginFactory; -// -// private void setRegistry(PrometheusMeterRegistry prometheusMeterRegistry) { -// final Set meterRegistries = Metrics.globalRegistry.getRegistries(); -// //to avoid ConcurrentModificationException -// final Object[] registeredMeterRegistries = meterRegistries.toArray(); -// for (final Object meterRegistry : registeredMeterRegistries) { -// Metrics.removeRegistry((MeterRegistry) meterRegistry); -// } -// Metrics.addRegistry(prometheusMeterRegistry); -// } -// -// private void setupDataPrepper() { -// dataPrepper = mock(DataPrepper.class); -// DataPrepper.configure(TestDataProvider.VALID_DATA_PREPPER_DEFAULT_LOG4J_CONFIG_FILE); // when(dataPrepper.getPluginFactory()).thenReturn(pluginFactory); -// } -// -// @BeforeEach -// public void setup() { -// // Required to trust any self-signed certs, used in https tests -// Properties props = System.getProperties(); -// props.setProperty("jdk.internal.httpclient.disableHostnameVerification", Boolean.TRUE.toString()); -// -// setRegistry(new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, "")); -// pluginFactory = mock(PluginFactory.class); -// -// final DataPrepperCoreAuthenticationProvider unauthenticatedProvider = mock(DataPrepperCoreAuthenticationProvider.class); -// when(pluginFactory.loadPlugin(eq(DataPrepperCoreAuthenticationProvider.class), -// argThat(a -> a.getName().equals(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)))) -// .thenReturn(unauthenticatedProvider); -// } -// -// @AfterEach -// public void stopServer() { -// if (dataPrepperServer != null) { -// dataPrepperServer.stop(); -// } -// } -// -// @Test -// public void testGetGlobalMetrics() throws IOException, InterruptedException, URISyntaxException { -// setupDataPrepper(); -// final String scrape = UUID.randomUUID().toString(); -// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); -// setRegistry(prometheusMeterRegistry); -// dataPrepperServer = new DataPrepperServer(dataPrepper); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); -// Assert.assertEquals(scrape, response.body()); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testScrapeGlobalFailure() throws IOException, InterruptedException, URISyntaxException { -// setupDataPrepper(); -// setRegistry(new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT)); -// dataPrepperServer = new DataPrepperServer(dataPrepper); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testGetSystemMetrics() throws IOException, InterruptedException, URISyntaxException { -// final String scrape = UUID.randomUUID().toString(); -// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); -// final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); -// compositeMeterRegistry.add(prometheusMeterRegistry); -// setupDataPrepper(); -// final DataPrepperConfiguration dataPrepperConfiguration = DataPrepper.getConfiguration(); -// try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { -// dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); -// dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); -// dataPrepperServer = new DataPrepperServer(dataPrepper); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); -// dataPrepperServer.stop(); -// } -// } -// -// @Test -// public void testScrapeSystemMetricsFailure() throws IOException, InterruptedException, URISyntaxException { -// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT); -// final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); -// compositeMeterRegistry.add(prometheusMeterRegistry); -// setupDataPrepper(); -// final DataPrepperConfiguration dataPrepperConfiguration = DataPrepper.getConfiguration(); -// try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { -// dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); -// dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); -// dataPrepperServer = new DataPrepperServer(dataPrepper); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); -// dataPrepperServer.stop(); -// } -// } -// -// @Test -// public void testNonPrometheusMeterRegistry() throws Exception { -// final CloudWatchMeterRegistry cloudWatchMeterRegistry = mock(CloudWatchMeterRegistry.class); -// final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); -// compositeMeterRegistry.add(cloudWatchMeterRegistry); -// final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.fromFile( -// new File(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE)); -// setupDataPrepper(); -// try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { -// dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); -// dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); -// dataPrepperServer = new DataPrepperServer(dataPrepper); -// dataPrepperServer.start(); -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) -// .GET().build(); -// HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// } catch (ConnectException ex) { -// dataPrepperServer.stop(); -// //there should not be any Prometheus endpoints available -// assertThat(ex.getMessage(), Matchers.is("Connection refused")); -// } -// } -// -// @Test -// public void testMultipleMeterRegistries() throws Exception { -// //for system metrics -// final CloudWatchMeterRegistry cloudWatchMeterRegistryForSystem = mock(CloudWatchMeterRegistry.class); -// final PrometheusMeterRegistry prometheusMeterRegistryForSystem = -// new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); -// final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); -// compositeMeterRegistry.add(cloudWatchMeterRegistryForSystem); -// compositeMeterRegistry.add(prometheusMeterRegistryForSystem); -// -// //for data prepper metrics -// final PrometheusMeterRegistry prometheusMeterRegistryForDataPrepper = -// new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); -// setRegistry(prometheusMeterRegistryForDataPrepper); -// -// final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.fromFile( -// new File(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE)); -// setupDataPrepper(); -// try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { -// dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); -// dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); -// dataPrepperServer = new DataPrepperServer(dataPrepper); -// dataPrepperServer.start(); -// -// //test prometheus registry -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + dataPrepperConfiguration.getServerPort() -// + "/metrics/sys")).GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); -// dataPrepperServer.stop(); -// } -// } -// -// @Test -// public void testListPipelines() throws URISyntaxException, IOException, InterruptedException { -// setupDataPrepper(); -// final String pipelineName = "testPipeline"; -// Mockito.when(dataPrepper.getTransformationPipelines()).thenReturn( -// Collections.singletonMap("testPipeline", null) -// ); -// dataPrepperServer = new DataPrepperServer(dataPrepper); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// final String expectedResponse = OBJECT_MAPPER.writeValueAsString( -// Collections.singletonMap("pipelines", Arrays.asList( -// Collections.singletonMap("name", pipelineName) -// )) -// ); -// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); -// Assert.assertEquals(expectedResponse, response.body()); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testListPipelinesFailure() throws URISyntaxException, IOException, InterruptedException { -// setupDataPrepper(); -// Mockito.when(dataPrepper.getTransformationPipelines()).thenThrow(new RuntimeException("")); -// dataPrepperServer = new DataPrepperServer(dataPrepper); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testShutdown() throws URISyntaxException, IOException, InterruptedException { -// setupDataPrepper(); -// dataPrepperServer = new DataPrepperServer(dataPrepper); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); -// Mockito.verify(dataPrepper).shutdown(); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testShutdownFailure() throws URISyntaxException, IOException, InterruptedException { -// setupDataPrepper(); -// Mockito.doThrow(new RuntimeException("")).when(dataPrepper).shutdown(); -// dataPrepperServer = new DataPrepperServer(dataPrepper); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); -// Mockito.verify(dataPrepper).shutdown(); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testGetMetricsWithHttps() throws Exception { -// setupDataPrepper(); -// DataPrepper.configure(TestDataProvider.VALID_DATA_PREPPER_CONFIG_FILE_WITH_TLS); -// final String scrape = UUID.randomUUID().toString(); -// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); -// setRegistry(prometheusMeterRegistry); -// dataPrepperServer = new DataPrepperServer(dataPrepper); -// dataPrepperServer.start(); -// -// HttpClient httpsClient = HttpClient.newBuilder() -// .connectTimeout(Duration.ofSeconds(3)) -// .sslContext(getSslContextTrustingInsecure()) -// .build(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("https://localhost:" + port + "/metrics/prometheus")) -// .GET() -// .timeout(Duration.ofSeconds(3)) -// .build(); -// -// HttpResponse response = httpsClient.send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); -// Assert.assertEquals(scrape, response.body()); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testTlsWithInvalidPassword() { -// setupDataPrepper(); -// DataPrepper.configure(TestDataProvider.INVALID_KEYSTORE_PASSWORD_DATA_PREPPER_CONFIG_FILE); -// -// assertThrows(IllegalStateException.class, () -> new DataPrepperServer(dataPrepper)); -// } -// -// private SSLContext getSslContextTrustingInsecure() throws Exception { -// TrustManager[] trustAllCerts = new TrustManager[]{ -// new X509TrustManager() { -// public java.security.cert.X509Certificate[] getAcceptedIssuers() { -// return null; -// } -// -// public void checkClientTrusted( -// java.security.cert.X509Certificate[] certs, String authType) { -// } -// -// public void checkServerTrusted( -// java.security.cert.X509Certificate[] certs, String authType) { -// } -// } -// }; -// -// SSLContext sslContext = SSLContext.getInstance("SSL"); -// sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); -// -// return sslContext; -// } -// -// private static class PrometheusRegistryMockScrape extends PrometheusMeterRegistry { -// final String scrape; -// -// public PrometheusRegistryMockScrape(PrometheusConfig config, final String scrape) { -// super(config); -// this.scrape = scrape; -// } -// -// @Override -// public String scrape() { -// return scrape; -// } -// } -// -// private static class PrometheusRegistryThrowingScrape extends PrometheusMeterRegistry { -// -// public PrometheusRegistryThrowingScrape(PrometheusConfig config) { -// super(config); -// } -// -// @Override -// public String scrape() { -// throw new RuntimeException(""); -// } -// } +// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); + } + + @AfterEach + public void stopServer() { + if (dataPrepperServer != null) { + dataPrepperServer.stop(); + } + } + + @Test + public void testGetGlobalMetrics() throws IOException, InterruptedException, URISyntaxException { + when(dataPrepperConfiguration.getServerPort()).thenReturn(port); + final String scrape = UUID.randomUUID().toString(); + final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); + setRegistry(prometheusMeterRegistry); + dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); + + dataPrepperServer.start(); + + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) + .GET().build(); + HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + Assert.assertEquals(scrape, response.body()); + dataPrepperServer.stop(); + } + + @Test + public void testScrapeGlobalFailure() throws IOException, InterruptedException, URISyntaxException { + when(dataPrepperConfiguration.getServerPort()).thenReturn(port); + setRegistry(new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT)); + dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); + dataPrepperServer.start(); + + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) + .GET().build(); + HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); + dataPrepperServer.stop(); + } + + @Test + public void testGetSystemMetrics() throws IOException, InterruptedException, URISyntaxException { + when(dataPrepperConfiguration.getServerPort()).thenReturn(port); + final String scrape = UUID.randomUUID().toString(); + final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); + + when(systemMeterRegistry.getRegistries()) + .thenReturn(Set.of(prometheusMeterRegistry)); + + dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); + dataPrepperServer.start(); + + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) + .GET().build(); + HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + dataPrepperServer.stop(); + } + + @Test + public void testScrapeSystemMetricsFailure() throws IOException, InterruptedException, URISyntaxException { + when(dataPrepperConfiguration.getServerPort()).thenReturn(port); + final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT); + + when(systemMeterRegistry.getRegistries()) + .thenReturn(Set.of(prometheusMeterRegistry)); + + dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); + dataPrepperServer.start(); + + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) + .GET().build(); + HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); + dataPrepperServer.stop(); + } + + @Test + public void testNonPrometheusMeterRegistry() throws Exception { + final CloudWatchMeterRegistry cloudWatchMeterRegistry = mock(CloudWatchMeterRegistry.class); + final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); + compositeMeterRegistry.add(cloudWatchMeterRegistry); + final DataPrepperConfiguration config = makeConfig(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE); + try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { + dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); + dataPrepperServer.start(); + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) + .GET().build(); + HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + } catch (ConnectException ex) { + dataPrepperServer.stop(); + //there should not be any Prometheus endpoints available + assertThat(ex.getMessage(), Matchers.is("Connection refused")); + } + } + + @Test + public void testMultipleMeterRegistries() throws Exception { + //for system metrics + final CloudWatchMeterRegistry cloudWatchMeterRegistryForSystem = mock(CloudWatchMeterRegistry.class); + final PrometheusMeterRegistry prometheusMeterRegistryForSystem = + new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); + + when(systemMeterRegistry.getRegistries()) + .thenReturn(Set.of(cloudWatchMeterRegistryForSystem, prometheusMeterRegistryForSystem)); + + //for data prepper metrics + final PrometheusMeterRegistry prometheusMeterRegistryForDataPrepper = + new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); + setRegistry(prometheusMeterRegistryForDataPrepper); + + final DataPrepperConfiguration config = makeConfig(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE); + + dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); + dataPrepperServer.start(); + + //test prometheus registry + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + config.getServerPort() + + "/metrics/sys")).GET().build(); + HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + dataPrepperServer.stop(); + } + + @Test + public void testListPipelines() throws URISyntaxException, IOException, InterruptedException { + when(dataPrepperConfiguration.getServerPort()).thenReturn(port); + final String pipelineName = "testPipeline"; + Mockito.when(dataPrepper.getTransformationPipelines()).thenReturn( + Collections.singletonMap("testPipeline", null) + ); + dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); + dataPrepperServer.start(); + + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) + .GET().build(); + HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + final String expectedResponse = JSON_OBJECT_MAPPER.writeValueAsString( + Collections.singletonMap("pipelines", Arrays.asList( + Collections.singletonMap("name", pipelineName) + )) + ); + Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + Assert.assertEquals(expectedResponse, response.body()); + dataPrepperServer.stop(); + } + + @Test + public void testListPipelinesFailure() throws URISyntaxException, IOException, InterruptedException { + when(dataPrepperConfiguration.getServerPort()).thenReturn(port); + Mockito.when(dataPrepper.getTransformationPipelines()).thenThrow(new RuntimeException("")); + dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); + dataPrepperServer.start(); + + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) + .GET().build(); + HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); + dataPrepperServer.stop(); + } + + @Test + public void testShutdown() throws URISyntaxException, IOException, InterruptedException { + when(dataPrepperConfiguration.getServerPort()).thenReturn(port); + + dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); + dataPrepperServer.start(); + + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) + .GET().build(); + HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + Mockito.verify(dataPrepper).shutdown(); + dataPrepperServer.stop(); + } + + @Test + public void testShutdownFailure() throws URISyntaxException, IOException, InterruptedException { + when(dataPrepperConfiguration.getServerPort()).thenReturn(port); + Mockito.doThrow(new RuntimeException("")).when(dataPrepper).shutdown(); + dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); + dataPrepperServer.start(); + + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) + .GET().build(); + HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); + Mockito.verify(dataPrepper).shutdown(); + dataPrepperServer.stop(); + } + + @Test + public void testGetMetricsWithHttps() throws Exception { + DataPrepperConfiguration config = DataPrepperServerTest.makeConfig( + TestDataProvider.VALID_DATA_PREPPER_CONFIG_FILE_WITH_TLS); + final String scrape = UUID.randomUUID().toString(); + final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); + setRegistry(prometheusMeterRegistry); + dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); + dataPrepperServer.start(); + + HttpClient httpsClient = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(3)) + .sslContext(getSslContextTrustingInsecure()) + .build(); + + HttpRequest request = HttpRequest.newBuilder(new URI("https://localhost:" + port + "/metrics/prometheus")) + .GET() + .timeout(Duration.ofSeconds(3)) + .build(); + + HttpResponse response = httpsClient.send(request, HttpResponse.BodyHandlers.ofString()); + Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + Assert.assertEquals(scrape, response.body()); + dataPrepperServer.stop(); + } + + @Test + public void testTlsWithInvalidPassword() throws IOException { + DataPrepperConfiguration config = DataPrepperServerTest.makeConfig( + TestDataProvider.INVALID_KEYSTORE_PASSWORD_DATA_PREPPER_CONFIG_FILE); + + assertThrows(IllegalStateException.class, () -> new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry)); + } + + private SSLContext getSslContextTrustingInsecure() throws Exception { + TrustManager[] trustAllCerts = new TrustManager[]{ + new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + public void checkClientTrusted( + java.security.cert.X509Certificate[] certs, String authType) { + } + + public void checkServerTrusted( + java.security.cert.X509Certificate[] certs, String authType) { + } + } + }; + + SSLContext sslContext = SSLContext.getInstance("SSL"); + sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); + + return sslContext; + } + + private static class PrometheusRegistryMockScrape extends PrometheusMeterRegistry { + final String scrape; + + public PrometheusRegistryMockScrape(PrometheusConfig config, final String scrape) { + super(config); + this.scrape = scrape; + } + + @Override + public String scrape() { + return scrape; + } + } + + private static class PrometheusRegistryThrowingScrape extends PrometheusMeterRegistry { + + public PrometheusRegistryThrowingScrape(PrometheusConfig config) { + super(config); + } + + @Override + public String scrape() { + throw new RuntimeException(""); + } + } + + private static DataPrepperConfiguration makeConfig(String filePath) throws IOException { + final File configurationFile = new File(filePath); + return OBJECT_MAPPER.readValue(configurationFile, DataPrepperConfiguration.class); + } } From 8ce2584466b9aa764b33d37e2507d72485c773bc Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Thu, 23 Dec 2021 12:22:54 -0600 Subject: [PATCH 10/43] Refactored DataPrepperConfigurationTests Signed-off-by: Steven Bayer --- ...DataPrepperConfigurationConfiguration.java | 68 --------- .../model/DataPrepperConfigurationTests.java | 135 ++++++++++-------- 2 files changed, 73 insertions(+), 130 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java index f6b728555f..fb809a00cd 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java @@ -1,21 +1,7 @@ package com.amazon.dataprepper.parser.config; -import com.amazon.dataprepper.model.plugin.PluginFactory; -import com.amazon.dataprepper.parser.PipelineParser; import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; -import com.amazon.dataprepper.parser.model.MetricRegistryType; -import com.amazon.dataprepper.plugin.DefaultPluginFactory; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Metrics; -import io.micrometer.core.instrument.binder.MeterBinder; -import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; -import io.micrometer.core.instrument.binder.system.ProcessorMetrics; -import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import org.opensearch.dataprepper.logstash.LogstashConfigConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,7 +14,6 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.List; @Configuration @ComponentScan(basePackageClasses = com.amazon.dataprepper.DataPrepper.class) @@ -37,59 +22,6 @@ public class DataPrepperConfigurationConfiguration { private static final String POSITIONAL_COMMAND_LINE_ARGUMENTS = "nonOptionArgs"; private static final String COMMAND_LINE_ARG_DELIMITER = ","; - @Bean - public YAMLFactory yamlFactory() { - return new YAMLFactory(); - } - - @Bean - public ObjectMapper objectMapper(final YAMLFactory yamlFactory) { - return new ObjectMapper(yamlFactory); - } - - @Bean - public ClassLoaderMetrics classLoaderMetrics() { - return new ClassLoaderMetrics(); - } - - @Bean - public JvmMemoryMetrics jvmMemoryMetrics() { - return new JvmMemoryMetrics(); - } - - @Bean - public JvmGcMetrics jvmGcMetrics() { - return new JvmGcMetrics(); - } - - @Bean - public ProcessorMetrics processorMetrics() { - return new ProcessorMetrics(); - } - - @Bean - public JvmThreadMetrics jvmThreadMetrics() { - return new JvmThreadMetrics(); - } - - @Bean - public CompositeMeterRegistry systemMeterRegistry( - final List meterBinders, - final DataPrepperConfiguration dataPrepperConfiguration - ) { - final CompositeMeterRegistry meterRegistry = new CompositeMeterRegistry(); - - meterBinders.forEach(binder -> binder.bindTo(meterRegistry)); - - dataPrepperConfiguration.getMetricRegistryTypes().forEach(metricRegistryType -> { - MeterRegistry registryForType = MetricRegistryType.getDefaultMeterRegistryForType(metricRegistryType); - meterRegistry.add(registryForType); - Metrics.addRegistry(registryForType); - }); - - return meterRegistry; - } - @Bean public DataPrepperArgs dataPrepperArgs(final Environment environment) { final String commandLineArgs = environment.getProperty(POSITIONAL_COMMAND_LINE_ARGUMENTS); diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java index dabaf66a44..c9df91cf0f 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java @@ -5,11 +5,16 @@ package com.amazon.dataprepper.parser.model; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; +import com.fasterxml.jackson.databind.exc.ValueInstantiationException; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.jupiter.api.Test; import java.io.File; +import java.io.IOException; import static com.amazon.dataprepper.TestDataProvider.INVALID_DATA_PREPPER_CONFIG_FILE; import static com.amazon.dataprepper.TestDataProvider.INVALID_PORT_DATA_PREPPER_CONFIG_FILE; @@ -25,67 +30,73 @@ import static org.junit.jupiter.api.Assertions.assertThrows; public class DataPrepperConfigurationTests { + private static ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()); -// @Test -// public void testParseConfig() { -// final DataPrepperConfiguration dataPrepperConfiguration = -// DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_CONFIG_FILE)); -// Assert.assertEquals(5678, dataPrepperConfiguration.getServerPort()); -// } -// -// @Test -// public void testSomeDefaultConfig() { -// final DataPrepperConfiguration dataPrepperConfiguration = -// DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_SOME_DEFAULT_CONFIG_FILE)); -// Assert.assertEquals(DataPrepperConfiguration.DEFAULT_CONFIG.getServerPort(), dataPrepperConfiguration.getServerPort()); -// } -// -// @Test -// public void testDefaultMetricsRegistry() { -// final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.DEFAULT_CONFIG; -// assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(1)); -// assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.Prometheus)); -// } -// -// @Test -// public void testCloudWatchMetricsRegistry() { -// final DataPrepperConfiguration dataPrepperConfiguration = -// DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE)); -// assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(1)); -// assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.CloudWatch)); -// } -// -// @Test -// public void testMultipleMetricsRegistry() { -// final DataPrepperConfiguration dataPrepperConfiguration = -// DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE)); -// assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(2)); -// assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.Prometheus)); -// assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.CloudWatch)); -// } -// -// @Test -// void testConfigurationWithHttpBasic() { -// final DataPrepperConfiguration dataPrepperConfiguration = -// DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_CONFIG_FILE_WITH_BASIC_AUTHENTICATION)); -// -// assertThat(dataPrepperConfiguration, notNullValue()); -// assertThat(dataPrepperConfiguration.getAuthentication(), notNullValue()); -// assertThat(dataPrepperConfiguration.getAuthentication().getPluginName(), equalTo("http_basic")); -// assertThat(dataPrepperConfiguration.getAuthentication().getPluginSettings(), notNullValue()); -// assertThat(dataPrepperConfiguration.getAuthentication().getPluginSettings(), hasKey("username")); -// assertThat(dataPrepperConfiguration.getAuthentication().getPluginSettings(), hasKey("password")); -// } -// -// @Test -// public void testInvalidConfig() { -// assertThrows(IllegalArgumentException.class, () -> -// DataPrepperConfiguration.fromFile(new File(INVALID_DATA_PREPPER_CONFIG_FILE))); -// } -// -// @Test -// public void testInvalidPortConfig() { -// assertThrows(IllegalArgumentException.class, () -> -// DataPrepperConfiguration.fromFile(new File(INVALID_PORT_DATA_PREPPER_CONFIG_FILE))); -// } + private static DataPrepperConfiguration makeConfig(String filePath) throws IOException { + final File configurationFile = new File(filePath); + return OBJECT_MAPPER.readValue(configurationFile, DataPrepperConfiguration.class); + } + + @Test + public void testParseConfig() throws IOException { + final DataPrepperConfiguration dataPrepperConfiguration = + makeConfig(VALID_DATA_PREPPER_CONFIG_FILE); + Assert.assertEquals(5678, dataPrepperConfiguration.getServerPort()); + } + + @Test + public void testSomeDefaultConfig() throws IOException { + final DataPrepperConfiguration dataPrepperConfiguration = + makeConfig(VALID_DATA_PREPPER_SOME_DEFAULT_CONFIG_FILE); + Assert.assertEquals(DataPrepperConfiguration.DEFAULT_CONFIG.getServerPort(), dataPrepperConfiguration.getServerPort()); + } + + @Test + public void testDefaultMetricsRegistry() { + final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.DEFAULT_CONFIG; + assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(1)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.Prometheus)); + } + + @Test + public void testCloudWatchMetricsRegistry() throws IOException { + final DataPrepperConfiguration dataPrepperConfiguration = + makeConfig(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(1)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.CloudWatch)); + } + + @Test + public void testMultipleMetricsRegistry() throws IOException { + final DataPrepperConfiguration dataPrepperConfiguration = + makeConfig(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(2)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.Prometheus)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.CloudWatch)); + } + + @Test + void testConfigurationWithHttpBasic() throws IOException { + final DataPrepperConfiguration dataPrepperConfiguration = + makeConfig(VALID_DATA_PREPPER_CONFIG_FILE_WITH_BASIC_AUTHENTICATION); + + assertThat(dataPrepperConfiguration, notNullValue()); + assertThat(dataPrepperConfiguration.getAuthentication(), notNullValue()); + assertThat(dataPrepperConfiguration.getAuthentication().getPluginName(), equalTo("http_basic")); + assertThat(dataPrepperConfiguration.getAuthentication().getPluginSettings(), notNullValue()); + assertThat(dataPrepperConfiguration.getAuthentication().getPluginSettings(), hasKey("username")); + assertThat(dataPrepperConfiguration.getAuthentication().getPluginSettings(), hasKey("password")); + } + + @Test + public void testInvalidConfig() { + assertThrows(UnrecognizedPropertyException.class, () -> + makeConfig(INVALID_DATA_PREPPER_CONFIG_FILE)); + } + + @Test + public void testInvalidPortConfig() { + assertThrows(ValueInstantiationException.class, () -> + makeConfig(INVALID_PORT_DATA_PREPPER_CONFIG_FILE)); + } } From 68d73b1b59bbbe41ee7713ea7672d3a39823f3da Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Thu, 23 Dec 2021 12:23:56 -0600 Subject: [PATCH 11/43] Renamed DataPrepperConfigurationConfiguration to DataPrepperAppConfiguration Signed-off-by: Steven Bayer --- .../dataprepper/DataPrepperExecute.java | 4 +- ...DataPrepperConfigurationConfiguration.java | 73 ------------------- 2 files changed, 2 insertions(+), 75 deletions(-) delete mode 100644 data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java index edca41c4ac..65de114dad 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java @@ -5,7 +5,7 @@ package com.amazon.dataprepper; -import com.amazon.dataprepper.parser.config.DataPrepperConfigurationConfiguration; +import com.amazon.dataprepper.parser.config.DataPrepperAppConfiguration; import org.opensearch.dataprepper.logstash.LogstashConfigConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +34,7 @@ public static void main(String[] args) { AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); context.getEnvironment().getPropertySources().addFirst(commandLinePropertySource); - context.register(DataPrepperConfigurationConfiguration.class); + context.register(DataPrepperAppConfiguration.class); context.refresh(); for (String name : context.getBeanDefinitionNames()) { diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java deleted file mode 100644 index fb809a00cd..0000000000 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.amazon.dataprepper.parser.config; - -import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.opensearch.dataprepper.logstash.LogstashConfigConverter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; - -@Configuration -@ComponentScan(basePackageClasses = com.amazon.dataprepper.DataPrepper.class) -public class DataPrepperConfigurationConfiguration { - private static final Logger LOG = LoggerFactory.getLogger(DataPrepperConfigurationConfiguration.class); - private static final String POSITIONAL_COMMAND_LINE_ARGUMENTS = "nonOptionArgs"; - private static final String COMMAND_LINE_ARG_DELIMITER = ","; - - @Bean - public DataPrepperArgs dataPrepperArgs(final Environment environment) { - final String commandLineArgs = environment.getProperty(POSITIONAL_COMMAND_LINE_ARGUMENTS); - - LOG.info("Command line args: {}", commandLineArgs); - - if (commandLineArgs != null) { - String[] args = commandLineArgs.split(COMMAND_LINE_ARG_DELIMITER); - return new DataPrepperArgs(args); - } - else { - throw new RuntimeException("Configuration file command line argument required but none found"); - } - } - - @Bean - public DataPrepperConfiguration dataPrepperConfiguration( - final DataPrepperArgs dataPrepperArgs, - final ObjectMapper objectMapper - ) { - final String dataPrepperConfigFileLocation = dataPrepperArgs.getDataPrepperConfigFileLocation(); - if (dataPrepperConfigFileLocation != null) { - final File configurationFile = new File(dataPrepperConfigFileLocation); - try { - return objectMapper.readValue(configurationFile, DataPrepperConfiguration.class); - } catch (final IOException e) { - throw new IllegalArgumentException("Invalid DataPrepper configuration file.", e); - } - } - else { - return new DataPrepperConfiguration(); - } - } - - private static String checkForLogstashConfigurationAndConvert(String configurationFileLocation) { - if (configurationFileLocation.endsWith(".conf")) { - final LogstashConfigConverter logstashConfigConverter = new LogstashConfigConverter(); - final Path configurationDirectory = Paths.get(configurationFileLocation).toAbsolutePath().getParent(); - - try { - configurationFileLocation = logstashConfigConverter.convertLogstashConfigurationToPipeline( - configurationFileLocation, String.valueOf(configurationDirectory)); - } catch (IOException e) { - LOG.error("Unable to read the Logstash configuration file", e); - } - } - return configurationFileLocation; - } -} From 5c1ff83a268eecdc1bb2edf8deaadbf759f5c8ba Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Mon, 3 Jan 2022 14:47:41 -0600 Subject: [PATCH 12/43] Remove debug messages Signed-off-by: Steven Bayer --- .../src/main/java/com/amazon/dataprepper/DataPrepper.java | 1 - .../main/java/com/amazon/dataprepper/DataPrepperExecute.java | 4 ---- .../amazon/dataprepper/pipeline/server/DataPrepperServer.java | 2 -- .../com/amazon/dataprepper/plugin/DefaultPluginFactory.java | 2 -- 4 files changed, 9 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java index 57a78c18e4..6c40d08250 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java @@ -9,7 +9,6 @@ import com.amazon.dataprepper.parser.PipelineParser; import com.amazon.dataprepper.pipeline.Pipeline; import com.amazon.dataprepper.pipeline.server.DataPrepperServer; -import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import io.micrometer.core.instrument.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java index 65de114dad..b126df52a1 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java @@ -37,10 +37,6 @@ public static void main(String[] args) { context.register(DataPrepperAppConfiguration.class); context.refresh(); - for (String name : context.getBeanDefinitionNames()) { - LOG.info("Bean Found: {}", name); - } - DataPrepper dataPrepper = context.getBean(DataPrepper.class); dataPrepper.execute(); } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java index 243f3293c4..5e7a93d3f5 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java @@ -24,7 +24,6 @@ import javax.inject.Inject; import javax.inject.Named; -import javax.inject.Singleton; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLParameters; import java.io.IOException; @@ -40,7 +39,6 @@ * Currently, only serves metrics in prometheus format. */ @Named -@Singleton public class DataPrepperServer { private static final Logger LOG = LoggerFactory.getLogger(DataPrepperServer.class); private final HttpServer server; diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java index c3cb8420a4..c8277e96ea 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java @@ -12,7 +12,6 @@ import javax.inject.Inject; import javax.inject.Named; -import javax.inject.Singleton; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -26,7 +25,6 @@ * @since 1.2 */ @Named -@Singleton public class DefaultPluginFactory implements PluginFactory { private final Collection pluginProviders; From eab5623055925687c2b661b64c353f45ae4efacb Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Mon, 3 Jan 2022 16:39:09 -0600 Subject: [PATCH 13/43] Added unit tests Signed-off-by: Steven Bayer --- .../parser/PipelineParserTests.java | 194 ++++++++++-------- 1 file changed, 103 insertions(+), 91 deletions(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/PipelineParserTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/PipelineParserTests.java index 7f8c5ae372..2452150399 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/PipelineParserTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/PipelineParserTests.java @@ -6,6 +6,7 @@ package com.amazon.dataprepper.parser; import com.amazon.dataprepper.model.plugin.PluginFactory; +import com.amazon.dataprepper.parser.config.DataPrepperArgs; import com.amazon.dataprepper.pipeline.Pipeline; import com.amazon.dataprepper.plugin.DefaultPluginFactory; import org.junit.jupiter.api.BeforeEach; @@ -30,95 +31,106 @@ class PipelineParserTests { -// private PluginFactory pluginFactory; -// -// @BeforeEach -// void setUp() { -// pluginFactory = new DefaultPluginFactory(); -// } -// -// @Test -// void parseConfiguration_with_multiple_valid_pipelines_creates_the_correct_pipelineMap() { -// final PipelineParser pipelineParser = new PipelineParser(VALID_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); -// final Map actualPipelineMap = pipelineParser.parseConfiguration(); -// assertThat(actualPipelineMap.keySet(), equalTo(VALID_MULTIPLE_PIPELINE_NAMES)); -// } -// -// @Test -// void parseConfiguration_with_invalid_root_pipeline_creates_empty_pipelinesMap() { -// final PipelineParser pipelineParser = new PipelineParser(CONNECTED_PIPELINE_ROOT_SOURCE_INCORRECT, pluginFactory); -// final Map connectedPipelines = pipelineParser.parseConfiguration(); -// assertThat(connectedPipelines.size(), equalTo(0)); -// } -// -// @Test -// void parseConfiguration_with_incorrect_child_pipeline_returns_empty_pipelinesMap() { -// final PipelineParser pipelineParser = new PipelineParser(CONNECTED_PIPELINE_CHILD_PIPELINE_INCORRECT, pluginFactory); -// final Map connectedPipelines = pipelineParser.parseConfiguration(); -// assertThat(connectedPipelines.size(), equalTo(0)); -// } -// -// @Test -// void parseConfiguration_with_a_single_pipeline_with_empty_source_settings_returns_that_pipeline() { -// final PipelineParser pipelineParser = new PipelineParser(VALID_SINGLE_PIPELINE_EMPTY_SOURCE_PLUGIN_FILE, pluginFactory); -// final Map actualPipelineMap = pipelineParser.parseConfiguration(); -// assertThat(actualPipelineMap.keySet().size(), equalTo(1)); -// } -// -// @Test -// void parseConfiguration_with_cycles_in_multiple_pipelines_should_throw() { -// final PipelineParser pipelineParser = new PipelineParser(CYCLE_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); -// -// final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); -// assertThat(actualException.getMessage(), -// equalTo("Provided configuration results in a loop, check pipeline: test-pipeline-1")); -// -// } -// -// @Test -// void parseConfiguration_with_incorrect_source_mapping_in_multiple_pipelines_should_throw() { -// final PipelineParser pipelineParser = new PipelineParser(INCORRECT_SOURCE_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); -// -// final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); -// assertThat(actualException.getMessage(), -// equalTo("Invalid configuration, expected source test-pipeline-1 for pipeline test-pipeline-2 is missing")); -// } -// -// @Test -// void parseConfiguration_with_missing_pipeline_name_should_throw() { -// final PipelineParser pipelineParser = new PipelineParser(MISSING_NAME_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); -// -// final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); -// assertThat(actualException.getMessage(), -// equalTo("name is a required attribute for sink pipeline plugin, " + -// "check pipeline: test-pipeline-1")); -// } -// -// @Test -// void parseConfiguration_with_missing_pipeline_name_in_multiple_pipelines_should_throw() { -// final PipelineParser pipelineParser = new PipelineParser(MISSING_PIPELINE_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); -// final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); -// assertThat(actualException.getMessage(), equalTo("Invalid configuration, no pipeline is defined with name test-pipeline-4")); -// } -// -// @Test -// void testMultipleSinks() { -// final PipelineParser pipelineParser = new PipelineParser(VALID_MULTIPLE_SINKS_CONFIG_FILE, pluginFactory); -// final Map pipelineMap = pipelineParser.parseConfiguration(); -// assertThat(pipelineMap.keySet().size(), equalTo(3)); -// } -// -// @Test -// void testMultipleProcessors() { -// final PipelineParser pipelineParser = new PipelineParser(VALID_MULTIPLE_PROCESSERS_CONFIG_FILE, pluginFactory); -// final Map pipelineMap = pipelineParser.parseConfiguration(); -// assertThat(pipelineMap.keySet().size(), equalTo(3)); -// } -// -// @Test -// void parseConfiguration_with_a_configuration_file_which_does_not_exist_should_throw() { -// final PipelineParser pipelineParser = new PipelineParser("file_does_no_exist.yml", pluginFactory); -// final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); -// assertThat(actualException.getMessage(), equalTo("Failed to parse the configuration file file_does_no_exist.yml")); -// } + private PluginFactory pluginFactory; + + @BeforeEach + void setUp() { + pluginFactory = new DefaultPluginFactory(); + } + + @Test + void parseConfiguration_with_multiple_valid_pipelines_creates_the_correct_pipelineMap() { + final DataPrepperArgs args = new DataPrepperArgs(VALID_MULTIPLE_PIPELINE_CONFIG_FILE); + final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final Map actualPipelineMap = pipelineParser.parseConfiguration(); + assertThat(actualPipelineMap.keySet(), equalTo(VALID_MULTIPLE_PIPELINE_NAMES)); + } + + @Test + void parseConfiguration_with_invalid_root_pipeline_creates_empty_pipelinesMap() { + final DataPrepperArgs args = new DataPrepperArgs(CONNECTED_PIPELINE_ROOT_SOURCE_INCORRECT); + final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final Map connectedPipelines = pipelineParser.parseConfiguration(); + assertThat(connectedPipelines.size(), equalTo(0)); + } + + @Test + void parseConfiguration_with_incorrect_child_pipeline_returns_empty_pipelinesMap() { + final DataPrepperArgs args = new DataPrepperArgs(CONNECTED_PIPELINE_CHILD_PIPELINE_INCORRECT); + final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final Map connectedPipelines = pipelineParser.parseConfiguration(); + assertThat(connectedPipelines.size(), equalTo(0)); + } + + @Test + void parseConfiguration_with_a_single_pipeline_with_empty_source_settings_returns_that_pipeline() { + final DataPrepperArgs args = new DataPrepperArgs(VALID_SINGLE_PIPELINE_EMPTY_SOURCE_PLUGIN_FILE); + final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final Map actualPipelineMap = pipelineParser.parseConfiguration(); + assertThat(actualPipelineMap.keySet().size(), equalTo(1)); + } + + @Test + void parseConfiguration_with_cycles_in_multiple_pipelines_should_throw() { + final DataPrepperArgs args = new DataPrepperArgs(CYCLE_MULTIPLE_PIPELINE_CONFIG_FILE); + final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + + final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); + assertThat(actualException.getMessage(), + equalTo("Provided configuration results in a loop, check pipeline: test-pipeline-1")); + + } + + @Test + void parseConfiguration_with_incorrect_source_mapping_in_multiple_pipelines_should_throw() { + final DataPrepperArgs args = new DataPrepperArgs(INCORRECT_SOURCE_MULTIPLE_PIPELINE_CONFIG_FILE); + final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + + final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); + assertThat(actualException.getMessage(), + equalTo("Invalid configuration, expected source test-pipeline-1 for pipeline test-pipeline-2 is missing")); + } + + @Test + void parseConfiguration_with_missing_pipeline_name_should_throw() { + final DataPrepperArgs args = new DataPrepperArgs(MISSING_NAME_MULTIPLE_PIPELINE_CONFIG_FILE); + final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + + final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); + assertThat(actualException.getMessage(), + equalTo("name is a required attribute for sink pipeline plugin, " + + "check pipeline: test-pipeline-1")); + } + + @Test + void parseConfiguration_with_missing_pipeline_name_in_multiple_pipelines_should_throw() { + final DataPrepperArgs args = new DataPrepperArgs(MISSING_PIPELINE_MULTIPLE_PIPELINE_CONFIG_FILE); + final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); + assertThat(actualException.getMessage(), equalTo("Invalid configuration, no pipeline is defined with name test-pipeline-4")); + } + + @Test + void testMultipleSinks() { + final DataPrepperArgs args = new DataPrepperArgs(VALID_MULTIPLE_SINKS_CONFIG_FILE); + final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final Map pipelineMap = pipelineParser.parseConfiguration(); + assertThat(pipelineMap.keySet().size(), equalTo(3)); + } + + @Test + void testMultipleProcessors() { + final DataPrepperArgs args = new DataPrepperArgs(VALID_MULTIPLE_PROCESSERS_CONFIG_FILE); + final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final Map pipelineMap = pipelineParser.parseConfiguration(); + assertThat(pipelineMap.keySet().size(), equalTo(3)); + } + + @Test + void parseConfiguration_with_a_configuration_file_which_does_not_exist_should_throw() { + final DataPrepperArgs args = new DataPrepperArgs("file_does_no_exist.yml"); + final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); + assertThat(actualException.getMessage(), equalTo("Failed to parse the configuration file file_does_no_exist.yml")); + } } From 6cd7ded0829293bfba39234f35cf941748c1e7cc Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Tue, 4 Jan 2022 12:30:05 -0600 Subject: [PATCH 14/43] Changed DataPrepperServer to Lazy bean Signed-off-by: Steven Bayer --- .../src/main/java/com/amazon/dataprepper/DataPrepper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java index 6c40d08250..fa3e6468c4 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java @@ -12,6 +12,7 @@ import io.micrometer.core.instrument.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Lazy; import javax.inject.Inject; import javax.inject.Named; @@ -34,6 +35,7 @@ public class DataPrepper { // TODO: Remove DataPrepperServer dependency on DataPrepper @Inject + @Lazy private DataPrepperServer dataPrepperServer; /** From 321d3ebe973c84029e3c76e6df0ae87512427be8 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Tue, 4 Jan 2022 13:49:57 -0600 Subject: [PATCH 15/43] Fixed bug - never checked for logstash config Signed-off-by: Steven Bayer --- .../dataprepper/DataPrepperExecute.java | 34 ++++--------------- .../parser/config/DataPrepperArgs.java | 27 +++++++++++++-- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java index b126df52a1..38438e4bfd 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java @@ -5,54 +5,34 @@ package com.amazon.dataprepper; -import com.amazon.dataprepper.parser.config.DataPrepperAppConfiguration; -import org.opensearch.dataprepper.logstash.LogstashConfigConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.ComponentScan; import org.springframework.core.env.SimpleCommandLinePropertySource; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; - /** * Execute entry into Data Prepper. */ +@ComponentScan public class DataPrepperExecute { private static final Logger LOG = LoggerFactory.getLogger(DataPrepperExecute.class); public static void main(String[] args) { java.security.Security.setProperty("networkaddress.cache.ttl", "60"); - for (String arg : args) { - LOG.info("Arg: {}", arg); - } - - //TODO: Load this into context + LOG.trace("Reading args"); SimpleCommandLinePropertySource commandLinePropertySource = new SimpleCommandLinePropertySource(args); + LOG.trace("Creating application context"); AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); context.getEnvironment().getPropertySources().addFirst(commandLinePropertySource); - context.register(DataPrepperAppConfiguration.class); + context.register(DataPrepperExecute.class); context.refresh(); DataPrepper dataPrepper = context.getBean(DataPrepper.class); - dataPrepper.execute(); - } - private static String checkForLogstashConfigurationAndConvert(String configurationFileLocation) { - if (configurationFileLocation.endsWith(".conf")) { - final LogstashConfigConverter logstashConfigConverter = new LogstashConfigConverter(); - final Path configurationDirectory = Paths.get(configurationFileLocation).toAbsolutePath().getParent(); - - try { - configurationFileLocation = logstashConfigConverter.convertLogstashConfigurationToPipeline( - configurationFileLocation, String.valueOf(configurationDirectory)); - } catch (IOException e) { - LOG.error("Unable to read the Logstash configuration file", e); - } - } - return configurationFileLocation; + LOG.trace("Starting Data Prepper execution"); + dataPrepper.execute(); } } \ No newline at end of file diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java index 7f9388b017..32464b080f 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java @@ -1,9 +1,13 @@ package com.amazon.dataprepper.parser.config; +import org.opensearch.dataprepper.logstash.LogstashConfigConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; public class DataPrepperArgs { private static final Logger LOG = LoggerFactory.getLogger(DataPrepperArgs.class); @@ -11,6 +15,23 @@ public class DataPrepperArgs { private static final Integer DATA_PREPPER_PIPELINE_CONFIG_POSITON = 0; private static final Integer DATA_PREPPER_CONFIG_POSITON = 1; + private static String checkForLogstashConfigurationAndConvert(String configurationFileLocation) { + if (configurationFileLocation.endsWith(".conf")) { + LOG.debug("Detected logstash configuration file, attempting to convert to Data Prepper pipeline"); + + final LogstashConfigConverter logstashConfigConverter = new LogstashConfigConverter(); + final Path configurationDirectory = Paths.get(configurationFileLocation).toAbsolutePath().getParent(); + + try { + configurationFileLocation = logstashConfigConverter.convertLogstashConfigurationToPipeline( + configurationFileLocation, String.valueOf(configurationDirectory)); + } catch (IOException e) { + LOG.error("Unable to read the Logstash configuration file", e); + } + } + return configurationFileLocation; + } + private final String pipelineConfigFileLocation; private final String dataPrepperConfigFileLocation; @@ -20,8 +41,10 @@ public DataPrepperArgs(final String ... args) { System.exit(1); } - this.pipelineConfigFileLocation = args[DATA_PREPPER_PIPELINE_CONFIG_POSITON]; - LOG.info("Using {} configuration file", pipelineConfigFileLocation); + String configurationFileLocation = args[DATA_PREPPER_PIPELINE_CONFIG_POSITON]; + LOG.info("Using {} configuration file", configurationFileLocation); + + this.pipelineConfigFileLocation = DataPrepperArgs.checkForLogstashConfigurationAndConvert(configurationFileLocation); if (args.length > DATA_PREPPER_CONFIG_POSITON) { this.dataPrepperConfigFileLocation = args[DATA_PREPPER_CONFIG_POSITON]; From 22be738fd6a31d0c39a2ac5182e9f941ad4b0e19 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Tue, 4 Jan 2022 14:37:19 -0600 Subject: [PATCH 16/43] Added unit tests for command line args Signed-off-by: Steven Bayer --- .../parser/config/DataPrepperArgs.java | 17 ++++++++++++++--- .../parser/config/DataPrepperArgsTest.java | 7 ++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java index 32464b080f..6df72f17c9 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java @@ -14,6 +14,7 @@ public class DataPrepperArgs { private static final Integer DATA_PREPPER_PIPELINE_CONFIG_POSITON = 0; private static final Integer DATA_PREPPER_CONFIG_POSITON = 1; + private static final Integer MAXIMUM_SUPPORTED_NUMBER_OF_ARGS = 2; private static String checkForLogstashConfigurationAndConvert(String configurationFileLocation) { if (configurationFileLocation.endsWith(".conf")) { @@ -26,7 +27,8 @@ private static String checkForLogstashConfigurationAndConvert(String configurati configurationFileLocation = logstashConfigConverter.convertLogstashConfigurationToPipeline( configurationFileLocation, String.valueOf(configurationDirectory)); } catch (IOException e) { - LOG.error("Unable to read the Logstash configuration file", e); + LOG.warn("Unable to read the Logstash configuration file", e); + throw new IllegalArgumentException("Invalid Logstash configuration file", e); } } return configurationFileLocation; @@ -37,8 +39,11 @@ private static String checkForLogstashConfigurationAndConvert(String configurati public DataPrepperArgs(final String ... args) { if (args == null || args.length == 0) { - LOG.error("Configuration file command line argument required but none found"); - System.exit(1); + invalidArgumentsReceived("Configuration file command line argument required but none found"); + } + else if (args.length > MAXIMUM_SUPPORTED_NUMBER_OF_ARGS) { + invalidArgumentsReceived( + "Data Prepper supports a maximum of " + MAXIMUM_SUPPORTED_NUMBER_OF_ARGS + " command line arguments"); } String configurationFileLocation = args[DATA_PREPPER_PIPELINE_CONFIG_POSITON]; @@ -54,6 +59,12 @@ public DataPrepperArgs(final String ... args) { } } + private void invalidArgumentsReceived(final String msg) { + LOG.warn("Invalid Data Prepper arguments received." + + " Valid argument format: []"); + throw new IllegalArgumentException(msg); + } + @Nullable public String getPipelineConfigFileLocation() { return pipelineConfigFileLocation; diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java index b914a17633..8e6de0cdcf 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java @@ -1,14 +1,11 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - package com.amazon.dataprepper.parser.config; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; +import java.io.IOException; + import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.is; From d70ab4cd48de3aa2fd14d7333366828b343538fb Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Tue, 4 Jan 2022 14:49:13 -0600 Subject: [PATCH 17/43] Removed unused import Signed-off-by: Steven Bayer --- .../amazon/dataprepper/parser/config/DataPrepperArgsTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java index 8e6de0cdcf..dd63b4737d 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java @@ -4,8 +4,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; -import java.io.IOException; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.is; From 2c4ad8ec36303d5740286a4155b5281ddc5d4084 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Tue, 4 Jan 2022 15:40:11 -0600 Subject: [PATCH 18/43] Renamed test from foo Signed-off-by: Steven Bayer --- .../src/test/java/com/amazon/dataprepper/DataPrepperTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java index b0ecff7fbc..4ae811e298 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java @@ -124,7 +124,7 @@ public void testShutdownDataPrepperServer() { } @Test - public void foo() throws NoSuchFieldException, IllegalAccessException { + public void testGivenEnvVarNotSetThenDefaultServiceNameReturned() throws NoSuchFieldException, IllegalAccessException { final Field dataPrepperServerField = DataPrepper.class.getDeclaredField("DEFAULT_SERVICE_NAME"); dataPrepperServerField.setAccessible(true); String expected = (String) dataPrepperServerField.get(null); From 5f1a1317dced7dbe747b2c12feeb70631f0ff5b0 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Wed, 5 Jan 2022 11:46:10 -0600 Subject: [PATCH 19/43] Added copywrite comments Signed-off-by: Steven Bayer --- .../dataprepper/parser/PipelineParser.java | 9 ++--- .../parser/config/DataPrepperArgs.java | 5 +++ .../parser/PipelineParserTests.java | 34 ++++++------------- .../parser/config/DataPrepperArgsTest.java | 5 +++ 4 files changed, 23 insertions(+), 30 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/PipelineParser.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/PipelineParser.java index 6dc27ef01a..e23b0aad88 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/PipelineParser.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/PipelineParser.java @@ -14,7 +14,6 @@ import com.amazon.dataprepper.model.processor.Processor; import com.amazon.dataprepper.model.sink.Sink; import com.amazon.dataprepper.model.source.Source; -import com.amazon.dataprepper.parser.config.DataPrepperArgs; import com.amazon.dataprepper.parser.model.PipelineConfiguration; import com.amazon.dataprepper.pipeline.Pipeline; import com.amazon.dataprepper.pipeline.PipelineConnector; @@ -25,8 +24,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Named; import java.io.File; import java.io.IOException; import java.util.HashMap; @@ -40,7 +37,6 @@ import static java.lang.String.format; @SuppressWarnings("rawtypes") -@Named public class PipelineParser { private static final Logger LOG = LoggerFactory.getLogger(PipelineParser.class); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()) @@ -51,9 +47,8 @@ public class PipelineParser { private final Map sourceConnectorMap = new HashMap<>(); //TODO Remove this and rely only on pipelineMap private final PluginFactory pluginFactory; - @Inject - public PipelineParser(final DataPrepperArgs dataPrepperArgs, final PluginFactory pluginFactory) { - this.pipelineConfigurationFileLocation = dataPrepperArgs.getPipelineConfigFileLocation(); + public PipelineParser(final String pipelineConfigurationFileLocation, final PluginFactory pluginFactory) { + this.pipelineConfigurationFileLocation = pipelineConfigurationFileLocation; this.pluginFactory = Objects.requireNonNull(pluginFactory); } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java index 6df72f17c9..4f20cf05b4 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package com.amazon.dataprepper.parser.config; import org.opensearch.dataprepper.logstash.LogstashConfigConverter; diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/PipelineParserTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/PipelineParserTests.java index 2452150399..ce25b8d8d0 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/PipelineParserTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/PipelineParserTests.java @@ -6,7 +6,6 @@ package com.amazon.dataprepper.parser; import com.amazon.dataprepper.model.plugin.PluginFactory; -import com.amazon.dataprepper.parser.config.DataPrepperArgs; import com.amazon.dataprepper.pipeline.Pipeline; import com.amazon.dataprepper.plugin.DefaultPluginFactory; import org.junit.jupiter.api.BeforeEach; @@ -40,40 +39,35 @@ void setUp() { @Test void parseConfiguration_with_multiple_valid_pipelines_creates_the_correct_pipelineMap() { - final DataPrepperArgs args = new DataPrepperArgs(VALID_MULTIPLE_PIPELINE_CONFIG_FILE); - final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final PipelineParser pipelineParser = new PipelineParser(VALID_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); final Map actualPipelineMap = pipelineParser.parseConfiguration(); assertThat(actualPipelineMap.keySet(), equalTo(VALID_MULTIPLE_PIPELINE_NAMES)); } @Test void parseConfiguration_with_invalid_root_pipeline_creates_empty_pipelinesMap() { - final DataPrepperArgs args = new DataPrepperArgs(CONNECTED_PIPELINE_ROOT_SOURCE_INCORRECT); - final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final PipelineParser pipelineParser = new PipelineParser(CONNECTED_PIPELINE_ROOT_SOURCE_INCORRECT, pluginFactory); final Map connectedPipelines = pipelineParser.parseConfiguration(); assertThat(connectedPipelines.size(), equalTo(0)); } @Test void parseConfiguration_with_incorrect_child_pipeline_returns_empty_pipelinesMap() { - final DataPrepperArgs args = new DataPrepperArgs(CONNECTED_PIPELINE_CHILD_PIPELINE_INCORRECT); - final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final PipelineParser pipelineParser = new PipelineParser(CONNECTED_PIPELINE_CHILD_PIPELINE_INCORRECT, pluginFactory); final Map connectedPipelines = pipelineParser.parseConfiguration(); assertThat(connectedPipelines.size(), equalTo(0)); } @Test void parseConfiguration_with_a_single_pipeline_with_empty_source_settings_returns_that_pipeline() { - final DataPrepperArgs args = new DataPrepperArgs(VALID_SINGLE_PIPELINE_EMPTY_SOURCE_PLUGIN_FILE); - final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final PipelineParser pipelineParser = new PipelineParser(VALID_SINGLE_PIPELINE_EMPTY_SOURCE_PLUGIN_FILE, pluginFactory); final Map actualPipelineMap = pipelineParser.parseConfiguration(); assertThat(actualPipelineMap.keySet().size(), equalTo(1)); } @Test void parseConfiguration_with_cycles_in_multiple_pipelines_should_throw() { - final DataPrepperArgs args = new DataPrepperArgs(CYCLE_MULTIPLE_PIPELINE_CONFIG_FILE); - final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final PipelineParser pipelineParser = new PipelineParser(CYCLE_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); assertThat(actualException.getMessage(), @@ -83,8 +77,7 @@ void parseConfiguration_with_cycles_in_multiple_pipelines_should_throw() { @Test void parseConfiguration_with_incorrect_source_mapping_in_multiple_pipelines_should_throw() { - final DataPrepperArgs args = new DataPrepperArgs(INCORRECT_SOURCE_MULTIPLE_PIPELINE_CONFIG_FILE); - final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final PipelineParser pipelineParser = new PipelineParser(INCORRECT_SOURCE_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); assertThat(actualException.getMessage(), @@ -93,8 +86,7 @@ void parseConfiguration_with_incorrect_source_mapping_in_multiple_pipelines_shou @Test void parseConfiguration_with_missing_pipeline_name_should_throw() { - final DataPrepperArgs args = new DataPrepperArgs(MISSING_NAME_MULTIPLE_PIPELINE_CONFIG_FILE); - final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final PipelineParser pipelineParser = new PipelineParser(MISSING_NAME_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); assertThat(actualException.getMessage(), @@ -104,32 +96,28 @@ void parseConfiguration_with_missing_pipeline_name_should_throw() { @Test void parseConfiguration_with_missing_pipeline_name_in_multiple_pipelines_should_throw() { - final DataPrepperArgs args = new DataPrepperArgs(MISSING_PIPELINE_MULTIPLE_PIPELINE_CONFIG_FILE); - final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final PipelineParser pipelineParser = new PipelineParser(MISSING_PIPELINE_MULTIPLE_PIPELINE_CONFIG_FILE, pluginFactory); final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); assertThat(actualException.getMessage(), equalTo("Invalid configuration, no pipeline is defined with name test-pipeline-4")); } @Test void testMultipleSinks() { - final DataPrepperArgs args = new DataPrepperArgs(VALID_MULTIPLE_SINKS_CONFIG_FILE); - final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final PipelineParser pipelineParser = new PipelineParser(VALID_MULTIPLE_SINKS_CONFIG_FILE, pluginFactory); final Map pipelineMap = pipelineParser.parseConfiguration(); assertThat(pipelineMap.keySet().size(), equalTo(3)); } @Test void testMultipleProcessors() { - final DataPrepperArgs args = new DataPrepperArgs(VALID_MULTIPLE_PROCESSERS_CONFIG_FILE); - final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final PipelineParser pipelineParser = new PipelineParser(VALID_MULTIPLE_PROCESSERS_CONFIG_FILE, pluginFactory); final Map pipelineMap = pipelineParser.parseConfiguration(); assertThat(pipelineMap.keySet().size(), equalTo(3)); } @Test void parseConfiguration_with_a_configuration_file_which_does_not_exist_should_throw() { - final DataPrepperArgs args = new DataPrepperArgs("file_does_no_exist.yml"); - final PipelineParser pipelineParser = new PipelineParser(args, pluginFactory); + final PipelineParser pipelineParser = new PipelineParser("file_does_no_exist.yml", pluginFactory); final RuntimeException actualException = assertThrows(RuntimeException.class, pipelineParser::parseConfiguration); assertThat(actualException.getMessage(), equalTo("Failed to parse the configuration file file_does_no_exist.yml")); } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java index dd63b4737d..b914a17633 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package com.amazon.dataprepper.parser.config; import org.junit.jupiter.api.Test; From 6c54d842e57c1b470bd8ea739b47aa21df6e956c Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Wed, 5 Jan 2022 11:55:50 -0600 Subject: [PATCH 20/43] Remove commented code Signed-off-by: Steven Bayer --- .../com/amazon/dataprepper/server/DataPrepperServerTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java index bf840208b1..29b3d93dc6 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java @@ -98,9 +98,6 @@ public void setup() { when(pluginFactory.loadPlugin(eq(DataPrepperCoreAuthenticationProvider.class), argThat(a -> a.getName().equals(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)))) .thenReturn(unauthenticatedProvider); - -// when(dataPrepper.getPluginFactory()).thenReturn(pluginFactory); -// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); } @AfterEach From c5557b81de304ec515e19f4e94e12d54548cb271 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Wed, 5 Jan 2022 11:57:04 -0600 Subject: [PATCH 21/43] refactor test to assert against hard coded string Signed-off-by: Steven Bayer --- .../java/com/amazon/dataprepper/DataPrepperTests.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java index 4ae811e298..d2dd19413a 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java @@ -124,14 +124,9 @@ public void testShutdownDataPrepperServer() { } @Test - public void testGivenEnvVarNotSetThenDefaultServiceNameReturned() throws NoSuchFieldException, IllegalAccessException { - final Field dataPrepperServerField = DataPrepper.class.getDeclaredField("DEFAULT_SERVICE_NAME"); - dataPrepperServerField.setAccessible(true); - String expected = (String) dataPrepperServerField.get(null); - dataPrepperServerField.setAccessible(false); - + public void testGivenEnvVarNotSetThenDefaultServiceNameReturned() { String actual = DataPrepper.getServiceNameForMetrics(); - assertThat(actual, Matchers.is(expected)); + assertThat(actual, Matchers.is("dataprepper")); } } From 75e9e5234ef2b305c3c502d3ad56a36d06b12e79 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Wed, 5 Jan 2022 12:02:57 -0600 Subject: [PATCH 22/43] added final modifier to main class variables Signed-off-by: Steven Bayer --- .../java/com/amazon/dataprepper/DataPrepperExecute.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java index 38438e4bfd..01d702e439 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java @@ -18,19 +18,19 @@ public class DataPrepperExecute { private static final Logger LOG = LoggerFactory.getLogger(DataPrepperExecute.class); - public static void main(String[] args) { + public static void main(final String ... args) { java.security.Security.setProperty("networkaddress.cache.ttl", "60"); LOG.trace("Reading args"); - SimpleCommandLinePropertySource commandLinePropertySource = new SimpleCommandLinePropertySource(args); + final SimpleCommandLinePropertySource commandLinePropertySource = new SimpleCommandLinePropertySource(args); LOG.trace("Creating application context"); - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + final AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); context.getEnvironment().getPropertySources().addFirst(commandLinePropertySource); context.register(DataPrepperExecute.class); context.refresh(); - DataPrepper dataPrepper = context.getBean(DataPrepper.class); + final DataPrepper dataPrepper = context.getBean(DataPrepper.class); LOG.trace("Starting Data Prepper execution"); dataPrepper.execute(); From 9ef10609215f19f801ddc0e6604e0786cb8ed504 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Thu, 6 Jan 2022 12:12:01 -0600 Subject: [PATCH 23/43] Fixed misplaced Nullable Signed-off-by: Steven Bayer --- .../com/amazon/dataprepper/parser/config/DataPrepperArgs.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java index 4f20cf05b4..cdf9520b6b 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java @@ -70,11 +70,11 @@ private void invalidArgumentsReceived(final String msg) { throw new IllegalArgumentException(msg); } - @Nullable public String getPipelineConfigFileLocation() { return pipelineConfigFileLocation; } + @Nullable public String getDataPrepperConfigFileLocation() { return dataPrepperConfigFileLocation; } From e6d80419631d4eb7d17c09af7580e185abec9444 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Mon, 20 Dec 2021 10:17:44 -0600 Subject: [PATCH 24/43] Add Spring Core dependency Signed-off-by: Steven Bayer --- .../DataPrepperConfigurationConfiguration.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java new file mode 100644 index 0000000000..6464d46bbf --- /dev/null +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java @@ -0,0 +1,16 @@ +package com.amazon.dataprepper.parser.config; + +import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import javax.inject.Named; + +@Configuration +public class DataPrepperConfigurationConfiguration { + + @Bean + public DataPrepperConfiguration dataPrepperDefaultConfiguration(ApplicationArguments args) { + return new DataPrepperConfiguration(); + } +} From 85c84aeaba09af8b8c996b266e01f83e01f9ef6c Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Thu, 23 Dec 2021 14:54:31 -0600 Subject: [PATCH 25/43] Converted DataPrepperServer to use Spring Dependency Injection Signed-off-by: Steven Bayer --- .../config/DataPrepperAppConfiguration.java | 26 ++++ .../parser/config/MetricsConfig.java | 68 +++++++++-- .../parser/model/MetricRegistryType.java | 28 ----- .../pipeline/server/DataPrepperServer.java | 111 +++--------------- .../DataPrepperServerConfiguration.java | 111 ++++++++++++++++++ shared-config/log4j2.properties | 3 + 6 files changed, 219 insertions(+), 128 deletions(-) create mode 100644 data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java index 699bb7b395..3d446f5700 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java @@ -5,8 +5,10 @@ package com.amazon.dataprepper.parser.config; +import com.amazon.dataprepper.model.configuration.PluginModel; import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; +import org.opensearch.dataprepper.logstash.LogstashConfigConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Bean; @@ -14,8 +16,12 @@ import org.springframework.core.env.CommandLinePropertySource; import org.springframework.core.env.Environment; +import javax.swing.*; import java.io.File; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Optional; @Configuration public class DataPrepperAppConfiguration { @@ -55,4 +61,24 @@ public DataPrepperConfiguration dataPrepperConfiguration( return new DataPrepperConfiguration(); } } + + @Bean + public Optional pluginModel(final DataPrepperConfiguration dataPrepperConfiguration) { + return Optional.ofNullable(dataPrepperConfiguration.getAuthentication()); + } + + private static String checkForLogstashConfigurationAndConvert(String configurationFileLocation) { + if (configurationFileLocation.endsWith(".conf")) { + final LogstashConfigConverter logstashConfigConverter = new LogstashConfigConverter(); + final Path configurationDirectory = Paths.get(configurationFileLocation).toAbsolutePath().getParent(); + + try { + configurationFileLocation = logstashConfigConverter.convertLogstashConfigurationToPipeline( + configurationFileLocation, String.valueOf(configurationDirectory)); + } catch (IOException e) { + LOG.error("Unable to read the Logstash configuration file", e); + } + } + return configurationFileLocation; + } } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java index d55572da08..9acaa1945e 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java @@ -7,10 +7,13 @@ import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.parser.model.MetricRegistryType; +import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.binder.MeterBinder; import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; @@ -18,10 +21,17 @@ import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; import io.micrometer.core.instrument.binder.system.ProcessorMetrics; import io.micrometer.core.instrument.composite.CompositeMeterRegistry; +import io.micrometer.prometheus.PrometheusConfig; +import io.micrometer.prometheus.PrometheusMeterRegistry; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import java.util.Arrays; import java.util.List; +import java.util.Optional; + +import static com.amazon.dataprepper.DataPrepper.getServiceNameForMetrics; +import static com.amazon.dataprepper.metrics.MetricNames.SERVICE_NAME; @Configuration public class MetricsConfig { @@ -60,21 +70,63 @@ public JvmThreadMetrics jvmThreadMetrics() { return new JvmThreadMetrics(); } + private void configureMetricRegistry(MeterRegistry meterRegistry) { + meterRegistry.config() + .commonTags(Arrays.asList( + Tag.of(SERVICE_NAME, getServiceNameForMetrics()) + )); + } + + @Bean + public Optional prometheusMeterRegistry(final DataPrepperConfiguration dataPrepperConfiguration) { + if (dataPrepperConfiguration.getMetricRegistryTypes().contains(MetricRegistryType.Prometheus)) { + PrometheusMeterRegistry meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + configureMetricRegistry(meterRegistry); + return Optional.of(meterRegistry); + } + else { + return Optional.empty(); + } + } + + @Bean + public CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider() { + return new CloudWatchMeterRegistryProvider(); + } + + @Bean + public Optional cloudWatchMeterRegistry( + final DataPrepperConfiguration dataPrepperConfiguration, + CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider + ) { + if (dataPrepperConfiguration.getMetricRegistryTypes().contains(MetricRegistryType.CloudWatch)) { + CloudWatchMeterRegistry meterRegistry = cloudWatchMeterRegistryProvider.getCloudWatchMeterRegistry(); + configureMetricRegistry(meterRegistry); + return Optional.of(meterRegistry); + } + else { + return Optional.empty(); + } + } + @Bean public CompositeMeterRegistry systemMeterRegistry( final List meterBinders, + final List> optionalMeterRegistries, final DataPrepperConfiguration dataPrepperConfiguration ) { - final CompositeMeterRegistry meterRegistry = new CompositeMeterRegistry(); + final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); - meterBinders.forEach(binder -> binder.bindTo(meterRegistry)); + meterBinders.forEach(binder -> binder.bindTo(compositeMeterRegistry)); - dataPrepperConfiguration.getMetricRegistryTypes().forEach(metricRegistryType -> { - MeterRegistry registryForType = MetricRegistryType.getDefaultMeterRegistryForType(metricRegistryType); - meterRegistry.add(registryForType); - Metrics.addRegistry(registryForType); - }); + optionalMeterRegistries.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(meterRegistry -> { + compositeMeterRegistry.add(meterRegistry); + Metrics.addRegistry(meterRegistry); + }); - return meterRegistry; + return compositeMeterRegistry; } } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java index 1c2c17af2b..1a0178da68 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java @@ -5,35 +5,7 @@ package com.amazon.dataprepper.parser.model; -import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Tag; -import io.micrometer.prometheus.PrometheusConfig; -import io.micrometer.prometheus.PrometheusMeterRegistry; - -import java.util.Arrays; - -import static com.amazon.dataprepper.DataPrepper.getServiceNameForMetrics; -import static com.amazon.dataprepper.metrics.MetricNames.SERVICE_NAME; -import static java.lang.String.format; - public enum MetricRegistryType { Prometheus, CloudWatch; - - public static MeterRegistry getDefaultMeterRegistryForType(final MetricRegistryType metricRegistryType) { - MeterRegistry meterRegistry = null; - switch (metricRegistryType) { - case Prometheus: - meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); - break; - case CloudWatch: - meterRegistry = new CloudWatchMeterRegistryProvider().getCloudWatchMeterRegistry(); - break; - default: - throw new IllegalArgumentException(format("Invalid metricRegistryType %s", metricRegistryType)); - } - meterRegistry.config().commonTags(Arrays.asList(Tag.of(SERVICE_NAME, getServiceNameForMetrics()))); - return meterRegistry; - } } \ No newline at end of file diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java index 5e7a93d3f5..53c7e72b4c 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java @@ -5,30 +5,18 @@ package com.amazon.dataprepper.pipeline.server; -import com.amazon.dataprepper.DataPrepper; -import com.amazon.dataprepper.model.configuration.PluginModel; -import com.amazon.dataprepper.model.configuration.PluginSetting; -import com.amazon.dataprepper.model.plugin.PluginFactory; -import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.sun.net.httpserver.Authenticator; +import com.sun.net.httpserver.HttpContext; import com.sun.net.httpserver.HttpServer; -import com.sun.net.httpserver.HttpsConfigurator; -import com.sun.net.httpserver.HttpsParameters; -import com.sun.net.httpserver.HttpsServer; import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Metrics; -import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import io.micrometer.prometheus.PrometheusMeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.inject.Inject; import javax.inject.Named; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLParameters; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Collections; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -45,65 +33,26 @@ public class DataPrepperServer { @Inject public DataPrepperServer( - final DataPrepperConfiguration dataPrepperConfiguration, - final PluginFactory pluginFactory, - final DataPrepper dataPrepper, - final CompositeMeterRegistry systemMeterRegistry + final Optional prometheusMeterRegistry, + final Optional optionalAuthenticator, + final ListPipelinesHandler listPipelinesHandler, + final ShutdownHandler shutdownHandler, + final HttpServer server ) { - final int port = dataPrepperConfiguration.getServerPort(); - final boolean ssl = dataPrepperConfiguration.ssl(); - final String keyStoreFilePath = dataPrepperConfiguration.getKeyStoreFilePath(); - final String keyStorePassword = dataPrepperConfiguration.getKeyStorePassword(); - final String privateKeyPassword = dataPrepperConfiguration.getPrivateKeyPassword(); + this.server = server; - final PluginModel authenticationConfiguration = dataPrepperConfiguration.getAuthentication(); - final PluginSetting authenticationPluginSetting; - - if (authenticationConfiguration == null || authenticationConfiguration.getPluginName().equals(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)) { - LOG.warn("Creating data prepper server without authentication. This is not secure."); - LOG.warn("In order to set up Http Basic authentication for the data prepper server, go here: https://github.com/opensearch-project/data-prepper/blob/main/docs/core_apis.md#authentication"); - } - - if(authenticationConfiguration != null) { - authenticationPluginSetting = - new PluginSetting(authenticationConfiguration.getPluginName(), authenticationConfiguration.getPluginSettings()); - } else { - authenticationPluginSetting = - new PluginSetting(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME, Collections.emptyMap()); - } - - final DataPrepperCoreAuthenticationProvider authenticationProvider = pluginFactory.loadPlugin(DataPrepperCoreAuthenticationProvider.class, authenticationPluginSetting); - final Authenticator authenticator = authenticationProvider.getAuthenticator(); - - try { - if (ssl) { - LOG.info("Creating Data Prepper server with TLS"); - this.server = createHttpsServer(port, keyStoreFilePath, keyStorePassword, privateKeyPassword); - } else { - LOG.warn("Creating Data Prepper server without TLS. This is not secure."); - LOG.warn("In order to set up TLS for the Data Prepper server, go here: https://github.com/opensearch-project/data-prepper/blob/main/docs/configuration.md#server-configuration"); - this.server = createHttpServer(port); - } - } catch (final IOException ex) { - throw new RuntimeException("Failed to create server", ex); - } - - getPrometheusMeterRegistryFromRegistries(Metrics.globalRegistry.getRegistries()).ifPresent(meterRegistry -> { - final PrometheusMeterRegistry prometheusMeterRegistryForDataPrepper = (PrometheusMeterRegistry) meterRegistry; - server.createContext("/metrics/prometheus", new PrometheusMetricsHandler(prometheusMeterRegistryForDataPrepper)) - .setAuthenticator(authenticator); + List contextList = new ArrayList<>(4); + prometheusMeterRegistry.ifPresent(metricRegistry -> { + contextList.add(server.createContext("/metrics/prometheus", new PrometheusMetricsHandler(metricRegistry))); + contextList.add(server.createContext("/metrics/sys", new PrometheusMetricsHandler(metricRegistry))); }); - getPrometheusMeterRegistryFromRegistries(systemMeterRegistry.getRegistries()).ifPresent( - meterRegistry -> { - final PrometheusMeterRegistry prometheusMeterRegistryForSystem = (PrometheusMeterRegistry) meterRegistry; - server.createContext("/metrics/sys", new PrometheusMetricsHandler(prometheusMeterRegistryForSystem)) - .setAuthenticator(authenticator); - }); - server.createContext("/list", new ListPipelinesHandler(dataPrepper)) - .setAuthenticator(authenticator); - server.createContext("/shutdown", new ShutdownHandler(dataPrepper)) - .setAuthenticator(authenticator); + contextList.add(server.createContext("/list", listPipelinesHandler)); + contextList.add(server.createContext("/shutdown", shutdownHandler)); + + optionalAuthenticator.ifPresent( + authenticator -> contextList.forEach( + context -> context.setAuthenticator(authenticator))); } private Optional getPrometheusMeterRegistryFromRegistries(final Set meterRegistries) { @@ -127,26 +76,4 @@ public void stop() { server.stop(0); LOG.info("Data Prepper server stopped"); } - - private HttpServer createHttpServer(final int port) throws IOException { - return HttpServer.create(new InetSocketAddress(port), 0); - } - - private HttpServer createHttpsServer(final int port, - final String keyStoreFilePath, - final String keyStorePassword, - final String privateKeyPassword) throws IOException { - final SSLContext sslContext = SslUtil.createSslContext(keyStoreFilePath, keyStorePassword, privateKeyPassword); - - final HttpsServer server = HttpsServer.create(new InetSocketAddress(port), 0); - server.setHttpsConfigurator(new HttpsConfigurator(sslContext) { - public void configure(HttpsParameters params) { - SSLContext context = getSSLContext(); - SSLParameters sslparams = context.getDefaultSSLParameters(); - params.setSSLParameters(sslparams); - } - }); - - return server; - } } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java new file mode 100644 index 0000000000..19929aa148 --- /dev/null +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java @@ -0,0 +1,111 @@ +package com.amazon.dataprepper.pipeline.server.config; + +import com.amazon.dataprepper.DataPrepper; +import com.amazon.dataprepper.model.configuration.PluginModel; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.model.plugin.PluginFactory; +import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; +import com.amazon.dataprepper.pipeline.server.DataPrepperCoreAuthenticationProvider; +import com.amazon.dataprepper.pipeline.server.ListPipelinesHandler; +import com.amazon.dataprepper.pipeline.server.ShutdownHandler; +import com.amazon.dataprepper.pipeline.server.SslUtil; +import com.sun.net.httpserver.Authenticator; +import com.sun.net.httpserver.HttpServer; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsParameters; +import com.sun.net.httpserver.HttpsServer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.Optional; + +@Configuration +public class DataPrepperServerConfiguration { + private static final Logger LOG = LoggerFactory.getLogger(DataPrepperServerConfiguration.class); + + @Bean + public HttpServer httpServer(final DataPrepperConfiguration dataPrepperConfiguration) { + InetSocketAddress socketAddress = new InetSocketAddress(dataPrepperConfiguration.getServerPort()); + try { + if (dataPrepperConfiguration.ssl()) { + LOG.info("Creating Data Prepper server with TLS"); + final SSLContext sslContext = SslUtil.createSslContext( + dataPrepperConfiguration.getKeyStoreFilePath(), + dataPrepperConfiguration.getKeyStorePassword(), + dataPrepperConfiguration.getPrivateKeyPassword() + ); + + final HttpsServer server = HttpsServer.create(socketAddress, 0); + server.setHttpsConfigurator(new HttpsConfigurator(sslContext) { + public void configure(HttpsParameters params) { + SSLContext context = getSSLContext(); + SSLParameters sslparams = context.getDefaultSSLParameters(); + params.setSSLParameters(sslparams); + } + }); + + return server; + } else { + return HttpServer.create(socketAddress, 0); + } + } catch (final IOException ex) { + throw new RuntimeException("Failed to create server", ex); + } + } + + private void printInsecurePluginModelWarning() { + LOG.warn("Creating data prepper server without authentication. This is not secure."); + LOG.warn("In order to set up Http Basic authentication for the data prepper server, go here: https://github.com/opensearch-project/data-prepper/blob/main/docs/core_apis.md#authentication"); + } + + @Bean + public PluginSetting pluginSetting(final Optional optionalPluginModel) { + if (optionalPluginModel.isPresent()) { + PluginModel pluginModel = optionalPluginModel.get(); + String pluginName = pluginModel.getPluginName(); + if (pluginName.equals(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)) { + printInsecurePluginModelWarning(); + } + return new PluginSetting(pluginName, pluginModel.getPluginSettings()); + } + else { + printInsecurePluginModelWarning(); + return new PluginSetting( + DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME, + Collections.emptyMap()); + } + } + + @Bean + public DataPrepperCoreAuthenticationProvider authenticationProvider( + final PluginFactory pluginFactory, + final PluginSetting pluginSetting + ) { + return pluginFactory.loadPlugin( + DataPrepperCoreAuthenticationProvider.class, + pluginSetting + ); + } + + @Bean + public Optional optionalAuthenticator(final DataPrepperCoreAuthenticationProvider authenticationProvider) { + return Optional.ofNullable(authenticationProvider.getAuthenticator()); + } + + @Bean + public ListPipelinesHandler listPipelinesHandler(final DataPrepper dataPrepper) { + return new ListPipelinesHandler(dataPrepper); + } + + @Bean + public ShutdownHandler shutdownHandler(final DataPrepper dataPrepper) { + return new ShutdownHandler(dataPrepper); + } +} diff --git a/shared-config/log4j2.properties b/shared-config/log4j2.properties index df001bc6c8..9c5d0e50f9 100644 --- a/shared-config/log4j2.properties +++ b/shared-config/log4j2.properties @@ -28,3 +28,6 @@ logger.parser.level = info logger.plugins.name = com.amazon.dataprepper.plugins logger.plugins.level = info + +logger.springframework.name = org.springframework +logger.springframework.level = info From 835bee55bc491842ecf5e3bc2ac7b96b8e97b49e Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Thu, 23 Dec 2021 15:30:12 -0600 Subject: [PATCH 26/43] Added DataPrepperServer constructor test Signed-off-by: Steven Bayer --- .../server/DataPrepperServerTest.java | 690 +++++++++--------- 1 file changed, 342 insertions(+), 348 deletions(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java index 29b3d93dc6..7285f6b4c3 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java @@ -5,377 +5,371 @@ package com.amazon.dataprepper.server; -import com.amazon.dataprepper.DataPrepper; -import com.amazon.dataprepper.TestDataProvider; -import com.amazon.dataprepper.model.plugin.PluginFactory; -import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; -import com.amazon.dataprepper.pipeline.server.DataPrepperCoreAuthenticationProvider; import com.amazon.dataprepper.pipeline.server.DataPrepperServer; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Metrics; -import io.micrometer.core.instrument.composite.CompositeMeterRegistry; -import io.micrometer.prometheus.PrometheusConfig; +import com.amazon.dataprepper.pipeline.server.ListPipelinesHandler; +import com.amazon.dataprepper.pipeline.server.ShutdownHandler; +import com.sun.net.httpserver.Authenticator; +import com.sun.net.httpserver.HttpContext; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; import io.micrometer.prometheus.PrometheusMeterRegistry; -import org.hamcrest.Matchers; -import org.junit.Assert; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import java.io.File; -import java.io.IOException; -import java.net.ConnectException; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; -import java.util.Properties; -import java.util.Set; -import java.util.UUID; +import java.util.Optional; -import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE; -import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.eq; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; + @ExtendWith(MockitoExtension.class) public class DataPrepperServerTest { + + private static final PrometheusMeterRegistry prometheusMeterRegistry = mock(PrometheusMeterRegistry.class); + private static final Authenticator authenticator = mock(Authenticator.class); + @Mock - DataPrepperConfiguration dataPrepperConfiguration; - @Mock - PluginFactory pluginFactory; - @Mock - DataPrepper dataPrepper; + ListPipelinesHandler listPipelinesHandler; @Mock - DataPrepperCoreAuthenticationProvider unauthenticatedProvider; + ShutdownHandler shutdownHandler; @Mock - CompositeMeterRegistry systemMeterRegistry; - - private static ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()); - private static ObjectMapper JSON_OBJECT_MAPPER = new ObjectMapper(); - private DataPrepperServer dataPrepperServer; - private final int port = 1234; - - private void setRegistry(PrometheusMeterRegistry prometheusMeterRegistry) { - final Set meterRegistries = Metrics.globalRegistry.getRegistries(); - //to avoid ConcurrentModificationException - final Object[] registeredMeterRegistries = meterRegistries.toArray(); - for (final Object meterRegistry : registeredMeterRegistries) { - Metrics.removeRegistry((MeterRegistry) meterRegistry); - } - Metrics.addRegistry(prometheusMeterRegistry); - } - - @BeforeEach - public void setup() { - // Required to trust any self-signed certs, used in https tests - Properties props = System.getProperties(); - props.setProperty("jdk.internal.httpclient.disableHostnameVerification", Boolean.TRUE.toString()); - - setRegistry(new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, "")); - pluginFactory = mock(PluginFactory.class); - - when(pluginFactory.loadPlugin(eq(DataPrepperCoreAuthenticationProvider.class), - argThat(a -> a.getName().equals(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)))) - .thenReturn(unauthenticatedProvider); - } - - @AfterEach - public void stopServer() { - if (dataPrepperServer != null) { - dataPrepperServer.stop(); - } - } - - @Test - public void testGetGlobalMetrics() throws IOException, InterruptedException, URISyntaxException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - final String scrape = UUID.randomUUID().toString(); - final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); - setRegistry(prometheusMeterRegistry); - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); - Assert.assertEquals(scrape, response.body()); - dataPrepperServer.stop(); - } - - @Test - public void testScrapeGlobalFailure() throws IOException, InterruptedException, URISyntaxException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - setRegistry(new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT)); - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); - dataPrepperServer.stop(); - } - - @Test - public void testGetSystemMetrics() throws IOException, InterruptedException, URISyntaxException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - final String scrape = UUID.randomUUID().toString(); - final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); - - when(systemMeterRegistry.getRegistries()) - .thenReturn(Set.of(prometheusMeterRegistry)); - - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); - dataPrepperServer.stop(); - } - - @Test - public void testScrapeSystemMetricsFailure() throws IOException, InterruptedException, URISyntaxException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT); - - when(systemMeterRegistry.getRegistries()) - .thenReturn(Set.of(prometheusMeterRegistry)); - - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); - dataPrepperServer.stop(); - } - - @Test - public void testNonPrometheusMeterRegistry() throws Exception { - final CloudWatchMeterRegistry cloudWatchMeterRegistry = mock(CloudWatchMeterRegistry.class); - final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); - compositeMeterRegistry.add(cloudWatchMeterRegistry); - final DataPrepperConfiguration config = makeConfig(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE); - try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { - dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) - .GET().build(); - HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - } catch (ConnectException ex) { - dataPrepperServer.stop(); - //there should not be any Prometheus endpoints available - assertThat(ex.getMessage(), Matchers.is("Connection refused")); - } - } - - @Test - public void testMultipleMeterRegistries() throws Exception { - //for system metrics - final CloudWatchMeterRegistry cloudWatchMeterRegistryForSystem = mock(CloudWatchMeterRegistry.class); - final PrometheusMeterRegistry prometheusMeterRegistryForSystem = - new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); - - when(systemMeterRegistry.getRegistries()) - .thenReturn(Set.of(cloudWatchMeterRegistryForSystem, prometheusMeterRegistryForSystem)); + HttpServer server; - //for data prepper metrics - final PrometheusMeterRegistry prometheusMeterRegistryForDataPrepper = - new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); - setRegistry(prometheusMeterRegistryForDataPrepper); + DataPrepperServer dataPrepperServer; - final DataPrepperConfiguration config = makeConfig(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE); - - dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - //test prometheus registry - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + config.getServerPort() - + "/metrics/sys")).GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); - dataPrepperServer.stop(); + @BeforeAll + public static void beforeAll() { } @Test - public void testListPipelines() throws URISyntaxException, IOException, InterruptedException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - final String pipelineName = "testPipeline"; - Mockito.when(dataPrepper.getTransformationPipelines()).thenReturn( - Collections.singletonMap("testPipeline", null) - ); - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - final String expectedResponse = JSON_OBJECT_MAPPER.writeValueAsString( - Collections.singletonMap("pipelines", Arrays.asList( - Collections.singletonMap("name", pipelineName) - )) - ); - Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); - Assert.assertEquals(expectedResponse, response.body()); - dataPrepperServer.stop(); + public void testDataPrepperServerConstructor() { + when(server.createContext(anyString(), any(HttpHandler.class))) + .thenReturn(mock(HttpContext.class)); + + dataPrepperServer = new DataPrepperServer( + Optional.of(prometheusMeterRegistry), + Optional.of(authenticator), + listPipelinesHandler, + shutdownHandler, + server); + + assertThat(dataPrepperServer, is(notNullValue())); + verify(server, times(4)).createContext(anyString(), any(HttpHandler.class)); } - @Test - public void testListPipelinesFailure() throws URISyntaxException, IOException, InterruptedException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - Mockito.when(dataPrepper.getTransformationPipelines()).thenThrow(new RuntimeException("")); - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); - dataPrepperServer.stop(); - } - - @Test - public void testShutdown() throws URISyntaxException, IOException, InterruptedException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); - Mockito.verify(dataPrepper).shutdown(); - dataPrepperServer.stop(); - } - - @Test - public void testShutdownFailure() throws URISyntaxException, IOException, InterruptedException { - when(dataPrepperConfiguration.getServerPort()).thenReturn(port); - Mockito.doThrow(new RuntimeException("")).when(dataPrepper).shutdown(); - dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) - .GET().build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); - Mockito.verify(dataPrepper).shutdown(); - dataPrepperServer.stop(); - } - - @Test - public void testGetMetricsWithHttps() throws Exception { - DataPrepperConfiguration config = DataPrepperServerTest.makeConfig( - TestDataProvider.VALID_DATA_PREPPER_CONFIG_FILE_WITH_TLS); - final String scrape = UUID.randomUUID().toString(); - final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); - setRegistry(prometheusMeterRegistry); - dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); - dataPrepperServer.start(); - - HttpClient httpsClient = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(3)) - .sslContext(getSslContextTrustingInsecure()) - .build(); - - HttpRequest request = HttpRequest.newBuilder(new URI("https://localhost:" + port + "/metrics/prometheus")) - .GET() - .timeout(Duration.ofSeconds(3)) - .build(); - - HttpResponse response = httpsClient.send(request, HttpResponse.BodyHandlers.ofString()); - Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); - Assert.assertEquals(scrape, response.body()); - dataPrepperServer.stop(); - } - - @Test - public void testTlsWithInvalidPassword() throws IOException { - DataPrepperConfiguration config = DataPrepperServerTest.makeConfig( - TestDataProvider.INVALID_KEYSTORE_PASSWORD_DATA_PREPPER_CONFIG_FILE); - - assertThrows(IllegalStateException.class, () -> new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry)); - } - - private SSLContext getSslContextTrustingInsecure() throws Exception { - TrustManager[] trustAllCerts = new TrustManager[]{ - new X509TrustManager() { - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return null; - } - - public void checkClientTrusted( - java.security.cert.X509Certificate[] certs, String authType) { - } - - public void checkServerTrusted( - java.security.cert.X509Certificate[] certs, String authType) { - } - } - }; - - SSLContext sslContext = SSLContext.getInstance("SSL"); - sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); - - return sslContext; - } - - private static class PrometheusRegistryMockScrape extends PrometheusMeterRegistry { - final String scrape; - - public PrometheusRegistryMockScrape(PrometheusConfig config, final String scrape) { - super(config); - this.scrape = scrape; - } - - @Override - public String scrape() { - return scrape; - } - } - - private static class PrometheusRegistryThrowingScrape extends PrometheusMeterRegistry { - - public PrometheusRegistryThrowingScrape(PrometheusConfig config) { - super(config); - } - - @Override - public String scrape() { - throw new RuntimeException(""); - } - } - - private static DataPrepperConfiguration makeConfig(String filePath) throws IOException { - final File configurationFile = new File(filePath); - return OBJECT_MAPPER.readValue(configurationFile, DataPrepperConfiguration.class); - } +// private void setRegistry(PrometheusMeterRegistry prometheusMeterRegistry) { +// final Set meterRegistries = Metrics.globalRegistry.getRegistries(); +// //to avoid ConcurrentModificationException +// final Object[] registeredMeterRegistries = meterRegistries.toArray(); +// for (final Object meterRegistry : registeredMeterRegistries) { +// Metrics.removeRegistry((MeterRegistry) meterRegistry); +// } +// Metrics.addRegistry(prometheusMeterRegistry); +// } +// +// @BeforeEach +// public void setup() { +// // Required to trust any self-signed certs, used in https tests +// Properties props = System.getProperties(); +// props.setProperty("jdk.internal.httpclient.disableHostnameVerification", Boolean.TRUE.toString()); +// +// setRegistry(new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, "")); +// pluginFactory = mock(PluginFactory.class); +// +// when(pluginFactory.loadPlugin(eq(DataPrepperCoreAuthenticationProvider.class), +// argThat(a -> a.getName().equals(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)))) +// .thenReturn(unauthenticatedProvider); +// +//// when(dataPrepper.getPluginFactory()).thenReturn(pluginFactory); +//// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); +// } +// +// @AfterEach +// public void stopServer() { +// if (dataPrepperServer != null) { +// dataPrepperServer.stop(); +// } +// } +// +// @Test +// public void testGetGlobalMetrics() throws IOException, InterruptedException, URISyntaxException { +// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); +// final String scrape = UUID.randomUUID().toString(); +// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); +// setRegistry(prometheusMeterRegistry); +// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); +// +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); +// Assert.assertEquals(scrape, response.body()); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testScrapeGlobalFailure() throws IOException, InterruptedException, URISyntaxException { +// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); +// setRegistry(new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT)); +// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testGetSystemMetrics() throws IOException, InterruptedException, URISyntaxException { +// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); +// final String scrape = UUID.randomUUID().toString(); +// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); +// +// when(systemMeterRegistry.getRegistries()) +// .thenReturn(Set.of(prometheusMeterRegistry)); +// +// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testScrapeSystemMetricsFailure() throws IOException, InterruptedException, URISyntaxException { +// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); +// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT); +// +// when(systemMeterRegistry.getRegistries()) +// .thenReturn(Set.of(prometheusMeterRegistry)); +// +// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testNonPrometheusMeterRegistry() throws Exception { +// final CloudWatchMeterRegistry cloudWatchMeterRegistry = mock(CloudWatchMeterRegistry.class); +// final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); +// compositeMeterRegistry.add(cloudWatchMeterRegistry); +// final DataPrepperConfiguration config = makeConfig(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE); +// try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { +// dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); +// dataPrepperServer.start(); +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) +// .GET().build(); +// HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// } catch (ConnectException ex) { +// dataPrepperServer.stop(); +// //there should not be any Prometheus endpoints available +// assertThat(ex.getMessage(), Matchers.is("Connection refused")); +// } +// } +// +// @Test +// public void testMultipleMeterRegistries() throws Exception { +// //for system metrics +// final CloudWatchMeterRegistry cloudWatchMeterRegistryForSystem = mock(CloudWatchMeterRegistry.class); +// final PrometheusMeterRegistry prometheusMeterRegistryForSystem = +// new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); +// +// when(systemMeterRegistry.getRegistries()) +// .thenReturn(Set.of(cloudWatchMeterRegistryForSystem, prometheusMeterRegistryForSystem)); +// +// //for data prepper metrics +// final PrometheusMeterRegistry prometheusMeterRegistryForDataPrepper = +// new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); +// setRegistry(prometheusMeterRegistryForDataPrepper); +// +// final DataPrepperConfiguration config = makeConfig(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE); +// +// dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); +// dataPrepperServer.start(); +// +// //test prometheus registry +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + config.getServerPort() +// + "/metrics/sys")).GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testListPipelines() throws URISyntaxException, IOException, InterruptedException { +// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); +// final String pipelineName = "testPipeline"; +// Mockito.when(dataPrepper.getTransformationPipelines()).thenReturn( +// Collections.singletonMap("testPipeline", null) +// ); +// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// final String expectedResponse = JSON_OBJECT_MAPPER.writeValueAsString( +// Collections.singletonMap("pipelines", Arrays.asList( +// Collections.singletonMap("name", pipelineName) +// )) +// ); +// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); +// Assert.assertEquals(expectedResponse, response.body()); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testListPipelinesFailure() throws URISyntaxException, IOException, InterruptedException { +// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); +// Mockito.when(dataPrepper.getTransformationPipelines()).thenThrow(new RuntimeException("")); +// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testShutdown() throws URISyntaxException, IOException, InterruptedException { +// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); +// +// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); +// Mockito.verify(dataPrepper).shutdown(); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testShutdownFailure() throws URISyntaxException, IOException, InterruptedException { +// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); +// Mockito.doThrow(new RuntimeException("")).when(dataPrepper).shutdown(); +// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); +// dataPrepperServer.start(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) +// .GET().build(); +// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); +// Mockito.verify(dataPrepper).shutdown(); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testGetMetricsWithHttps() throws Exception { +// DataPrepperConfiguration config = DataPrepperServerTest.makeConfig( +// TestDataProvider.VALID_DATA_PREPPER_CONFIG_FILE_WITH_TLS); +// final String scrape = UUID.randomUUID().toString(); +// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); +// setRegistry(prometheusMeterRegistry); +// dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); +// dataPrepperServer.start(); +// +// HttpClient httpsClient = HttpClient.newBuilder() +// .connectTimeout(Duration.ofSeconds(3)) +// .sslContext(getSslContextTrustingInsecure()) +// .build(); +// +// HttpRequest request = HttpRequest.newBuilder(new URI("https://localhost:" + port + "/metrics/prometheus")) +// .GET() +// .timeout(Duration.ofSeconds(3)) +// .build(); +// +// HttpResponse response = httpsClient.send(request, HttpResponse.BodyHandlers.ofString()); +// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); +// Assert.assertEquals(scrape, response.body()); +// dataPrepperServer.stop(); +// } +// +// @Test +// public void testTlsWithInvalidPassword() throws IOException { +// DataPrepperConfiguration config = DataPrepperServerTest.makeConfig( +// TestDataProvider.INVALID_KEYSTORE_PASSWORD_DATA_PREPPER_CONFIG_FILE); +// +// assertThrows(IllegalStateException.class, () -> new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry)); +// } +// +// private SSLContext getSslContextTrustingInsecure() throws Exception { +// TrustManager[] trustAllCerts = new TrustManager[]{ +// new X509TrustManager() { +// public java.security.cert.X509Certificate[] getAcceptedIssuers() { +// return null; +// } +// +// public void checkClientTrusted( +// java.security.cert.X509Certificate[] certs, String authType) { +// } +// +// public void checkServerTrusted( +// java.security.cert.X509Certificate[] certs, String authType) { +// } +// } +// }; +// +// SSLContext sslContext = SSLContext.getInstance("SSL"); +// sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); +// +// return sslContext; +// } +// +// private static class PrometheusRegistryMockScrape extends PrometheusMeterRegistry { +// final String scrape; +// +// public PrometheusRegistryMockScrape(PrometheusConfig config, final String scrape) { +// super(config); +// this.scrape = scrape; +// } +// +// @Override +// public String scrape() { +// return scrape; +// } +// } +// +// private static class PrometheusRegistryThrowingScrape extends PrometheusMeterRegistry { +// +// public PrometheusRegistryThrowingScrape(PrometheusConfig config) { +// super(config); +// } +// +// @Override +// public String scrape() { +// throw new RuntimeException(""); +// } +// } +// +// private static DataPrepperConfiguration makeConfig(String filePath) throws IOException { +// final File configurationFile = new File(filePath); +// return OBJECT_MAPPER.readValue(configurationFile, DataPrepperConfiguration.class); +// } } From 71f13e823572c73906bd71f56e74348a32568ef7 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Wed, 5 Jan 2022 09:33:36 -0600 Subject: [PATCH 27/43] remove unused swing dependency Signed-off-by: Steven Bayer --- .../dataprepper/parser/config/DataPrepperAppConfiguration.java | 1 - 1 file changed, 1 deletion(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java index 3d446f5700..40ccbade81 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java @@ -16,7 +16,6 @@ import org.springframework.core.env.CommandLinePropertySource; import org.springframework.core.env.Environment; -import javax.swing.*; import java.io.File; import java.io.IOException; import java.nio.file.Path; From ec611941fa0b4fbdded43484995bd32b0c0f21c1 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Wed, 5 Jan 2022 17:20:20 -0600 Subject: [PATCH 28/43] Added testing for new classes Signed-off-by: Steven Bayer --- ...DataPrepperConfigurationConfiguration.java | 16 - .../parser/config/MetricsConfig.java | 47 ++- .../pipeline/server/DataPrepperServer.java | 31 -- .../DataPrepperServerConfiguration.java | 27 +- .../parser/config/MetricsConfigTest.java | 176 +++++++-- .../server/DataPrepperServerTest.java | 361 ++---------------- 6 files changed, 237 insertions(+), 421 deletions(-) delete mode 100644 data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java deleted file mode 100644 index 6464d46bbf..0000000000 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperConfigurationConfiguration.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.amazon.dataprepper.parser.config; - -import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import javax.inject.Named; - -@Configuration -public class DataPrepperConfigurationConfiguration { - - @Bean - public DataPrepperConfiguration dataPrepperDefaultConfiguration(ApplicationArguments args) { - return new DataPrepperConfiguration(); - } -} diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java index 9acaa1945e..09fb522043 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java @@ -8,8 +8,12 @@ import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.parser.model.MetricRegistryType; import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; +import com.amazon.dataprepper.pipeline.server.PrometheusMetricsHandler; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.sun.net.httpserver.Authenticator; +import com.sun.net.httpserver.HttpContext; +import com.sun.net.httpserver.HttpServer; import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Metrics; @@ -23,18 +27,27 @@ import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import io.micrometer.prometheus.PrometheusConfig; import io.micrometer.prometheus.PrometheusMeterRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static com.amazon.dataprepper.DataPrepper.getServiceNameForMetrics; import static com.amazon.dataprepper.metrics.MetricNames.SERVICE_NAME; +import static java.lang.String.format; @Configuration public class MetricsConfig { + private static final Logger LOG = LoggerFactory.getLogger(MetricsConfig.class); + private static final String METRICS_CONTEXT_PREFIX = "/metrics"; + @Bean public YAMLFactory yamlFactory() { return new YAMLFactory(); @@ -70,18 +83,34 @@ public JvmThreadMetrics jvmThreadMetrics() { return new JvmThreadMetrics(); } - private void configureMetricRegistry(MeterRegistry meterRegistry) { + private void configureMetricRegistry(final MeterRegistry meterRegistry) { meterRegistry.config() - .commonTags(Arrays.asList( + .commonTags(Collections.singletonList( Tag.of(SERVICE_NAME, getServiceNameForMetrics()) )); + } @Bean - public Optional prometheusMeterRegistry(final DataPrepperConfiguration dataPrepperConfiguration) { + public Optional prometheusMeterRegistry( + final DataPrepperConfiguration dataPrepperConfiguration, + final Optional optionalAuthenticator, + final HttpServer server + ) { if (dataPrepperConfiguration.getMetricRegistryTypes().contains(MetricRegistryType.Prometheus)) { - PrometheusMeterRegistry meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + final PrometheusMeterRegistry meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); configureMetricRegistry(meterRegistry); + + final PrometheusMetricsHandler metricsHandler = new PrometheusMetricsHandler(meterRegistry); + + final List contextList = new ArrayList<>(2); + contextList.add(server.createContext(METRICS_CONTEXT_PREFIX + "/prometheus", metricsHandler)); + contextList.add(server.createContext(METRICS_CONTEXT_PREFIX + "/sys", metricsHandler)); + + optionalAuthenticator.ifPresent( + authenticator -> contextList.forEach( + context -> context.setAuthenticator(authenticator))); + return Optional.of(meterRegistry); } else { @@ -95,12 +124,12 @@ public CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider() { } @Bean - public Optional cloudWatchMeterRegistry( + public Optional cloudWatchMeterRegistry( final DataPrepperConfiguration dataPrepperConfiguration, - CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider + final CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider ) { if (dataPrepperConfiguration.getMetricRegistryTypes().contains(MetricRegistryType.CloudWatch)) { - CloudWatchMeterRegistry meterRegistry = cloudWatchMeterRegistryProvider.getCloudWatchMeterRegistry(); + final CloudWatchMeterRegistry meterRegistry = cloudWatchMeterRegistryProvider.getCloudWatchMeterRegistry(); configureMetricRegistry(meterRegistry); return Optional.of(meterRegistry); } @@ -112,11 +141,11 @@ public Optional cloudWatchMeterRegistry( @Bean public CompositeMeterRegistry systemMeterRegistry( final List meterBinders, - final List> optionalMeterRegistries, - final DataPrepperConfiguration dataPrepperConfiguration + final List> optionalMeterRegistries ) { final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); + LOG.debug("{} Meter Binder beans registered.", meterBinders.size()); meterBinders.forEach(binder -> binder.bindTo(compositeMeterRegistry)); optionalMeterRegistries.stream() diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java index 53c7e72b4c..40aba95e2f 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java @@ -5,20 +5,12 @@ package com.amazon.dataprepper.pipeline.server; -import com.sun.net.httpserver.Authenticator; -import com.sun.net.httpserver.HttpContext; import com.sun.net.httpserver.HttpServer; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.prometheus.PrometheusMeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.inject.Inject; import javax.inject.Named; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.Set; @@ -33,31 +25,9 @@ public class DataPrepperServer { @Inject public DataPrepperServer( - final Optional prometheusMeterRegistry, - final Optional optionalAuthenticator, - final ListPipelinesHandler listPipelinesHandler, - final ShutdownHandler shutdownHandler, final HttpServer server ) { this.server = server; - - List contextList = new ArrayList<>(4); - prometheusMeterRegistry.ifPresent(metricRegistry -> { - contextList.add(server.createContext("/metrics/prometheus", new PrometheusMetricsHandler(metricRegistry))); - contextList.add(server.createContext("/metrics/sys", new PrometheusMetricsHandler(metricRegistry))); - }); - - contextList.add(server.createContext("/list", listPipelinesHandler)); - contextList.add(server.createContext("/shutdown", shutdownHandler)); - - optionalAuthenticator.ifPresent( - authenticator -> contextList.forEach( - context -> context.setAuthenticator(authenticator))); - } - - private Optional getPrometheusMeterRegistryFromRegistries(final Set meterRegistries) { - return meterRegistries.stream().filter(meterRegistry -> - meterRegistry instanceof PrometheusMeterRegistry).findFirst(); } /** @@ -66,7 +36,6 @@ private Optional getPrometheusMeterRegistryFromRegistries(final S public void start() { server.start(); LOG.info("Data Prepper server running at :{}", server.getAddress().getPort()); - } /** diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java index 19929aa148..ba8a360331 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java @@ -10,6 +10,7 @@ import com.amazon.dataprepper.pipeline.server.ShutdownHandler; import com.amazon.dataprepper.pipeline.server.SslUtil; import com.sun.net.httpserver.Authenticator; +import com.sun.net.httpserver.HttpContext; import com.sun.net.httpserver.HttpServer; import com.sun.net.httpserver.HttpsConfigurator; import com.sun.net.httpserver.HttpsParameters; @@ -100,12 +101,30 @@ public Optional optionalAuthenticator(final DataPrepperCoreAuthen } @Bean - public ListPipelinesHandler listPipelinesHandler(final DataPrepper dataPrepper) { - return new ListPipelinesHandler(dataPrepper); + public ListPipelinesHandler listPipelinesHandler( + final DataPrepper dataPrepper, + final Optional optionalAuthenticator, + final HttpServer server + ) { + final ListPipelinesHandler listPipelinesHandler = new ListPipelinesHandler(dataPrepper); + + final HttpContext context = server.createContext("/list", listPipelinesHandler); + optionalAuthenticator.ifPresent(context::setAuthenticator); + + return listPipelinesHandler; } @Bean - public ShutdownHandler shutdownHandler(final DataPrepper dataPrepper) { - return new ShutdownHandler(dataPrepper); + public ShutdownHandler shutdownHandler( + final DataPrepper dataPrepper, + final Optional optionalAuthenticator, + final HttpServer server + ) { + final ShutdownHandler shutdownHandler = new ShutdownHandler(dataPrepper); + + final HttpContext context = server.createContext("/shutdown", shutdownHandler); + optionalAuthenticator.ifPresent(context::setAuthenticator); + + return shutdownHandler; } } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java index 4ec8b2c3c2..7fde811b98 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java @@ -7,8 +7,14 @@ import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.parser.model.MetricRegistryType; +import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; +import com.amazon.dataprepper.pipeline.server.PrometheusMetricsHandler; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.sun.net.httpserver.Authenticator; +import com.sun.net.httpserver.HttpContext; +import com.sun.net.httpserver.HttpServer; +import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.MeterBinder; import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; @@ -17,17 +23,23 @@ import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; import io.micrometer.core.instrument.binder.system.ProcessorMetrics; import io.micrometer.core.instrument.composite.CompositeMeterRegistry; +import io.micrometer.prometheus.PrometheusMeterRegistry; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.Random; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.isA; +import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -40,68 +52,185 @@ class MetricsConfigTest { @Test public void testYamlFactory() { - YAMLFactory factory = metricsConfig.yamlFactory(); + final YAMLFactory factory = metricsConfig.yamlFactory(); assertThat(factory, isA(YAMLFactory.class)); } @Test public void testObjectMapper() { - ObjectMapper mapper = metricsConfig.objectMapper(null); + final ObjectMapper mapper = metricsConfig.objectMapper(null); assertThat(mapper, isA(ObjectMapper.class)); } @Test public void testClassLoaderMetrics() { - ClassLoaderMetrics actual = metricsConfig.classLoaderMetrics(); + final ClassLoaderMetrics actual = metricsConfig.classLoaderMetrics(); assertThat(actual, isA(ClassLoaderMetrics.class)); } @Test public void testJvmMemoryMetrics() { - JvmMemoryMetrics actual = metricsConfig.jvmMemoryMetrics(); + final JvmMemoryMetrics actual = metricsConfig.jvmMemoryMetrics(); assertThat(actual, isA(JvmMemoryMetrics.class)); } @Test public void testJvmGcMetrics() { - JvmGcMetrics actual = metricsConfig.jvmGcMetrics(); + final JvmGcMetrics actual = metricsConfig.jvmGcMetrics(); assertThat(actual, isA(JvmGcMetrics.class)); } @Test public void testProcessorMetrics() { - ProcessorMetrics actual = metricsConfig.processorMetrics(); + final ProcessorMetrics actual = metricsConfig.processorMetrics(); assertThat(actual, isA(ProcessorMetrics.class)); } @Test public void testJvmThreadMetrics() { - JvmThreadMetrics actual = metricsConfig.jvmThreadMetrics(); + final JvmThreadMetrics actual = metricsConfig.jvmThreadMetrics(); assertThat(actual, isA(JvmThreadMetrics.class)); } @Test - public void testGivenListOfMeterBindersWhenSystemMeterRegistryThenAllMeterBindersRegistered() { - Random r = new Random(); - int copies = r.nextInt(10) + 5; - MeterBinder binder = mock(MeterBinder.class); - List meterBinders = Collections.nCopies(copies, binder); - DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); + public void testGivenConfigWithPrometheusMeterRegistryThenMeterRegistryCreated() { + final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); + final Optional optionalAuthenticator = Optional.empty(); + final HttpServer server = mock(HttpServer.class); + final HttpContext context = mock(HttpContext.class); + + when(dataPrepperConfiguration.getMetricRegistryTypes()) + .thenReturn(Collections.singletonList(MetricRegistryType.Prometheus)); + when(server.createContext(anyString(), any(PrometheusMetricsHandler.class))) + .thenReturn(context); + + final Optional optionalMeterRegistry = metricsConfig.prometheusMeterRegistry( + dataPrepperConfiguration, + optionalAuthenticator, + server); + + assertThat(optionalMeterRegistry.isPresent(), is(true)); + + final MeterRegistry registry = optionalMeterRegistry.get(); + assertThat(registry, isA(PrometheusMeterRegistry.class)); + + verify(server, times(2)) + .createContext(anyString(), any(PrometheusMetricsHandler.class)); + } + + @Test + public void testGivenConfigWithPrometheusMeterRegistryAndAuthenticatorThenMeterRegistryCreated() { + final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); + final Authenticator authenticator = mock(Authenticator.class); + final Optional optionalAuthenticator = Optional.of(authenticator); + final HttpServer server = mock(HttpServer.class); + final HttpContext context = mock(HttpContext.class); + + when(dataPrepperConfiguration.getMetricRegistryTypes()) + .thenReturn(Collections.singletonList(MetricRegistryType.Prometheus)); + when(server.createContext(anyString(), any(PrometheusMetricsHandler.class))) + .thenReturn(context); + + final Optional optionalMeterRegistry = metricsConfig.prometheusMeterRegistry( + dataPrepperConfiguration, + optionalAuthenticator, + server); + + assertThat(optionalMeterRegistry.isPresent(), is(true)); + + final MeterRegistry registry = optionalMeterRegistry.get(); + assertThat(registry, isA(PrometheusMeterRegistry.class)); + + verify(server, times(2)) + .createContext(anyString(), any(PrometheusMetricsHandler.class)); + + verify(context, times(2)) + .setAuthenticator(eq(authenticator)); + } + + @Test + public void testGivenEmptyConfigThenMeterRegistryCreated() { + final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); + + when(dataPrepperConfiguration.getMetricRegistryTypes()) + .thenReturn(Collections.emptyList()); + + final Optional optionalMeterRegistry = metricsConfig.prometheusMeterRegistry( +dataPrepperConfiguration, + Optional.empty(), + null); + + assertThat(optionalMeterRegistry.isPresent(), is(false)); + } + + @Test + public void testCloudWatchMeterRegistryProviderCreated() { + CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider = metricsConfig.cloudWatchMeterRegistryProvider(); + + assertThat(cloudWatchMeterRegistryProvider, isA(CloudWatchMeterRegistryProvider.class)); + } + + @Test + public void testGivenConfigWithCloudWatchMeterRegistryThenMeterRegistryCreated() { + final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); + final CloudWatchMeterRegistryProvider registryProvider = mock(CloudWatchMeterRegistryProvider.class); + final CloudWatchMeterRegistry meterRegistry = mock(CloudWatchMeterRegistry.class); + final MeterRegistry.Config config = mock(MeterRegistry.Config.class); + + when(dataPrepperConfiguration.getMetricRegistryTypes()) + .thenReturn(Collections.singletonList(MetricRegistryType.CloudWatch)); + when(registryProvider.getCloudWatchMeterRegistry()) + .thenReturn(meterRegistry); + when(meterRegistry.config()) + .thenReturn(config); + + final Optional optionalMeterRegistry = metricsConfig.cloudWatchMeterRegistry( + dataPrepperConfiguration, + registryProvider); + + assertThat(optionalMeterRegistry.isPresent(), is(true)); + + final MeterRegistry registry = optionalMeterRegistry.get(); + assertThat(registry, isA(CloudWatchMeterRegistry.class)); - MetricRegistryType registryType = mock(MetricRegistryType.class); - List registryTypes = Collections.nCopies(copies, registryType); + verify(config, times(1)) + .commonTags(anyList()); + } + + @Test + public void testGivenConfigWithNoCloudWatchMeterRegistryThenNoMeterRegistryCreated() { + final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); when(dataPrepperConfiguration.getMetricRegistryTypes()) - .thenReturn(registryTypes); + .thenReturn(Collections.singletonList(MetricRegistryType.Prometheus)); + + final Optional optionalMeterRegistry = metricsConfig.cloudWatchMeterRegistry( + dataPrepperConfiguration, + null); + + assertThat(optionalMeterRegistry.isPresent(), is(false)); + } - CompositeMeterRegistry meterRegistry = metricsConfig.systemMeterRegistry(meterBinders, dataPrepperConfiguration); + @Test + public void testGivenListOfMeterBindersWhenSystemMeterRegistryThenAllMeterBindersRegistered() { + final Random r = new Random(); + final int copies = r.nextInt(10) + 5; + final MeterBinder binder = mock(MeterBinder.class); + final List meterBinders = Collections.nCopies(copies, binder); + + final MeterRegistry meterRegistryMock = mock(MeterRegistry.class); + final List> optionalMeterRegistries = Collections.nCopies(1, Optional.of(meterRegistryMock)); + + final CompositeMeterRegistry meterRegistry = metricsConfig.systemMeterRegistry( + meterBinders, + optionalMeterRegistries); assertThat(meterRegistry, isA(CompositeMeterRegistry.class)); verify(binder, times(copies)).bindTo(any(MeterRegistry.class)); @@ -109,15 +238,14 @@ public void testGivenListOfMeterBindersWhenSystemMeterRegistryThenAllMeterBinder @Test public void testGivenEmptyListOfMeterBindersWhenSystemMeterRegistryThenNoMeterBindersRegistered() { - List meterBinders = Collections.emptyList(); - DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); - - List registryTypes = Collections.emptyList(); + final List meterBinders = Collections.emptyList(); - when(dataPrepperConfiguration.getMetricRegistryTypes()) - .thenReturn(registryTypes); + final MeterRegistry meterRegistryMock = mock(MeterRegistry.class); + final List> optionalMeterRegistries = Collections.nCopies(1, Optional.of(meterRegistryMock)); - CompositeMeterRegistry meterRegistry = metricsConfig.systemMeterRegistry(meterBinders, dataPrepperConfiguration); + final CompositeMeterRegistry meterRegistry = metricsConfig.systemMeterRegistry( + meterBinders, + optionalMeterRegistries); assertThat(meterRegistry, isA(CompositeMeterRegistry.class)); } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java index 7285f6b4c3..1be62c7247 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java @@ -6,26 +6,20 @@ package com.amazon.dataprepper.server; import com.amazon.dataprepper.pipeline.server.DataPrepperServer; -import com.amazon.dataprepper.pipeline.server.ListPipelinesHandler; -import com.amazon.dataprepper.pipeline.server.ShutdownHandler; -import com.sun.net.httpserver.Authenticator; -import com.sun.net.httpserver.HttpContext; -import com.sun.net.httpserver.HttpHandler; import com.sun.net.httpserver.HttpServer; -import io.micrometer.prometheus.PrometheusMeterRegistry; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.Optional; +import java.net.InetSocketAddress; +import java.util.Random; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -34,342 +28,35 @@ @ExtendWith(MockitoExtension.class) public class DataPrepperServerTest { - - private static final PrometheusMeterRegistry prometheusMeterRegistry = mock(PrometheusMeterRegistry.class); - private static final Authenticator authenticator = mock(Authenticator.class); - - @Mock - ListPipelinesHandler listPipelinesHandler; @Mock - ShutdownHandler shutdownHandler; - @Mock - HttpServer server; + private HttpServer server; - DataPrepperServer dataPrepperServer; + @InjectMocks + private DataPrepperServer dataPrepperServer; - @BeforeAll - public static void beforeAll() { + @Test + public void testDataPrepperServerConstructor() { + assertThat(dataPrepperServer, is(notNullValue())); } @Test - public void testDataPrepperServerConstructor() { - when(server.createContext(anyString(), any(HttpHandler.class))) - .thenReturn(mock(HttpContext.class)); + public void testGivenValidServerWhenStartThenShouldCallServerStart() { + final InetSocketAddress socketAddress = mock(InetSocketAddress.class); - dataPrepperServer = new DataPrepperServer( - Optional.of(prometheusMeterRegistry), - Optional.of(authenticator), - listPipelinesHandler, - shutdownHandler, - server); + when(server.getAddress()) + .thenReturn(socketAddress); - assertThat(dataPrepperServer, is(notNullValue())); - verify(server, times(4)).createContext(anyString(), any(HttpHandler.class)); + dataPrepperServer.start(); + + verify(server, times(1)).start(); + verify(server, times(1)).getAddress(); + verify(socketAddress, times(1)).getPort(); } -// private void setRegistry(PrometheusMeterRegistry prometheusMeterRegistry) { -// final Set meterRegistries = Metrics.globalRegistry.getRegistries(); -// //to avoid ConcurrentModificationException -// final Object[] registeredMeterRegistries = meterRegistries.toArray(); -// for (final Object meterRegistry : registeredMeterRegistries) { -// Metrics.removeRegistry((MeterRegistry) meterRegistry); -// } -// Metrics.addRegistry(prometheusMeterRegistry); -// } -// -// @BeforeEach -// public void setup() { -// // Required to trust any self-signed certs, used in https tests -// Properties props = System.getProperties(); -// props.setProperty("jdk.internal.httpclient.disableHostnameVerification", Boolean.TRUE.toString()); -// -// setRegistry(new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, "")); -// pluginFactory = mock(PluginFactory.class); -// -// when(pluginFactory.loadPlugin(eq(DataPrepperCoreAuthenticationProvider.class), -// argThat(a -> a.getName().equals(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)))) -// .thenReturn(unauthenticatedProvider); -// -//// when(dataPrepper.getPluginFactory()).thenReturn(pluginFactory); -//// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); -// } -// -// @AfterEach -// public void stopServer() { -// if (dataPrepperServer != null) { -// dataPrepperServer.stop(); -// } -// } -// -// @Test -// public void testGetGlobalMetrics() throws IOException, InterruptedException, URISyntaxException { -// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); -// final String scrape = UUID.randomUUID().toString(); -// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); -// setRegistry(prometheusMeterRegistry); -// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); -// -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); -// Assert.assertEquals(scrape, response.body()); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testScrapeGlobalFailure() throws IOException, InterruptedException, URISyntaxException { -// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); -// setRegistry(new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT)); -// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/prometheus")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testGetSystemMetrics() throws IOException, InterruptedException, URISyntaxException { -// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); -// final String scrape = UUID.randomUUID().toString(); -// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); -// -// when(systemMeterRegistry.getRegistries()) -// .thenReturn(Set.of(prometheusMeterRegistry)); -// -// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testScrapeSystemMetricsFailure() throws IOException, InterruptedException, URISyntaxException { -// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); -// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT); -// -// when(systemMeterRegistry.getRegistries()) -// .thenReturn(Set.of(prometheusMeterRegistry)); -// -// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testNonPrometheusMeterRegistry() throws Exception { -// final CloudWatchMeterRegistry cloudWatchMeterRegistry = mock(CloudWatchMeterRegistry.class); -// final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); -// compositeMeterRegistry.add(cloudWatchMeterRegistry); -// final DataPrepperConfiguration config = makeConfig(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE); -// try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { -// dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); -// dataPrepperServer.start(); -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) -// .GET().build(); -// HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// } catch (ConnectException ex) { -// dataPrepperServer.stop(); -// //there should not be any Prometheus endpoints available -// assertThat(ex.getMessage(), Matchers.is("Connection refused")); -// } -// } -// -// @Test -// public void testMultipleMeterRegistries() throws Exception { -// //for system metrics -// final CloudWatchMeterRegistry cloudWatchMeterRegistryForSystem = mock(CloudWatchMeterRegistry.class); -// final PrometheusMeterRegistry prometheusMeterRegistryForSystem = -// new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); -// -// when(systemMeterRegistry.getRegistries()) -// .thenReturn(Set.of(cloudWatchMeterRegistryForSystem, prometheusMeterRegistryForSystem)); -// -// //for data prepper metrics -// final PrometheusMeterRegistry prometheusMeterRegistryForDataPrepper = -// new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); -// setRegistry(prometheusMeterRegistryForDataPrepper); -// -// final DataPrepperConfiguration config = makeConfig(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE); -// -// dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); -// dataPrepperServer.start(); -// -// //test prometheus registry -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + config.getServerPort() -// + "/metrics/sys")).GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testListPipelines() throws URISyntaxException, IOException, InterruptedException { -// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); -// final String pipelineName = "testPipeline"; -// Mockito.when(dataPrepper.getTransformationPipelines()).thenReturn( -// Collections.singletonMap("testPipeline", null) -// ); -// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// final String expectedResponse = JSON_OBJECT_MAPPER.writeValueAsString( -// Collections.singletonMap("pipelines", Arrays.asList( -// Collections.singletonMap("name", pipelineName) -// )) -// ); -// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); -// Assert.assertEquals(expectedResponse, response.body()); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testListPipelinesFailure() throws URISyntaxException, IOException, InterruptedException { -// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); -// Mockito.when(dataPrepper.getTransformationPipelines()).thenThrow(new RuntimeException("")); -// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/list")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testShutdown() throws URISyntaxException, IOException, InterruptedException { -// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); -// -// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); -// Mockito.verify(dataPrepper).shutdown(); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testShutdownFailure() throws URISyntaxException, IOException, InterruptedException { -// when(dataPrepperConfiguration.getServerPort()).thenReturn(port); -// Mockito.doThrow(new RuntimeException("")).when(dataPrepper).shutdown(); -// dataPrepperServer = new DataPrepperServer(dataPrepperConfiguration, pluginFactory, dataPrepper, systemMeterRegistry); -// dataPrepperServer.start(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/shutdown")) -// .GET().build(); -// HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); -// Mockito.verify(dataPrepper).shutdown(); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testGetMetricsWithHttps() throws Exception { -// DataPrepperConfiguration config = DataPrepperServerTest.makeConfig( -// TestDataProvider.VALID_DATA_PREPPER_CONFIG_FILE_WITH_TLS); -// final String scrape = UUID.randomUUID().toString(); -// final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); -// setRegistry(prometheusMeterRegistry); -// dataPrepperServer = new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry); -// dataPrepperServer.start(); -// -// HttpClient httpsClient = HttpClient.newBuilder() -// .connectTimeout(Duration.ofSeconds(3)) -// .sslContext(getSslContextTrustingInsecure()) -// .build(); -// -// HttpRequest request = HttpRequest.newBuilder(new URI("https://localhost:" + port + "/metrics/prometheus")) -// .GET() -// .timeout(Duration.ofSeconds(3)) -// .build(); -// -// HttpResponse response = httpsClient.send(request, HttpResponse.BodyHandlers.ofString()); -// Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); -// Assert.assertEquals(scrape, response.body()); -// dataPrepperServer.stop(); -// } -// -// @Test -// public void testTlsWithInvalidPassword() throws IOException { -// DataPrepperConfiguration config = DataPrepperServerTest.makeConfig( -// TestDataProvider.INVALID_KEYSTORE_PASSWORD_DATA_PREPPER_CONFIG_FILE); -// -// assertThrows(IllegalStateException.class, () -> new DataPrepperServer(config, pluginFactory, dataPrepper, systemMeterRegistry)); -// } -// -// private SSLContext getSslContextTrustingInsecure() throws Exception { -// TrustManager[] trustAllCerts = new TrustManager[]{ -// new X509TrustManager() { -// public java.security.cert.X509Certificate[] getAcceptedIssuers() { -// return null; -// } -// -// public void checkClientTrusted( -// java.security.cert.X509Certificate[] certs, String authType) { -// } -// -// public void checkServerTrusted( -// java.security.cert.X509Certificate[] certs, String authType) { -// } -// } -// }; -// -// SSLContext sslContext = SSLContext.getInstance("SSL"); -// sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); -// -// return sslContext; -// } -// -// private static class PrometheusRegistryMockScrape extends PrometheusMeterRegistry { -// final String scrape; -// -// public PrometheusRegistryMockScrape(PrometheusConfig config, final String scrape) { -// super(config); -// this.scrape = scrape; -// } -// -// @Override -// public String scrape() { -// return scrape; -// } -// } -// -// private static class PrometheusRegistryThrowingScrape extends PrometheusMeterRegistry { -// -// public PrometheusRegistryThrowingScrape(PrometheusConfig config) { -// super(config); -// } -// -// @Override -// public String scrape() { -// throw new RuntimeException(""); -// } -// } -// -// private static DataPrepperConfiguration makeConfig(String filePath) throws IOException { -// final File configurationFile = new File(filePath); -// return OBJECT_MAPPER.readValue(configurationFile, DataPrepperConfiguration.class); -// } + @Test + public void testGivenValidServerWhenStopThenShouldCallServerStopWithNoDely() { + dataPrepperServer.stop(); + verify(server, times(1)).stop(eq(0)); + } } From eebc8c7a5338d8794bac04ecf70077d9165a191f Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Thu, 6 Jan 2022 10:53:19 -0600 Subject: [PATCH 29/43] Added Server config unit test Signed-off-by: Steven Bayer --- .../config/DataPrepperAppConfiguration.java | 15 -- .../DataPrepperServerConfiguration.java | 17 ++- .../DataPrepperAppConfigurationTest.java | 12 ++ .../DataPrepperServerConfigurationTest.java | 138 ++++++++++++++++++ 4 files changed, 159 insertions(+), 23 deletions(-) create mode 100644 data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java index 40ccbade81..5763fd168d 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java @@ -65,19 +65,4 @@ public DataPrepperConfiguration dataPrepperConfiguration( public Optional pluginModel(final DataPrepperConfiguration dataPrepperConfiguration) { return Optional.ofNullable(dataPrepperConfiguration.getAuthentication()); } - - private static String checkForLogstashConfigurationAndConvert(String configurationFileLocation) { - if (configurationFileLocation.endsWith(".conf")) { - final LogstashConfigConverter logstashConfigConverter = new LogstashConfigConverter(); - final Path configurationDirectory = Paths.get(configurationFileLocation).toAbsolutePath().getParent(); - - try { - configurationFileLocation = logstashConfigConverter.convertLogstashConfigurationToPipeline( - configurationFileLocation, String.valueOf(configurationDirectory)); - } catch (IOException e) { - LOG.error("Unable to read the Logstash configuration file", e); - } - } - return configurationFileLocation; - } } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java index ba8a360331..ae3f5d6fb7 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java @@ -33,7 +33,7 @@ public class DataPrepperServerConfiguration { @Bean public HttpServer httpServer(final DataPrepperConfiguration dataPrepperConfiguration) { - InetSocketAddress socketAddress = new InetSocketAddress(dataPrepperConfiguration.getServerPort()); + final InetSocketAddress socketAddress = new InetSocketAddress(dataPrepperConfiguration.getServerPort()); try { if (dataPrepperConfiguration.ssl()) { LOG.info("Creating Data Prepper server with TLS"); @@ -45,10 +45,10 @@ public HttpServer httpServer(final DataPrepperConfiguration dataPrepperConfigura final HttpsServer server = HttpsServer.create(socketAddress, 0); server.setHttpsConfigurator(new HttpsConfigurator(sslContext) { - public void configure(HttpsParameters params) { - SSLContext context = getSSLContext(); - SSLParameters sslparams = context.getDefaultSSLParameters(); - params.setSSLParameters(sslparams); + public void configure(final HttpsParameters params) { + final SSLContext context = getSSLContext(); + final SSLParameters sslParams = context.getDefaultSSLParameters(); + params.setSSLParameters(sslParams); } }); @@ -63,14 +63,15 @@ public void configure(HttpsParameters params) { private void printInsecurePluginModelWarning() { LOG.warn("Creating data prepper server without authentication. This is not secure."); - LOG.warn("In order to set up Http Basic authentication for the data prepper server, go here: https://github.com/opensearch-project/data-prepper/blob/main/docs/core_apis.md#authentication"); + LOG.warn("In order to set up Http Basic authentication for the data prepper server, " + + "go here: https://github.com/opensearch-project/data-prepper/blob/main/docs/core_apis.md#authentication"); } @Bean public PluginSetting pluginSetting(final Optional optionalPluginModel) { if (optionalPluginModel.isPresent()) { - PluginModel pluginModel = optionalPluginModel.get(); - String pluginName = pluginModel.getPluginName(); + final PluginModel pluginModel = optionalPluginModel.get(); + final String pluginName = pluginModel.getPluginName(); if (pluginName.equals(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)) { printInsecurePluginModelWarning(); } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfigurationTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfigurationTest.java index 431b12f599..947361ca83 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfigurationTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfigurationTest.java @@ -6,6 +6,7 @@ package com.amazon.dataprepper.parser.config; import com.amazon.dataprepper.TestDataProvider; +import com.amazon.dataprepper.model.configuration.PluginModel; import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; @@ -15,6 +16,7 @@ import java.io.File; import java.io.IOException; +import java.util.Optional; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; @@ -115,4 +117,14 @@ public void testGivenInvalidPipelineConfigArgThenExceptionThrown() throws IOExce assertThrows(IllegalArgumentException.class, () -> appConfiguration.dataPrepperConfiguration(dataPrepperArgs, objectMapper)); } + + @Test + public void testPluginModelFromDataPrepperConfigurationAuthentication() { + final DataPrepperConfiguration configuration = mock(DataPrepperConfiguration.class); + + Optional optional = appConfiguration.pluginModel(configuration); + + assertThat(optional.isPresent(), is(false)); + verify(configuration, times(1)).getAuthentication(); + } } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java new file mode 100644 index 0000000000..bd1fe7e7f4 --- /dev/null +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java @@ -0,0 +1,138 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.amazon.dataprepper.pipeline.server.config; + +import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; +import com.sun.net.httpserver.HttpServer; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsServer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.net.DatagramSocket; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.util.Random; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isA; +import static org.hamcrest.Matchers.not; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DataPrepperServerConfigurationTest { + private static final Random r = new Random(); + private static final Integer MAX_PORT = 65535; + private static final String P12_KEYSTORE = "src/test/resources/tls/test_keystore.p12"; + + private static boolean available(final int port) { + ServerSocket ss = null; + DatagramSocket ds = null; + try { + ss = new ServerSocket(port); + ss.setReuseAddress(true); + ds = new DatagramSocket(port); + ds.setReuseAddress(true); + return true; + } catch (final IOException ignored) { + } finally { + if (ds != null) { + ds.close(); + } + + if (ss != null) { + try { + ss.close(); + } catch (final IOException ignored) { + } + } + } + + return false; + } + + private static int getPort() { + final int MAX_RETRIES = 10; + for (int i = 0; i < MAX_RETRIES; i++) { + final int electivePort = r.nextInt(MAX_PORT); + if (available(electivePort)) { + return electivePort; + } + } + throw new RuntimeException("No available port found"); + } + + @Mock + private DataPrepperConfiguration configuration; + + private final DataPrepperServerConfiguration serverConfiguration = new DataPrepperServerConfiguration(); + + @Test + public void testGivenNoSslThenInsecureServerCreated() { + final int expectedPort = getPort(); + when(configuration.ssl()) + .thenReturn(false); + when(configuration.getServerPort()) + .thenReturn(expectedPort); + + HttpServer server = serverConfiguration.httpServer(configuration); + + assertThat(server, isA(HttpServer.class)); + assertThat(server, not(isA(HttpsServer.class))); + assertThat(server.getAddress().getPort(), is(expectedPort)); + } + + @Test + public void testGivenSslConfigThenHttpsServerCreater() { + final int expectedPort = getPort(); + + when(configuration.ssl()) + .thenReturn(true); + when(configuration.getKeyStoreFilePath()) + .thenReturn(P12_KEYSTORE); + when(configuration.getKeyStorePassword()) + .thenReturn(""); + when(configuration.getPrivateKeyPassword()) + .thenReturn(""); + when(configuration.getServerPort()) + .thenReturn(expectedPort); + + HttpServer server = serverConfiguration.httpServer(configuration); + + assertThat(server, isA(HttpServer.class)); + assertThat(server, isA(HttpsServer.class)); + assertThat(server.getAddress().getPort(), is(expectedPort)); + + HttpsServer httpsServer = (HttpsServer) server; + + assertThat(httpsServer.getHttpsConfigurator(), isA(HttpsConfigurator.class)); + + verify(configuration, times(1)).getKeyStoreFilePath(); + verify(configuration, times(1)).getKeyStorePassword(); + verify(configuration, times(1)).getPrivateKeyPassword(); + } + + @Test + public void testGivenPortInUseThenExceptionThrown() throws IOException { + final int port = getPort(); + InetSocketAddress socketAddress = new InetSocketAddress(port); + + HttpServer portBlockingServer = HttpServer.create(socketAddress, 0); + portBlockingServer.start(); + + when(configuration.getServerPort()) + .thenReturn(port); + + assertThrows(RuntimeException.class, () -> serverConfiguration.httpServer(configuration)); + } +} \ No newline at end of file From 7d8f1f62fbefa087fe588970e2041bbcb9cdd081 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Thu, 6 Jan 2022 11:48:24 -0600 Subject: [PATCH 30/43] Added DataPrepperServerConfiguration unit tests Signed-off-by: Steven Bayer --- .../parser/config/DataPrepperArgsTest.java | 7 + .../DataPrepperServerConfigurationTest.java | 159 +++++++++++++++++- 2 files changed, 163 insertions(+), 3 deletions(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java index b914a17633..fb9be1aae1 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java @@ -49,6 +49,13 @@ public void testGivenThreeArgumentThenThrowException() { @Test public void testGivenNoArgumentThenThrowException() { + assertThrows( + IllegalArgumentException.class, + () -> new DataPrepperArgs(new String[]{})); + } + + @Test + public void testGivenNullArgumentThenThrowException() { assertThrows( IllegalArgumentException.class, () -> new DataPrepperArgs()); diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java index bd1fe7e7f4..1d5e2f04e2 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java @@ -5,7 +5,16 @@ package com.amazon.dataprepper.pipeline.server.config; +import com.amazon.dataprepper.DataPrepper; +import com.amazon.dataprepper.model.configuration.PluginModel; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.model.plugin.PluginFactory; import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; +import com.amazon.dataprepper.pipeline.server.DataPrepperCoreAuthenticationProvider; +import com.amazon.dataprepper.pipeline.server.ListPipelinesHandler; +import com.amazon.dataprepper.pipeline.server.ShutdownHandler; +import com.sun.net.httpserver.Authenticator; +import com.sun.net.httpserver.HttpContext; import com.sun.net.httpserver.HttpServer; import com.sun.net.httpserver.HttpsConfigurator; import com.sun.net.httpserver.HttpsServer; @@ -18,6 +27,9 @@ import java.net.DatagramSocket; import java.net.InetSocketAddress; import java.net.ServerSocket; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import java.util.Random; import static org.hamcrest.MatcherAssert.assertThat; @@ -25,8 +37,12 @@ import static org.hamcrest.Matchers.isA; import static org.hamcrest.Matchers.not; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -130,9 +146,146 @@ public void testGivenPortInUseThenExceptionThrown() throws IOException { HttpServer portBlockingServer = HttpServer.create(socketAddress, 0); portBlockingServer.start(); - when(configuration.getServerPort()) - .thenReturn(port); + try { + when(configuration.getServerPort()) + .thenReturn(port); + + assertThrows(RuntimeException.class, () -> serverConfiguration.httpServer(configuration)); + } catch (final Exception ignored) { + } finally { + portBlockingServer.stop(0); + } + } + + @Test + public void testGivingNoConfigThenCreateInsecureSettings() { + PluginSetting pluginSetting = serverConfiguration.pluginSetting(Optional.empty()); + + assertThat(pluginSetting.getName(), is("unauthenticated")); + assertThat(pluginSetting.getSettings().isEmpty(), is(true)); + } + + @Test + public void testGivingInsecureConfigThenCreateInsecureSettings() { + PluginModel pluginModel = mock(PluginModel.class); + + when(pluginModel.getPluginName()) + .thenReturn("unauthenticated"); + + PluginSetting pluginSetting = serverConfiguration.pluginSetting(Optional.of(pluginModel)); + + assertThat(pluginSetting.getName(), is("unauthenticated")); + assertThat(pluginSetting.getSettings().isEmpty(), is(true)); + } + + @Test + public void testGivingSecureConfigThenCreateInsecureSettings() { + final PluginModel pluginModel = mock(PluginModel.class); + final Map settings = new HashMap<>(); + + when(pluginModel.getPluginName()) + .thenReturn("super secure plugin"); + + when(pluginModel.getPluginSettings()) + .thenReturn(settings); + + PluginSetting pluginSetting = serverConfiguration.pluginSetting(Optional.of(pluginModel)); + + assertThat(pluginSetting.getName(), is("super secure plugin")); + assertThat(pluginSetting.getSettings(), is(settings)); + } + + @Test + public void testGivenPluginFactoryAndPluginSettingsThenCreateAuthenticationProvider() { + final PluginFactory pluginFactory = mock(PluginFactory.class); + final PluginSetting pluginSetting = mock(PluginSetting.class); + final DataPrepperCoreAuthenticationProvider expected = mock(DataPrepperCoreAuthenticationProvider.class); + + when(pluginFactory.loadPlugin(eq(DataPrepperCoreAuthenticationProvider.class), eq(pluginSetting))) + .thenReturn(expected); + + DataPrepperCoreAuthenticationProvider authenticationProvider = serverConfiguration.authenticationProvider( + pluginFactory, + pluginSetting); + + assertThat(authenticationProvider, is(expected)); + } + + @Test + public void testGivenGetAuthenticatorReturnsValueThenReturnOptionalContainingValue() { + final DataPrepperCoreAuthenticationProvider provider = mock(DataPrepperCoreAuthenticationProvider.class); + final Authenticator authenticator = mock(Authenticator.class); + + when(provider.getAuthenticator()) + .thenReturn(authenticator); + + Optional optional = serverConfiguration.optionalAuthenticator(provider); + + assertThat(optional.isPresent(), is(true)); + assertThat(optional.get(), is(authenticator)); + } + + @Test + public void testGivenValidInputWithNoAuthenticatorThenServerListContextCreated() { + final DataPrepper dataPrepper = mock(DataPrepper.class); + final HttpServer server = mock(HttpServer.class); + final HttpContext context = mock(HttpContext.class); + + when(server.createContext(eq("/list"), any(ListPipelinesHandler.class))) + .thenReturn(context); + + ListPipelinesHandler handler = serverConfiguration.listPipelinesHandler(dataPrepper, Optional.empty(), server); + + assertThat(handler, isA(ListPipelinesHandler.class)); + verifyNoInteractions(context); + } + + @Test + public void testGivenValidInputWithAuthenticatorThenServerListContextCreated() { + final DataPrepper dataPrepper = mock(DataPrepper.class); + final Authenticator authenticator = mock(Authenticator.class); + final HttpServer server = mock(HttpServer.class); + final HttpContext context = mock(HttpContext.class); + + when(server.createContext(eq("/list"), any(ListPipelinesHandler.class))) + .thenReturn(context); + + ListPipelinesHandler handler = serverConfiguration.listPipelinesHandler(dataPrepper, Optional.of(authenticator), server); + + assertThat(handler, isA(ListPipelinesHandler.class)); + verify(context, times(1)) + .setAuthenticator(eq(authenticator)); + } + + @Test + public void testGivenValidInputWithNoAuthenticatorThenServerShutdownContextCreated() { + final DataPrepper dataPrepper = mock(DataPrepper.class); + final HttpServer server = mock(HttpServer.class); + final HttpContext context = mock(HttpContext.class); + + when(server.createContext(eq("/shutdown"), any(ShutdownHandler.class))) + .thenReturn(context); + + ShutdownHandler handler = serverConfiguration.shutdownHandler(dataPrepper, Optional.empty(), server); + + assertThat(handler, isA(ShutdownHandler.class)); + verifyNoInteractions(context); + } + + @Test + public void testGivenValidInputWithAuthenticatorThenServerShutdownContextCreated() { + final DataPrepper dataPrepper = mock(DataPrepper.class); + final Authenticator authenticator = mock(Authenticator.class); + final HttpServer server = mock(HttpServer.class); + final HttpContext context = mock(HttpContext.class); + + when(server.createContext(eq("/shutdown"), any(ShutdownHandler.class))) + .thenReturn(context); + + ShutdownHandler handler = serverConfiguration.shutdownHandler(dataPrepper, Optional.of(authenticator), server); - assertThrows(RuntimeException.class, () -> serverConfiguration.httpServer(configuration)); + assertThat(handler, isA(ShutdownHandler.class)); + verify(context, times(1)) + .setAuthenticator(eq(authenticator)); } } \ No newline at end of file From a2ccce50a620ba9cada90728ddb7c98719e7b699 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Thu, 6 Jan 2022 11:54:32 -0600 Subject: [PATCH 31/43] Added final modifier where possible Signed-off-by: Steven Bayer --- .../config/DataPrepperAppConfiguration.java | 3 -- .../parser/config/DataPrepperArgs.java | 4 +-- .../DataPrepperServerConfiguration.java | 5 +++ .../DataPrepperAppConfigurationTest.java | 2 +- .../parser/config/DataPrepperArgsTest.java | 8 ++--- .../PipelineParserConfigurationTest.java | 2 +- .../DataPrepperServerConfigurationTest.java | 32 +++++++++---------- 7 files changed, 29 insertions(+), 27 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java index 5763fd168d..d4316fc1a1 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java @@ -8,7 +8,6 @@ import com.amazon.dataprepper.model.configuration.PluginModel; import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; -import org.opensearch.dataprepper.logstash.LogstashConfigConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Bean; @@ -18,8 +17,6 @@ import java.io.File; import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Optional; @Configuration diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java index cdf9520b6b..1a395aa392 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperArgs.java @@ -31,7 +31,7 @@ private static String checkForLogstashConfigurationAndConvert(String configurati try { configurationFileLocation = logstashConfigConverter.convertLogstashConfigurationToPipeline( configurationFileLocation, String.valueOf(configurationDirectory)); - } catch (IOException e) { + } catch (final IOException e) { LOG.warn("Unable to read the Logstash configuration file", e); throw new IllegalArgumentException("Invalid Logstash configuration file", e); } @@ -51,7 +51,7 @@ else if (args.length > MAXIMUM_SUPPORTED_NUMBER_OF_ARGS) { "Data Prepper supports a maximum of " + MAXIMUM_SUPPORTED_NUMBER_OF_ARGS + " command line arguments"); } - String configurationFileLocation = args[DATA_PREPPER_PIPELINE_CONFIG_POSITON]; + final String configurationFileLocation = args[DATA_PREPPER_PIPELINE_CONFIG_POSITON]; LOG.info("Using {} configuration file", configurationFileLocation); this.pipelineConfigFileLocation = DataPrepperArgs.checkForLogstashConfigurationAndConvert(configurationFileLocation); diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java index ae3f5d6fb7..e31418d4e0 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package com.amazon.dataprepper.pipeline.server.config; import com.amazon.dataprepper.DataPrepper; diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfigurationTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfigurationTest.java index 947361ca83..ac97a88756 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfigurationTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfigurationTest.java @@ -122,7 +122,7 @@ public void testGivenInvalidPipelineConfigArgThenExceptionThrown() throws IOExce public void testPluginModelFromDataPrepperConfigurationAuthentication() { final DataPrepperConfiguration configuration = mock(DataPrepperConfiguration.class); - Optional optional = appConfiguration.pluginModel(configuration); + final Optional optional = appConfiguration.pluginModel(configuration); assertThat(optional.isPresent(), is(false)); verify(configuration, times(1)).getAuthentication(); diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java index fb9be1aae1..31bfbf844b 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperArgsTest.java @@ -24,7 +24,7 @@ class DataPrepperArgsTest { @Test public void testGivenSingleArgumentThenAssignedToPipelineConfig() { - DataPrepperArgs args = new DataPrepperArgs(PIPELINE_FILE_PATH); + final DataPrepperArgs args = new DataPrepperArgs(PIPELINE_FILE_PATH); assertThat(args, is(notNullValue())); assertThat(args.getPipelineConfigFileLocation(), is(PIPELINE_FILE_PATH)); @@ -33,7 +33,7 @@ public void testGivenSingleArgumentThenAssignedToPipelineConfig() { @Test public void testGivenTwoArgumentThenAssignedCorrectly() { - DataPrepperArgs args = new DataPrepperArgs(PIPELINE_FILE_PATH, DP_CONFIG_YAML_FILE_PATH); + final DataPrepperArgs args = new DataPrepperArgs(PIPELINE_FILE_PATH, DP_CONFIG_YAML_FILE_PATH); assertThat(args, is(notNullValue())); assertThat(args.getPipelineConfigFileLocation(), is(PIPELINE_FILE_PATH)); @@ -58,12 +58,12 @@ public void testGivenNoArgumentThenThrowException() { public void testGivenNullArgumentThenThrowException() { assertThrows( IllegalArgumentException.class, - () -> new DataPrepperArgs()); + DataPrepperArgs::new); } @Test public void testGivenLogstashConfigPathThenPipelineConfigCreated() { - DataPrepperArgs args = new DataPrepperArgs(LOGSTASH_PIPELINE_FILE_PATH, DP_CONFIG_YAML_FILE_PATH); + final DataPrepperArgs args = new DataPrepperArgs(LOGSTASH_PIPELINE_FILE_PATH, DP_CONFIG_YAML_FILE_PATH); assertThat(args, is(notNullValue())); assertThat( diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/PipelineParserConfigurationTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/PipelineParserConfigurationTest.java index 2bb5c53e3e..c18af38636 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/PipelineParserConfigurationTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/PipelineParserConfigurationTest.java @@ -35,7 +35,7 @@ void pipelineParser() { when(args.getPipelineConfigFileLocation()) .thenReturn(pipelineConfigFileLocation); - PipelineParser pipelineParser = pipelineParserConfiguration.pipelineParser(args, pluginFactory); + final PipelineParser pipelineParser = pipelineParserConfiguration.pipelineParser(args, pluginFactory); assertThat(pipelineParser, is(notNullValue())); verify(args, times(1)).getPipelineConfigFileLocation(); diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java index 1d5e2f04e2..8dcb1b2ba4 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java @@ -101,7 +101,7 @@ public void testGivenNoSslThenInsecureServerCreated() { when(configuration.getServerPort()) .thenReturn(expectedPort); - HttpServer server = serverConfiguration.httpServer(configuration); + final HttpServer server = serverConfiguration.httpServer(configuration); assertThat(server, isA(HttpServer.class)); assertThat(server, not(isA(HttpsServer.class))); @@ -109,7 +109,7 @@ public void testGivenNoSslThenInsecureServerCreated() { } @Test - public void testGivenSslConfigThenHttpsServerCreater() { + public void testGivenSslConfigThenHttpsServerCreated() { final int expectedPort = getPort(); when(configuration.ssl()) @@ -123,13 +123,13 @@ public void testGivenSslConfigThenHttpsServerCreater() { when(configuration.getServerPort()) .thenReturn(expectedPort); - HttpServer server = serverConfiguration.httpServer(configuration); + final HttpServer server = serverConfiguration.httpServer(configuration); assertThat(server, isA(HttpServer.class)); assertThat(server, isA(HttpsServer.class)); assertThat(server.getAddress().getPort(), is(expectedPort)); - HttpsServer httpsServer = (HttpsServer) server; + final HttpsServer httpsServer = (HttpsServer) server; assertThat(httpsServer.getHttpsConfigurator(), isA(HttpsConfigurator.class)); @@ -141,9 +141,9 @@ public void testGivenSslConfigThenHttpsServerCreater() { @Test public void testGivenPortInUseThenExceptionThrown() throws IOException { final int port = getPort(); - InetSocketAddress socketAddress = new InetSocketAddress(port); + final InetSocketAddress socketAddress = new InetSocketAddress(port); - HttpServer portBlockingServer = HttpServer.create(socketAddress, 0); + final HttpServer portBlockingServer = HttpServer.create(socketAddress, 0); portBlockingServer.start(); try { @@ -159,7 +159,7 @@ public void testGivenPortInUseThenExceptionThrown() throws IOException { @Test public void testGivingNoConfigThenCreateInsecureSettings() { - PluginSetting pluginSetting = serverConfiguration.pluginSetting(Optional.empty()); + final PluginSetting pluginSetting = serverConfiguration.pluginSetting(Optional.empty()); assertThat(pluginSetting.getName(), is("unauthenticated")); assertThat(pluginSetting.getSettings().isEmpty(), is(true)); @@ -167,12 +167,12 @@ public void testGivingNoConfigThenCreateInsecureSettings() { @Test public void testGivingInsecureConfigThenCreateInsecureSettings() { - PluginModel pluginModel = mock(PluginModel.class); + final PluginModel pluginModel = mock(PluginModel.class); when(pluginModel.getPluginName()) .thenReturn("unauthenticated"); - PluginSetting pluginSetting = serverConfiguration.pluginSetting(Optional.of(pluginModel)); + final PluginSetting pluginSetting = serverConfiguration.pluginSetting(Optional.of(pluginModel)); assertThat(pluginSetting.getName(), is("unauthenticated")); assertThat(pluginSetting.getSettings().isEmpty(), is(true)); @@ -189,7 +189,7 @@ public void testGivingSecureConfigThenCreateInsecureSettings() { when(pluginModel.getPluginSettings()) .thenReturn(settings); - PluginSetting pluginSetting = serverConfiguration.pluginSetting(Optional.of(pluginModel)); + final PluginSetting pluginSetting = serverConfiguration.pluginSetting(Optional.of(pluginModel)); assertThat(pluginSetting.getName(), is("super secure plugin")); assertThat(pluginSetting.getSettings(), is(settings)); @@ -204,7 +204,7 @@ public void testGivenPluginFactoryAndPluginSettingsThenCreateAuthenticationProvi when(pluginFactory.loadPlugin(eq(DataPrepperCoreAuthenticationProvider.class), eq(pluginSetting))) .thenReturn(expected); - DataPrepperCoreAuthenticationProvider authenticationProvider = serverConfiguration.authenticationProvider( + final DataPrepperCoreAuthenticationProvider authenticationProvider = serverConfiguration.authenticationProvider( pluginFactory, pluginSetting); @@ -219,7 +219,7 @@ public void testGivenGetAuthenticatorReturnsValueThenReturnOptionalContainingVal when(provider.getAuthenticator()) .thenReturn(authenticator); - Optional optional = serverConfiguration.optionalAuthenticator(provider); + final Optional optional = serverConfiguration.optionalAuthenticator(provider); assertThat(optional.isPresent(), is(true)); assertThat(optional.get(), is(authenticator)); @@ -234,7 +234,7 @@ public void testGivenValidInputWithNoAuthenticatorThenServerListContextCreated() when(server.createContext(eq("/list"), any(ListPipelinesHandler.class))) .thenReturn(context); - ListPipelinesHandler handler = serverConfiguration.listPipelinesHandler(dataPrepper, Optional.empty(), server); + final ListPipelinesHandler handler = serverConfiguration.listPipelinesHandler(dataPrepper, Optional.empty(), server); assertThat(handler, isA(ListPipelinesHandler.class)); verifyNoInteractions(context); @@ -250,7 +250,7 @@ public void testGivenValidInputWithAuthenticatorThenServerListContextCreated() { when(server.createContext(eq("/list"), any(ListPipelinesHandler.class))) .thenReturn(context); - ListPipelinesHandler handler = serverConfiguration.listPipelinesHandler(dataPrepper, Optional.of(authenticator), server); + final ListPipelinesHandler handler = serverConfiguration.listPipelinesHandler(dataPrepper, Optional.of(authenticator), server); assertThat(handler, isA(ListPipelinesHandler.class)); verify(context, times(1)) @@ -266,7 +266,7 @@ public void testGivenValidInputWithNoAuthenticatorThenServerShutdownContextCreat when(server.createContext(eq("/shutdown"), any(ShutdownHandler.class))) .thenReturn(context); - ShutdownHandler handler = serverConfiguration.shutdownHandler(dataPrepper, Optional.empty(), server); + final ShutdownHandler handler = serverConfiguration.shutdownHandler(dataPrepper, Optional.empty(), server); assertThat(handler, isA(ShutdownHandler.class)); verifyNoInteractions(context); @@ -282,7 +282,7 @@ public void testGivenValidInputWithAuthenticatorThenServerShutdownContextCreated when(server.createContext(eq("/shutdown"), any(ShutdownHandler.class))) .thenReturn(context); - ShutdownHandler handler = serverConfiguration.shutdownHandler(dataPrepper, Optional.of(authenticator), server); + final ShutdownHandler handler = serverConfiguration.shutdownHandler(dataPrepper, Optional.of(authenticator), server); assertThat(handler, isA(ShutdownHandler.class)); verify(context, times(1)) From 2b1ed1548e290d571002a163757ef202ebfdc19f Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Thu, 6 Jan 2022 17:36:35 -0600 Subject: [PATCH 32/43] Only create Cloud Watch Provider if configured Signed-off-by: Steven Bayer --- .../parser/config/MetricsConfig.java | 27 +++++++++---------- .../PipelineThreadPoolExecutorTest.java | 12 ++++----- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java index 09fb522043..4bbb951a36 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java @@ -31,17 +31,15 @@ import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import software.amazon.awssdk.core.exception.SdkClientException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import static com.amazon.dataprepper.DataPrepper.getServiceNameForMetrics; import static com.amazon.dataprepper.metrics.MetricNames.SERVICE_NAME; -import static java.lang.String.format; @Configuration public class MetricsConfig { @@ -119,19 +117,18 @@ public Optional prometheusMeterRegistry( } @Bean - public CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider() { - return new CloudWatchMeterRegistryProvider(); - } - - @Bean - public Optional cloudWatchMeterRegistry( - final DataPrepperConfiguration dataPrepperConfiguration, - final CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider - ) { + public Optional cloudWatchMeterRegistry(final DataPrepperConfiguration dataPrepperConfiguration) { if (dataPrepperConfiguration.getMetricRegistryTypes().contains(MetricRegistryType.CloudWatch)) { - final CloudWatchMeterRegistry meterRegistry = cloudWatchMeterRegistryProvider.getCloudWatchMeterRegistry(); - configureMetricRegistry(meterRegistry); - return Optional.of(meterRegistry); + try { + final CloudWatchMeterRegistryProvider provider = new CloudWatchMeterRegistryProvider(); + final CloudWatchMeterRegistry meterRegistry = provider.getCloudWatchMeterRegistry(); + + configureMetricRegistry(meterRegistry); + return Optional.of(meterRegistry); + } catch (SdkClientException e) { + LOG.warn("Unable to configure Cloud Watch Meter Registry but Meter Registry was requested in Data Prepper Configuration"); + throw new RuntimeException("Unable to initialize Cloud Watch Meter Registry", e); + } } else { return Optional.empty(); diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/common/PipelineThreadPoolExecutorTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/common/PipelineThreadPoolExecutorTest.java index 11e34837a1..29a0df1a22 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/common/PipelineThreadPoolExecutorTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/common/PipelineThreadPoolExecutorTest.java @@ -6,11 +6,11 @@ package com.amazon.dataprepper.pipeline.common; import com.amazon.dataprepper.pipeline.Pipeline; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; @@ -20,7 +20,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class PipelineThreadPoolExecutorTest { private static final int NUM_THREADS = 1; @@ -37,7 +37,7 @@ public class PipelineThreadPoolExecutorTest { PipelineThreadPoolExecutor sut; - @Before + @BeforeEach public void setup() { sut = PipelineThreadPoolExecutor.newFixedThreadPool(NUM_THREADS, pipelineThreadFactory, From 4614301c3c49c7d624487f7e3224b219959fcd84 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Fri, 7 Jan 2022 10:39:22 -0600 Subject: [PATCH 33/43] Changed when AWSCloudMeter are registered Signed-off-by: Steven Bayer --- .../parser/config/MetricsConfigTest.java | 29 ++----------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java index 7fde811b98..f82b94b864 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java @@ -7,7 +7,6 @@ import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.parser.model.MetricRegistryType; -import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; import com.amazon.dataprepper.pipeline.server.PrometheusMetricsHandler; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; @@ -34,10 +33,9 @@ import java.util.Random; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.isA; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isA; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -170,38 +168,19 @@ public void testGivenEmptyConfigThenMeterRegistryCreated() { assertThat(optionalMeterRegistry.isPresent(), is(false)); } - @Test - public void testCloudWatchMeterRegistryProviderCreated() { - CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider = metricsConfig.cloudWatchMeterRegistryProvider(); - - assertThat(cloudWatchMeterRegistryProvider, isA(CloudWatchMeterRegistryProvider.class)); - } - @Test public void testGivenConfigWithCloudWatchMeterRegistryThenMeterRegistryCreated() { final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); - final CloudWatchMeterRegistryProvider registryProvider = mock(CloudWatchMeterRegistryProvider.class); - final CloudWatchMeterRegistry meterRegistry = mock(CloudWatchMeterRegistry.class); - final MeterRegistry.Config config = mock(MeterRegistry.Config.class); when(dataPrepperConfiguration.getMetricRegistryTypes()) .thenReturn(Collections.singletonList(MetricRegistryType.CloudWatch)); - when(registryProvider.getCloudWatchMeterRegistry()) - .thenReturn(meterRegistry); - when(meterRegistry.config()) - .thenReturn(config); - final Optional optionalMeterRegistry = metricsConfig.cloudWatchMeterRegistry( - dataPrepperConfiguration, - registryProvider); + final Optional optionalMeterRegistry = metricsConfig.cloudWatchMeterRegistry(dataPrepperConfiguration); assertThat(optionalMeterRegistry.isPresent(), is(true)); final MeterRegistry registry = optionalMeterRegistry.get(); assertThat(registry, isA(CloudWatchMeterRegistry.class)); - - verify(config, times(1)) - .commonTags(anyList()); } @Test @@ -211,9 +190,7 @@ public void testGivenConfigWithNoCloudWatchMeterRegistryThenNoMeterRegistryCreat when(dataPrepperConfiguration.getMetricRegistryTypes()) .thenReturn(Collections.singletonList(MetricRegistryType.Prometheus)); - final Optional optionalMeterRegistry = metricsConfig.cloudWatchMeterRegistry( - dataPrepperConfiguration, - null); + final Optional optionalMeterRegistry = metricsConfig.cloudWatchMeterRegistry(dataPrepperConfiguration); assertThat(optionalMeterRegistry.isPresent(), is(false)); } From ed97a93e6dd19b3138ea66bd4fef0d7570d396aa Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Fri, 7 Jan 2022 11:29:32 -0600 Subject: [PATCH 34/43] Removed unused import Signed-off-by: Steven Bayer --- .../com/amazon/dataprepper/server/DataPrepperServerTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java index 1be62c7247..b9aee70c0b 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java @@ -14,7 +14,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.net.InetSocketAddress; -import java.util.Random; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; From 35729ffec9f831c0cfc473e1e190a3b3424df947 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Fri, 7 Jan 2022 12:49:58 -0600 Subject: [PATCH 35/43] Removed invalid unit test Signed-off-by: Steven Bayer --- .../parser/config/MetricsConfigTest.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java index f82b94b864..bab7f7abfb 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java @@ -168,21 +168,6 @@ public void testGivenEmptyConfigThenMeterRegistryCreated() { assertThat(optionalMeterRegistry.isPresent(), is(false)); } - @Test - public void testGivenConfigWithCloudWatchMeterRegistryThenMeterRegistryCreated() { - final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); - - when(dataPrepperConfiguration.getMetricRegistryTypes()) - .thenReturn(Collections.singletonList(MetricRegistryType.CloudWatch)); - - final Optional optionalMeterRegistry = metricsConfig.cloudWatchMeterRegistry(dataPrepperConfiguration); - - assertThat(optionalMeterRegistry.isPresent(), is(true)); - - final MeterRegistry registry = optionalMeterRegistry.get(); - assertThat(registry, isA(CloudWatchMeterRegistry.class)); - } - @Test public void testGivenConfigWithNoCloudWatchMeterRegistryThenNoMeterRegistryCreated() { final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); From 70a8aa8ee5e0ceef0de196eb3505c912513ac24a Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Fri, 7 Jan 2022 12:55:10 -0600 Subject: [PATCH 36/43] Removed unused import Signed-off-by: Steven Bayer --- .../com/amazon/dataprepper/parser/config/MetricsConfigTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java index bab7f7abfb..75df0918b5 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java @@ -13,7 +13,6 @@ import com.sun.net.httpserver.Authenticator; import com.sun.net.httpserver.HttpContext; import com.sun.net.httpserver.HttpServer; -import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.MeterBinder; import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; From 73367acb4c3d76c6bcf36d4d73267fb464f1649b Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Fri, 7 Jan 2022 17:50:02 -0600 Subject: [PATCH 37/43] Added unit tests on ListPipelinesHandler Signed-off-by: Steven Bayer --- .../pipeline/server/ListPipelinesHandler.java | 19 +-- .../dataprepper/plugin/PluginCreator.java | 2 +- .../server/ListPipelinesHandlerTest.java | 110 ++++++++++++++++++ 3 files changed, 121 insertions(+), 10 deletions(-) create mode 100644 data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandlerTest.java diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandler.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandler.java index 7213515105..02021545fc 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandler.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandler.java @@ -6,17 +6,18 @@ package com.amazon.dataprepper.pipeline.server; import com.amazon.dataprepper.DataPrepper; -import java.io.IOException; -import java.net.HttpURLConnection; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; import com.fasterxml.jackson.databind.ObjectMapper; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + /** * HttpHandler to handle requests for listing pipelines running on the data prepper instance */ @@ -39,12 +40,12 @@ public PipelineListing(final String name) { } @Override - public void handle(HttpExchange exchange) throws IOException { + public void handle(final HttpExchange exchange) throws IOException { try { - List pipelines = dataPrepper.getTransformationPipelines() - .entrySet() + final List pipelines = dataPrepper.getTransformationPipelines() + .keySet() .stream() - .map(entry -> new PipelineListing(entry.getKey())) + .map(PipelineListing::new) .collect(Collectors.toList()); final byte[] response = OBJECT_MAPPER.writeValueAsString(Collections.singletonMap("pipelines", pipelines)).getBytes(); exchange.getResponseHeaders().add("Content-Type", "text/plain; charset=UTF-8"); diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/PluginCreator.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/PluginCreator.java index 49f9d493eb..03b8fc1215 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/PluginCreator.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/PluginCreator.java @@ -23,7 +23,7 @@ class PluginCreator { private static final Logger LOG = LoggerFactory.getLogger(PluginCreator.class); - T newPluginInstance(final Class pluginClass, + public T newPluginInstance(final Class pluginClass, final PluginArgumentsContext pluginArgumentsContext, final String pluginName) { Objects.requireNonNull(pluginClass); diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandlerTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandlerTest.java new file mode 100644 index 0000000000..c3fa623f9a --- /dev/null +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandlerTest.java @@ -0,0 +1,110 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.amazon.dataprepper.pipeline.server; + +import com.amazon.dataprepper.DataPrepper; +import com.amazon.dataprepper.pipeline.Pipeline; +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpExchange; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class ListPipelinesHandlerTest { + + @Test + public void testGivenNoPipelinesThenResponseWritten() throws IOException { + DataPrepper dataPrepper = mock(DataPrepper.class); + HttpExchange httpExchange = mock(HttpExchange.class); + Headers headers = mock(Headers.class); + OutputStream outputStream = mock(OutputStream.class); + Map transformationPipelines = new HashMap<>(); + + when(dataPrepper.getTransformationPipelines()) + .thenReturn(transformationPipelines); + + when(httpExchange.getResponseHeaders()) + .thenReturn(headers); + when(httpExchange.getResponseBody()) + .thenReturn(outputStream); + + ListPipelinesHandler handler = new ListPipelinesHandler(dataPrepper); + + handler.handle(httpExchange); + + verify(headers, times(1)) + .add(eq("Content-Type"), eq("text/plain; charset=UTF-8")); + verify(httpExchange, times(1)) + .sendResponseHeaders(eq(HttpURLConnection.HTTP_OK), anyLong()); + verify(outputStream, times(1)) + .write(any(byte[].class)); + verify(outputStream, times(1)) + .close(); + } + + @Test + public void testGivenPipelinesThenResponseWritten() throws IOException { + DataPrepper dataPrepper = mock(DataPrepper.class); + HttpExchange httpExchange = mock(HttpExchange.class); + Headers headers = mock(Headers.class); + OutputStream outputStream = mock(OutputStream.class); + Pipeline pipeline = mock(Pipeline.class); + Map transformationPipelines = new HashMap<>(); + transformationPipelines.put("Pipeline A", pipeline); + transformationPipelines.put("Pipeline B", pipeline); + transformationPipelines.put("Pipeline C", pipeline); + + when(dataPrepper.getTransformationPipelines()) + .thenReturn(transformationPipelines); + + when(httpExchange.getResponseHeaders()) + .thenReturn(headers); + when(httpExchange.getResponseBody()) + .thenReturn(outputStream); + + ListPipelinesHandler handler = new ListPipelinesHandler(dataPrepper); + + handler.handle(httpExchange); + + verify(headers, times(1)) + .add(eq("Content-Type"), eq("text/plain; charset=UTF-8")); + verify(httpExchange, times(1)) + .sendResponseHeaders(eq(HttpURLConnection.HTTP_OK), anyLong()); + verify(outputStream, times(1)) + .write(any(byte[].class)); + verify(outputStream, times(1)) + .close(); + } + + @Test + public void testGivenExceptionThrownThenErrorResponseWrittern() throws IOException { + HttpExchange httpExchange = mock(HttpExchange.class); + OutputStream outputStream = mock(OutputStream.class); + + when(httpExchange.getResponseBody()) + .thenReturn(outputStream); + + ListPipelinesHandler handler = new ListPipelinesHandler(null); + handler.handle(httpExchange); + + verify(httpExchange, times(1)) + .sendResponseHeaders(eq(HttpURLConnection.HTTP_INTERNAL_ERROR), eq(0L)); + verify(outputStream, times(1)) + .close(); + } +} \ No newline at end of file From e2849a40cf7b0148968f3f16aa9c9843d9cd26f1 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Mon, 10 Jan 2022 10:53:56 -0600 Subject: [PATCH 38/43] Add unit tests for PrometheusMetricsHandler Signed-off-by: Steven Bayer --- .../pipeline/server/ListPipelinesHandler.java | 2 +- .../server/PrometheusMetricsHandler.java | 9 +- .../server/PrometheusMetricsHandlerTest.java | 85 +++++++++++++++++++ 3 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandlerTest.java diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandler.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandler.java index 02021545fc..669bdd4554 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandler.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandler.java @@ -51,7 +51,7 @@ public void handle(final HttpExchange exchange) throws IOException { exchange.getResponseHeaders().add("Content-Type", "text/plain; charset=UTF-8"); exchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, response.length); exchange.getResponseBody().write(response); - } catch (Exception e) { + } catch (final Exception e) { LOG.error("Caught exception listing pipelines", e); exchange.sendResponseHeaders(HttpURLConnection.HTTP_INTERNAL_ERROR, 0); } finally { diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandler.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandler.java index 8bc689e389..f9f65f8e51 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandler.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandler.java @@ -13,13 +13,14 @@ import java.io.IOException; import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; /** * HttpHandler to handle requests for Prometheus metrics */ public class PrometheusMetricsHandler implements HttpHandler { - private PrometheusMeterRegistry prometheusMeterRegistry; + private final PrometheusMeterRegistry prometheusMeterRegistry; private final Logger LOG = LoggerFactory.getLogger(PrometheusMetricsHandler.class); public PrometheusMetricsHandler(final PrometheusMeterRegistry prometheusMeterRegistry) { @@ -27,13 +28,13 @@ public PrometheusMetricsHandler(final PrometheusMeterRegistry prometheusMeterReg } @Override - public void handle(HttpExchange exchange) throws IOException { + public void handle(final HttpExchange exchange) throws IOException { try { - byte response[] = prometheusMeterRegistry.scrape().getBytes("UTF-8"); + final byte[] response = prometheusMeterRegistry.scrape().getBytes(StandardCharsets.UTF_8); exchange.getResponseHeaders().add("Content-Type", "text/plain; charset=UTF-8"); exchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, response.length); exchange.getResponseBody().write(response); - } catch (Exception e) { + } catch (final Exception e) { LOG.error("Encountered exception scraping prometheus meter registry", e); exchange.sendResponseHeaders(HttpURLConnection.HTTP_INTERNAL_ERROR, 0); } finally { diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandlerTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandlerTest.java new file mode 100644 index 0000000000..f43fb98df5 --- /dev/null +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandlerTest.java @@ -0,0 +1,85 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.amazon.dataprepper.pipeline.server; + +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpExchange; +import io.micrometer.prometheus.PrometheusMeterRegistry; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class PrometheusMetricsHandlerTest { + + @Mock + PrometheusMeterRegistry meterRegistry; + + @InjectMocks + PrometheusMetricsHandler metricsHandler; + + @Mock + HttpExchange exchange; + + @Mock + OutputStream responseBody; + + @BeforeEach + public void beforeEach() { + when(exchange.getResponseBody()) + .thenReturn(responseBody); + } + + @Test + public void testResponseWritten() throws IOException { + final Headers headers = mock(Headers.class); + when(exchange.getResponseHeaders()) + .thenReturn(headers); + + final String testString = "I am a string used in a test"; + when(meterRegistry.scrape()) + .thenReturn(testString); + + metricsHandler.handle(exchange); + + + verify(headers, times(1)) + .add(eq("Content-Type"), eq("text/plain; charset=UTF-8")); + + final byte[] response = testString.getBytes(StandardCharsets.UTF_8); + verify(exchange, times(1)) + .sendResponseHeaders(eq(HttpURLConnection.HTTP_OK), eq((long) response.length)); + verify(responseBody, times(1)) + .write(response); + verify(responseBody, times(1)) + .close(); + } + + @Test + public void testHandleException() throws IOException { + metricsHandler.handle(exchange); + + verify(exchange, times(1)) + .sendResponseHeaders(eq(HttpURLConnection.HTTP_INTERNAL_ERROR), eq(0L)); + verify(responseBody, times(1)) + .close(); + } + +} From 1b5aee45d4433098e1b1c6527cdd3da2824798f0 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Tue, 18 Jan 2022 10:33:24 -0600 Subject: [PATCH 39/43] Remove Optional types from Spring configuration files Signed-off-by: Steven Bayer --- .../parser/config/MetricsConfig.java | 37 ++++++++-------- .../parser/config/MetricsConfigTest.java | 42 ++++++++----------- 2 files changed, 34 insertions(+), 45 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java index 4bbb951a36..e237aa3bee 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java @@ -29,6 +29,7 @@ import io.micrometer.prometheus.PrometheusMeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import software.amazon.awssdk.core.exception.SdkClientException; @@ -36,7 +37,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Optional; import static com.amazon.dataprepper.DataPrepper.getServiceNameForMetrics; import static com.amazon.dataprepper.metrics.MetricNames.SERVICE_NAME; @@ -90,9 +90,9 @@ private void configureMetricRegistry(final MeterRegistry meterRegistry) { } @Bean - public Optional prometheusMeterRegistry( + public MeterRegistry prometheusMeterRegistry( final DataPrepperConfiguration dataPrepperConfiguration, - final Optional optionalAuthenticator, + @Autowired(required = false) final Authenticator authenticator, final HttpServer server ) { if (dataPrepperConfiguration.getMetricRegistryTypes().contains(MetricRegistryType.Prometheus)) { @@ -105,53 +105,50 @@ public Optional prometheusMeterRegistry( contextList.add(server.createContext(METRICS_CONTEXT_PREFIX + "/prometheus", metricsHandler)); contextList.add(server.createContext(METRICS_CONTEXT_PREFIX + "/sys", metricsHandler)); - optionalAuthenticator.ifPresent( - authenticator -> contextList.forEach( - context -> context.setAuthenticator(authenticator))); + if (authenticator != null) { + contextList.forEach(context -> context.setAuthenticator(authenticator)); + } - return Optional.of(meterRegistry); + return meterRegistry; } else { - return Optional.empty(); + return null; } } @Bean - public Optional cloudWatchMeterRegistry(final DataPrepperConfiguration dataPrepperConfiguration) { + public MeterRegistry cloudWatchMeterRegistry(final DataPrepperConfiguration dataPrepperConfiguration) { if (dataPrepperConfiguration.getMetricRegistryTypes().contains(MetricRegistryType.CloudWatch)) { try { final CloudWatchMeterRegistryProvider provider = new CloudWatchMeterRegistryProvider(); final CloudWatchMeterRegistry meterRegistry = provider.getCloudWatchMeterRegistry(); configureMetricRegistry(meterRegistry); - return Optional.of(meterRegistry); - } catch (SdkClientException e) { + return meterRegistry; + } catch (final SdkClientException e) { LOG.warn("Unable to configure Cloud Watch Meter Registry but Meter Registry was requested in Data Prepper Configuration"); throw new RuntimeException("Unable to initialize Cloud Watch Meter Registry", e); } } else { - return Optional.empty(); + return null; } } @Bean public CompositeMeterRegistry systemMeterRegistry( final List meterBinders, - final List> optionalMeterRegistries + final List meterRegistries ) { final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); LOG.debug("{} Meter Binder beans registered.", meterBinders.size()); meterBinders.forEach(binder -> binder.bindTo(compositeMeterRegistry)); - optionalMeterRegistries.stream() - .filter(Optional::isPresent) - .map(Optional::get) - .forEach(meterRegistry -> { - compositeMeterRegistry.add(meterRegistry); - Metrics.addRegistry(meterRegistry); - }); + meterRegistries.forEach(meterRegistry -> { + compositeMeterRegistry.add(meterRegistry); + Metrics.addRegistry(meterRegistry); + }); return compositeMeterRegistry; } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java index 75df0918b5..db84f2659d 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java @@ -28,12 +28,12 @@ import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.Random; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isA; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -99,7 +99,6 @@ public void testJvmThreadMetrics() { @Test public void testGivenConfigWithPrometheusMeterRegistryThenMeterRegistryCreated() { final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); - final Optional optionalAuthenticator = Optional.empty(); final HttpServer server = mock(HttpServer.class); final HttpContext context = mock(HttpContext.class); @@ -108,15 +107,12 @@ public void testGivenConfigWithPrometheusMeterRegistryThenMeterRegistryCreated() when(server.createContext(anyString(), any(PrometheusMetricsHandler.class))) .thenReturn(context); - final Optional optionalMeterRegistry = metricsConfig.prometheusMeterRegistry( + final MeterRegistry meterRegistry = metricsConfig.prometheusMeterRegistry( dataPrepperConfiguration, - optionalAuthenticator, + null, server); - assertThat(optionalMeterRegistry.isPresent(), is(true)); - - final MeterRegistry registry = optionalMeterRegistry.get(); - assertThat(registry, isA(PrometheusMeterRegistry.class)); + assertThat(meterRegistry, isA(PrometheusMeterRegistry.class)); verify(server, times(2)) .createContext(anyString(), any(PrometheusMetricsHandler.class)); @@ -126,7 +122,6 @@ public void testGivenConfigWithPrometheusMeterRegistryThenMeterRegistryCreated() public void testGivenConfigWithPrometheusMeterRegistryAndAuthenticatorThenMeterRegistryCreated() { final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); final Authenticator authenticator = mock(Authenticator.class); - final Optional optionalAuthenticator = Optional.of(authenticator); final HttpServer server = mock(HttpServer.class); final HttpContext context = mock(HttpContext.class); @@ -135,15 +130,12 @@ public void testGivenConfigWithPrometheusMeterRegistryAndAuthenticatorThenMeterR when(server.createContext(anyString(), any(PrometheusMetricsHandler.class))) .thenReturn(context); - final Optional optionalMeterRegistry = metricsConfig.prometheusMeterRegistry( + final MeterRegistry meterRegistry = metricsConfig.prometheusMeterRegistry( dataPrepperConfiguration, - optionalAuthenticator, + authenticator, server); - assertThat(optionalMeterRegistry.isPresent(), is(true)); - - final MeterRegistry registry = optionalMeterRegistry.get(); - assertThat(registry, isA(PrometheusMeterRegistry.class)); + assertThat(meterRegistry, isA(PrometheusMeterRegistry.class)); verify(server, times(2)) .createContext(anyString(), any(PrometheusMetricsHandler.class)); @@ -159,12 +151,12 @@ public void testGivenEmptyConfigThenMeterRegistryCreated() { when(dataPrepperConfiguration.getMetricRegistryTypes()) .thenReturn(Collections.emptyList()); - final Optional optionalMeterRegistry = metricsConfig.prometheusMeterRegistry( -dataPrepperConfiguration, - Optional.empty(), + final MeterRegistry meterRegistry = metricsConfig.prometheusMeterRegistry( + dataPrepperConfiguration, + null, null); - assertThat(optionalMeterRegistry.isPresent(), is(false)); + assertThat(meterRegistry, is(nullValue())); } @Test @@ -174,9 +166,9 @@ public void testGivenConfigWithNoCloudWatchMeterRegistryThenNoMeterRegistryCreat when(dataPrepperConfiguration.getMetricRegistryTypes()) .thenReturn(Collections.singletonList(MetricRegistryType.Prometheus)); - final Optional optionalMeterRegistry = metricsConfig.cloudWatchMeterRegistry(dataPrepperConfiguration); + final MeterRegistry meterRegistry = metricsConfig.cloudWatchMeterRegistry(dataPrepperConfiguration); - assertThat(optionalMeterRegistry.isPresent(), is(false)); + assertThat(meterRegistry, is(nullValue())); } @Test @@ -187,11 +179,11 @@ public void testGivenListOfMeterBindersWhenSystemMeterRegistryThenAllMeterBinder final List meterBinders = Collections.nCopies(copies, binder); final MeterRegistry meterRegistryMock = mock(MeterRegistry.class); - final List> optionalMeterRegistries = Collections.nCopies(1, Optional.of(meterRegistryMock)); + final List meterRegistries = Collections.nCopies(1, meterRegistryMock); final CompositeMeterRegistry meterRegistry = metricsConfig.systemMeterRegistry( meterBinders, - optionalMeterRegistries); + meterRegistries); assertThat(meterRegistry, isA(CompositeMeterRegistry.class)); verify(binder, times(copies)).bindTo(any(MeterRegistry.class)); @@ -202,11 +194,11 @@ public void testGivenEmptyListOfMeterBindersWhenSystemMeterRegistryThenNoMeterBi final List meterBinders = Collections.emptyList(); final MeterRegistry meterRegistryMock = mock(MeterRegistry.class); - final List> optionalMeterRegistries = Collections.nCopies(1, Optional.of(meterRegistryMock)); + final List meterRegistries = Collections.nCopies(1, meterRegistryMock); final CompositeMeterRegistry meterRegistry = metricsConfig.systemMeterRegistry( meterBinders, - optionalMeterRegistries); + meterRegistries); assertThat(meterRegistry, isA(CompositeMeterRegistry.class)); } From 2f116f5b1086bcad519d35d807db40239b20b49f Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Tue, 18 Jan 2022 16:31:18 -0600 Subject: [PATCH 40/43] Remove Optional types from Spring configuration files, split meter creation and config Signed-off-by: Steven Bayer --- .../config/DataPrepperAppConfiguration.java | 7 +- .../parser/config/MetricsConfig.java | 32 +--- .../CloudWatchMeterRegistryProvider.java | 17 +- .../pipeline/server/HttpServerProvider.java | 64 +++++++ .../DataPrepperServerConfiguration.java | 89 ++++----- .../dataprepper/plugin/PluginCreator.java | 5 +- .../amazon/dataprepper/DataPrepperTests.java | 17 +- .../DataPrepperAppConfigurationTest.java | 24 ++- .../parser/config/MetricsConfigTest.java | 67 +++---- .../PipelineParserConfigurationTest.java | 3 +- .../server/HttpServerProviderTest.java | 144 ++++++++++++++ .../server/ListPipelinesHandlerTest.java | 53 +++--- .../DataPrepperServerConfigurationTest.java | 177 ++++++------------ .../server/DataPrepperServerTest.java | 9 +- 14 files changed, 403 insertions(+), 305 deletions(-) create mode 100644 data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/HttpServerProvider.java create mode 100644 data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/HttpServerProviderTest.java diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java index d4316fc1a1..f0c6ae7ea1 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfiguration.java @@ -17,7 +17,6 @@ import java.io.File; import java.io.IOException; -import java.util.Optional; @Configuration public class DataPrepperAppConfiguration { @@ -31,7 +30,7 @@ public DataPrepperArgs dataPrepperArgs(final Environment environment) { LOG.info("Command line args: {}", commandLineArgs); if (commandLineArgs != null) { - String[] args = commandLineArgs.split(COMMAND_LINE_ARG_DELIMITER); + final String[] args = commandLineArgs.split(COMMAND_LINE_ARG_DELIMITER); return new DataPrepperArgs(args); } else { @@ -59,7 +58,7 @@ public DataPrepperConfiguration dataPrepperConfiguration( } @Bean - public Optional pluginModel(final DataPrepperConfiguration dataPrepperConfiguration) { - return Optional.ofNullable(dataPrepperConfiguration.getAuthentication()); + public PluginModel authentication(final DataPrepperConfiguration dataPrepperConfiguration) { + return dataPrepperConfiguration.getAuthentication(); } } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java index e237aa3bee..83abb8869d 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java @@ -8,12 +8,8 @@ import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.parser.model.MetricRegistryType; import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; -import com.amazon.dataprepper.pipeline.server.PrometheusMetricsHandler; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.sun.net.httpserver.Authenticator; -import com.sun.net.httpserver.HttpContext; -import com.sun.net.httpserver.HttpServer; import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Metrics; @@ -29,12 +25,10 @@ import io.micrometer.prometheus.PrometheusMeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import software.amazon.awssdk.core.exception.SdkClientException; -import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -90,25 +84,11 @@ private void configureMetricRegistry(final MeterRegistry meterRegistry) { } @Bean - public MeterRegistry prometheusMeterRegistry( - final DataPrepperConfiguration dataPrepperConfiguration, - @Autowired(required = false) final Authenticator authenticator, - final HttpServer server - ) { + public MeterRegistry prometheusMeterRegistry(final DataPrepperConfiguration dataPrepperConfiguration) { if (dataPrepperConfiguration.getMetricRegistryTypes().contains(MetricRegistryType.Prometheus)) { final PrometheusMeterRegistry meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); configureMetricRegistry(meterRegistry); - final PrometheusMetricsHandler metricsHandler = new PrometheusMetricsHandler(meterRegistry); - - final List contextList = new ArrayList<>(2); - contextList.add(server.createContext(METRICS_CONTEXT_PREFIX + "/prometheus", metricsHandler)); - contextList.add(server.createContext(METRICS_CONTEXT_PREFIX + "/sys", metricsHandler)); - - if (authenticator != null) { - contextList.forEach(context -> context.setAuthenticator(authenticator)); - } - return meterRegistry; } else { @@ -117,13 +97,15 @@ public MeterRegistry prometheusMeterRegistry( } @Bean - public MeterRegistry cloudWatchMeterRegistry(final DataPrepperConfiguration dataPrepperConfiguration) { + public MeterRegistry cloudWatchMeterRegistry( + final DataPrepperConfiguration dataPrepperConfiguration, + final CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider + ) { if (dataPrepperConfiguration.getMetricRegistryTypes().contains(MetricRegistryType.CloudWatch)) { try { - final CloudWatchMeterRegistryProvider provider = new CloudWatchMeterRegistryProvider(); - final CloudWatchMeterRegistry meterRegistry = provider.getCloudWatchMeterRegistry(); - + final CloudWatchMeterRegistry meterRegistry = cloudWatchMeterRegistryProvider.getCloudWatchMeterRegistry(); configureMetricRegistry(meterRegistry); + return meterRegistry; } catch (final SdkClientException e) { LOG.warn("Unable to configure Cloud Watch Meter Registry but Meter Registry was requested in Data Prepper Configuration"); diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java index 469136c492..4467f46460 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java @@ -12,6 +12,8 @@ import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; +import javax.inject.Inject; +import javax.inject.Named; import java.io.IOException; import java.io.InputStream; import java.util.Properties; @@ -25,12 +27,14 @@ * {@link CloudWatchMeterRegistryProvider} also has a constructor with {@link CloudWatchAsyncClient} that will be used * for communication with Cloudwatch. */ +@Named public class CloudWatchMeterRegistryProvider { private static final String CLOUDWATCH_PROPERTIES = "cloudwatch.properties"; private static final Logger LOG = LoggerFactory.getLogger(CloudWatchMeterRegistryProvider.class); private final CloudWatchMeterRegistry cloudWatchMeterRegistry; + @Inject public CloudWatchMeterRegistryProvider() { this(CLOUDWATCH_PROPERTIES, CloudWatchAsyncClient.create()); } @@ -55,22 +59,17 @@ public CloudWatchMeterRegistry getCloudWatchMeterRegistry() { * Returns CloudWatchConfig using the properties from {@link #CLOUDWATCH_PROPERTIES} */ private CloudWatchConfig createCloudWatchConfig(final String cloudWatchPropertiesFilePath) { - CloudWatchConfig cloudWatchConfig = null; try (final InputStream inputStream = requireNonNull(getClass().getClassLoader() .getResourceAsStream(cloudWatchPropertiesFilePath))) { final Properties cloudwatchProperties = new Properties(); cloudwatchProperties.load(inputStream); - cloudWatchConfig = new CloudWatchConfig() { - @Override - public String get(final String key) { - return cloudwatchProperties.getProperty(key); - } - }; - } catch (IOException ex) { + return cloudwatchProperties::getProperty; + } catch (final IOException ex) { LOG.error("Encountered exception in creating CloudWatchConfig for CloudWatchMeterRegistry, " + "Proceeding without metrics", ex); + //If there is no registry attached, micrometer will make NoopMeters which are discarded. + return null; } - return cloudWatchConfig; } } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/HttpServerProvider.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/HttpServerProvider.java new file mode 100644 index 0000000000..2943d57f5e --- /dev/null +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/HttpServerProvider.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.amazon.dataprepper.pipeline.server; + +import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; +import com.sun.net.httpserver.HttpServer; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsParameters; +import com.sun.net.httpserver.HttpsServer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Provider; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import java.io.IOException; +import java.net.InetSocketAddress; + +@Named +public class HttpServerProvider implements Provider { + private static final Logger LOG = LoggerFactory.getLogger(HttpServerProvider.class); + + private final DataPrepperConfiguration dataPrepperConfiguration; + + @Inject + public HttpServerProvider(final DataPrepperConfiguration dataPrepperConfiguration) { + this.dataPrepperConfiguration = dataPrepperConfiguration; + } + + @Override + public HttpServer get() { + final InetSocketAddress socketAddress = new InetSocketAddress(dataPrepperConfiguration.getServerPort()); + try { + if (dataPrepperConfiguration.ssl()) { + LOG.info("Creating Data Prepper server with TLS"); + final SSLContext sslContext = SslUtil.createSslContext( + dataPrepperConfiguration.getKeyStoreFilePath(), + dataPrepperConfiguration.getKeyStorePassword(), + dataPrepperConfiguration.getPrivateKeyPassword() + ); + + final HttpsServer server = HttpsServer.create(socketAddress, 0); + server.setHttpsConfigurator(new HttpsConfigurator(sslContext) { + public void configure(final HttpsParameters params) { + final SSLContext context = getSSLContext(); + final SSLParameters sslParams = context.getDefaultSSLParameters(); + params.setSSLParameters(sslParams); + } + }); + + return server; + } else { + return HttpServer.create(socketAddress, 0); + } + } catch (final IOException ex) { + throw new RuntimeException("Failed to create server", ex); + } + } +} diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java index e31418d4e0..9782eef395 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java @@ -9,26 +9,23 @@ import com.amazon.dataprepper.model.configuration.PluginModel; import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.model.plugin.PluginFactory; -import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.pipeline.server.DataPrepperCoreAuthenticationProvider; +import com.amazon.dataprepper.pipeline.server.HttpServerProvider; import com.amazon.dataprepper.pipeline.server.ListPipelinesHandler; +import com.amazon.dataprepper.pipeline.server.PrometheusMetricsHandler; import com.amazon.dataprepper.pipeline.server.ShutdownHandler; -import com.amazon.dataprepper.pipeline.server.SslUtil; import com.sun.net.httpserver.Authenticator; import com.sun.net.httpserver.HttpContext; +import com.sun.net.httpserver.HttpHandler; import com.sun.net.httpserver.HttpServer; -import com.sun.net.httpserver.HttpsConfigurator; -import com.sun.net.httpserver.HttpsParameters; -import com.sun.net.httpserver.HttpsServer; +import io.micrometer.prometheus.PrometheusMeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLParameters; -import java.io.IOException; -import java.net.InetSocketAddress; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Optional; @@ -36,36 +33,41 @@ public class DataPrepperServerConfiguration { private static final Logger LOG = LoggerFactory.getLogger(DataPrepperServerConfiguration.class); - @Bean - public HttpServer httpServer(final DataPrepperConfiguration dataPrepperConfiguration) { - final InetSocketAddress socketAddress = new InetSocketAddress(dataPrepperConfiguration.getServerPort()); - try { - if (dataPrepperConfiguration.ssl()) { - LOG.info("Creating Data Prepper server with TLS"); - final SSLContext sslContext = SslUtil.createSslContext( - dataPrepperConfiguration.getKeyStoreFilePath(), - dataPrepperConfiguration.getKeyStorePassword(), - dataPrepperConfiguration.getPrivateKeyPassword() - ); - - final HttpsServer server = HttpsServer.create(socketAddress, 0); - server.setHttpsConfigurator(new HttpsConfigurator(sslContext) { - public void configure(final HttpsParameters params) { - final SSLContext context = getSSLContext(); - final SSLParameters sslParams = context.getDefaultSSLParameters(); - params.setSSLParameters(sslParams); - } - }); - - return server; - } else { - return HttpServer.create(socketAddress, 0); + private void createContext( + final HttpServer httpServer, + final HttpHandler httpHandler, + @Nullable final Authenticator authenticator, + final String ... paths + ) { + for (final String path : paths) { + final HttpContext context = httpServer.createContext(path, httpHandler); + + if (authenticator != null) { + context.setAuthenticator(authenticator); } - } catch (final IOException ex) { - throw new RuntimeException("Failed to create server", ex); } } + @Bean + public HttpServer httpServer( + final HttpServerProvider httpServerProvider, + final ListPipelinesHandler listPipelinesHandler, + @Autowired(required = false) @Nullable final PrometheusMeterRegistry prometheusMeterRegistry, + @Autowired(required = false) @Nullable final Authenticator authenticator + ) { + + final HttpServer server = httpServerProvider.get(); + + createContext(server, listPipelinesHandler, authenticator, "/list"); + + if (prometheusMeterRegistry != null) { + final PrometheusMetricsHandler prometheusMetricsHandler = new PrometheusMetricsHandler(prometheusMeterRegistry); + createContext(server, prometheusMetricsHandler, authenticator, "/metrics/prometheus", "/metrics/sys"); + } + + return server; + } + private void printInsecurePluginModelWarning() { LOG.warn("Creating data prepper server without authentication. This is not secure."); LOG.warn("In order to set up Http Basic authentication for the data prepper server, " + @@ -102,22 +104,13 @@ public DataPrepperCoreAuthenticationProvider authenticationProvider( } @Bean - public Optional optionalAuthenticator(final DataPrepperCoreAuthenticationProvider authenticationProvider) { - return Optional.ofNullable(authenticationProvider.getAuthenticator()); + public Authenticator authenticator(final DataPrepperCoreAuthenticationProvider authenticationProvider) { + return authenticationProvider.getAuthenticator(); } @Bean - public ListPipelinesHandler listPipelinesHandler( - final DataPrepper dataPrepper, - final Optional optionalAuthenticator, - final HttpServer server - ) { - final ListPipelinesHandler listPipelinesHandler = new ListPipelinesHandler(dataPrepper); - - final HttpContext context = server.createContext("/list", listPipelinesHandler); - optionalAuthenticator.ifPresent(context::setAuthenticator); - - return listPipelinesHandler; + public ListPipelinesHandler listPipelinesHandler(final DataPrepper dataPrepper) { + return new ListPipelinesHandler(dataPrepper); } @Bean diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/PluginCreator.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/PluginCreator.java index 03b8fc1215..02548f9b31 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/PluginCreator.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/PluginCreator.java @@ -23,7 +23,7 @@ class PluginCreator { private static final Logger LOG = LoggerFactory.getLogger(PluginCreator.class); - public T newPluginInstance(final Class pluginClass, + T newPluginInstance(final Class pluginClass, final PluginArgumentsContext pluginArgumentsContext, final String pluginName) { Objects.requireNonNull(pluginClass); @@ -38,7 +38,8 @@ public T newPluginInstance(final Class pluginClass, return (T) constructor.newInstance(constructorArguments); } catch (final IllegalAccessException | InstantiationException ex) { LOG.error("Encountered exception while instantiating the plugin {}", pluginClass.getSimpleName(), ex); - throw new InvalidPluginDefinitionException("Unable to access or instantiate the plugin '" + pluginClass.getSimpleName() + ".'", ex); + throw new InvalidPluginDefinitionException( + "Unable to access or instantiate the plugin '" + pluginClass.getSimpleName() + ".'", ex); } catch (final InvocationTargetException ex) { LOG.error("Encountered exception while instantiating the plugin {}", pluginClass.getSimpleName(), ex); throw new PluginInvocationException("Exception throw from the plugin'" + pluginClass.getSimpleName() + "'." , ex); diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java index d2dd19413a..f1b98a7011 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java @@ -25,7 +25,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -70,7 +69,7 @@ public void testGivenValidInputThenInstanceCreation() { @Test public void testGivenInvalidInputThenExceptionThrown() { - PipelineParser pipelineParser = mock(PipelineParser.class); + final PipelineParser pipelineParser = mock(PipelineParser.class); assertThrows( RuntimeException.class, @@ -92,23 +91,23 @@ public void testGivenValidPipelineParserThenReturnResultOfParseConfiguration() { public void testGivenValidPipelineParserWhenExecuteThenAllPipelinesExecuteAndServerStartAndReturnTrue() { assertThat(dataPrepper.execute(), Matchers.is(true)); - verify(pipeline, times(1)).execute(); - verify(dataPrepperServer, times(1)).start(); + verify(pipeline).execute(); + verify(dataPrepperServer).start(); } @Test public void testDataPrepperShutdown() { dataPrepper.shutdown(); - verify(pipeline, times(1)).shutdown(); + verify(pipeline).shutdown(); } @Test public void testDataPrepperShutdownPipeline() { - Pipeline randomPipeline = mock(Pipeline.class); + final Pipeline randomPipeline = mock(Pipeline.class); parseConfigurationFixture.put("Random Pipeline", randomPipeline); dataPrepper.shutdown("Random Pipeline"); - verify(randomPipeline, times(1)).shutdown(); + verify(randomPipeline).shutdown(); } @Test @@ -120,12 +119,12 @@ public void testDataPrepperShutdownNonExistentPipelineWithoutException() { public void testShutdownDataPrepperServer() { dataPrepper.shutdownDataPrepperServer(); - verify(dataPrepperServer, times(1)).stop(); + verify(dataPrepperServer).stop(); } @Test public void testGivenEnvVarNotSetThenDefaultServiceNameReturned() { - String actual = DataPrepper.getServiceNameForMetrics(); + final String actual = DataPrepper.getServiceNameForMetrics(); assertThat(actual, Matchers.is("dataprepper")); } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfigurationTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfigurationTest.java index ac97a88756..0c3bbdbcdd 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfigurationTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/DataPrepperAppConfigurationTest.java @@ -16,7 +16,6 @@ import java.io.File; import java.io.IOException; -import java.util.Optional; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; @@ -26,7 +25,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -81,7 +79,7 @@ public void testGivenNoPipelineConfigArgThenResultOfObjectMapperReadValueIsRetur final DataPrepperConfiguration configuration = appConfiguration.dataPrepperConfiguration(dataPrepperArgs, objectMapper); - verify(dataPrepperArgs, times(1)) + verify(dataPrepperArgs) .getDataPrepperConfigFileLocation(); assertThat(configuration, notNullValue()); } @@ -122,9 +120,23 @@ public void testGivenInvalidPipelineConfigArgThenExceptionThrown() throws IOExce public void testPluginModelFromDataPrepperConfigurationAuthentication() { final DataPrepperConfiguration configuration = mock(DataPrepperConfiguration.class); - final Optional optional = appConfiguration.pluginModel(configuration); + final PluginModel pluginModel = appConfiguration.authentication(configuration); - assertThat(optional.isPresent(), is(false)); - verify(configuration, times(1)).getAuthentication(); + assertThat(pluginModel, is(nullValue())); + verify(configuration).getAuthentication(); + } + + @Test + public void testGivenReturnAuthenticationThenBeanShouldEqualAuthentication() { + final DataPrepperConfiguration configuration = mock(DataPrepperConfiguration.class); + final PluginModel expected = mock(PluginModel.class); + + when(configuration.getAuthentication()) + .thenReturn(expected); + + final PluginModel pluginModel = appConfiguration.authentication(configuration); + + assertThat(pluginModel, is(expected)); + verify(configuration).getAuthentication(); } } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java index db84f2659d..a963fd801c 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/MetricsConfigTest.java @@ -7,12 +7,10 @@ import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.parser.model.MetricRegistryType; -import com.amazon.dataprepper.pipeline.server.PrometheusMetricsHandler; +import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.sun.net.httpserver.Authenticator; -import com.sun.net.httpserver.HttpContext; -import com.sun.net.httpserver.HttpServer; +import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.MeterBinder; import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; @@ -35,8 +33,6 @@ import static org.hamcrest.Matchers.isA; import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -99,76 +95,57 @@ public void testJvmThreadMetrics() { @Test public void testGivenConfigWithPrometheusMeterRegistryThenMeterRegistryCreated() { final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); - final HttpServer server = mock(HttpServer.class); - final HttpContext context = mock(HttpContext.class); when(dataPrepperConfiguration.getMetricRegistryTypes()) .thenReturn(Collections.singletonList(MetricRegistryType.Prometheus)); - when(server.createContext(anyString(), any(PrometheusMetricsHandler.class))) - .thenReturn(context); - final MeterRegistry meterRegistry = metricsConfig.prometheusMeterRegistry( - dataPrepperConfiguration, - null, - server); + final MeterRegistry meterRegistry = metricsConfig.prometheusMeterRegistry(dataPrepperConfiguration); assertThat(meterRegistry, isA(PrometheusMeterRegistry.class)); - - verify(server, times(2)) - .createContext(anyString(), any(PrometheusMetricsHandler.class)); } @Test - public void testGivenConfigWithPrometheusMeterRegistryAndAuthenticatorThenMeterRegistryCreated() { + public void testGivenEmptyConfigThenMeterRegistryCreated() { final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); - final Authenticator authenticator = mock(Authenticator.class); - final HttpServer server = mock(HttpServer.class); - final HttpContext context = mock(HttpContext.class); when(dataPrepperConfiguration.getMetricRegistryTypes()) - .thenReturn(Collections.singletonList(MetricRegistryType.Prometheus)); - when(server.createContext(anyString(), any(PrometheusMetricsHandler.class))) - .thenReturn(context); - - final MeterRegistry meterRegistry = metricsConfig.prometheusMeterRegistry( - dataPrepperConfiguration, - authenticator, - server); - - assertThat(meterRegistry, isA(PrometheusMeterRegistry.class)); + .thenReturn(Collections.emptyList()); - verify(server, times(2)) - .createContext(anyString(), any(PrometheusMetricsHandler.class)); + final MeterRegistry meterRegistry = metricsConfig.prometheusMeterRegistry(dataPrepperConfiguration); - verify(context, times(2)) - .setAuthenticator(eq(authenticator)); + assertThat(meterRegistry, is(nullValue())); } @Test - public void testGivenEmptyConfigThenMeterRegistryCreated() { + public void testGivenConfigWithNoCloudWatchMeterRegistryThenNoMeterRegistryCreated() { final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); when(dataPrepperConfiguration.getMetricRegistryTypes()) - .thenReturn(Collections.emptyList()); + .thenReturn(Collections.singletonList(MetricRegistryType.Prometheus)); - final MeterRegistry meterRegistry = metricsConfig.prometheusMeterRegistry( - dataPrepperConfiguration, - null, - null); + final MeterRegistry meterRegistry = metricsConfig.cloudWatchMeterRegistry(dataPrepperConfiguration, null); assertThat(meterRegistry, is(nullValue())); } @Test - public void testGivenConfigWithNoCloudWatchMeterRegistryThenNoMeterRegistryCreated() { + public void testGivenConfigWithCloudWatchMeterRegistryThenNoMeterRegistryCreated() { + final CloudWatchMeterRegistryProvider provider = mock(CloudWatchMeterRegistryProvider.class); + final CloudWatchMeterRegistry expected = mock(CloudWatchMeterRegistry.class); + final MeterRegistry.Config config = mock(MeterRegistry.Config.class); final DataPrepperConfiguration dataPrepperConfiguration = mock(DataPrepperConfiguration.class); + when(provider.getCloudWatchMeterRegistry()) + .thenReturn(expected); + when(expected.config()) + .thenReturn(config); + when(dataPrepperConfiguration.getMetricRegistryTypes()) - .thenReturn(Collections.singletonList(MetricRegistryType.Prometheus)); + .thenReturn(Collections.singletonList(MetricRegistryType.CloudWatch)); - final MeterRegistry meterRegistry = metricsConfig.cloudWatchMeterRegistry(dataPrepperConfiguration); + final MeterRegistry meterRegistry = metricsConfig.cloudWatchMeterRegistry(dataPrepperConfiguration, provider); - assertThat(meterRegistry, is(nullValue())); + assertThat(meterRegistry, is(expected)); } @Test diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/PipelineParserConfigurationTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/PipelineParserConfigurationTest.java index c18af38636..a3d6150baa 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/PipelineParserConfigurationTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/config/PipelineParserConfigurationTest.java @@ -15,7 +15,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -38,6 +37,6 @@ void pipelineParser() { final PipelineParser pipelineParser = pipelineParserConfiguration.pipelineParser(args, pluginFactory); assertThat(pipelineParser, is(notNullValue())); - verify(args, times(1)).getPipelineConfigFileLocation(); + verify(args).getPipelineConfigFileLocation(); } } \ No newline at end of file diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/HttpServerProviderTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/HttpServerProviderTest.java new file mode 100644 index 0000000000..f2361da931 --- /dev/null +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/HttpServerProviderTest.java @@ -0,0 +1,144 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.amazon.dataprepper.pipeline.server; + +import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; +import com.sun.net.httpserver.HttpServer; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsServer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.net.DatagramSocket; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.util.Random; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isA; +import static org.hamcrest.Matchers.not; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class HttpServerProviderTest { + private static final Random r = new Random(); + private static final Integer MAX_PORT = 65535; + private static final String P12_KEYSTORE = "src/test/resources/tls/test_keystore.p12"; + + private static boolean available(final int port) { + ServerSocket ss = null; + DatagramSocket ds = null; + try { + ss = new ServerSocket(port); + ss.setReuseAddress(true); + ds = new DatagramSocket(port); + ds.setReuseAddress(true); + return true; + } catch (final IOException ignored) { + } finally { + if (ds != null) { + ds.close(); + } + + if (ss != null) { + try { + ss.close(); + } catch (final IOException ignored) { + } + } + } + + return false; + } + + private static int getPort() { + final int MAX_RETRIES = 10; + for (int i = 0; i < MAX_RETRIES; i++) { + final int electivePort = r.nextInt(MAX_PORT); + if (available(electivePort)) { + return electivePort; + } + } + throw new RuntimeException("No available port found"); + } + + @Mock + private DataPrepperConfiguration configuration; + + @InjectMocks + private HttpServerProvider httpServerProvider; + + @Test + public void testGivenNoSslThenInsecureServerCreated() { + final int expectedPort = getPort(); + when(configuration.ssl()) + .thenReturn(false); + when(configuration.getServerPort()) + .thenReturn(expectedPort); + + final HttpServer server = httpServerProvider.get(); + + assertThat(server, isA(HttpServer.class)); + assertThat(server, not(isA(HttpsServer.class))); + assertThat(server.getAddress().getPort(), is(expectedPort)); + } + + @Test + public void testGivenSslConfigThenHttpsServerCreated() { + final int expectedPort = getPort(); + + when(configuration.ssl()) + .thenReturn(true); + when(configuration.getKeyStoreFilePath()) + .thenReturn(P12_KEYSTORE); + when(configuration.getKeyStorePassword()) + .thenReturn(""); + when(configuration.getPrivateKeyPassword()) + .thenReturn(""); + when(configuration.getServerPort()) + .thenReturn(expectedPort); + + final HttpServer server = httpServerProvider.get(); + + assertThat(server, isA(HttpServer.class)); + assertThat(server, isA(HttpsServer.class)); + assertThat(server.getAddress().getPort(), is(expectedPort)); + + final HttpsServer httpsServer = (HttpsServer) server; + + assertThat(httpsServer.getHttpsConfigurator(), isA(HttpsConfigurator.class)); + + verify(configuration).getKeyStoreFilePath(); + verify(configuration).getKeyStorePassword(); + verify(configuration).getPrivateKeyPassword(); + } + + @Test + public void testGivenPortInUseThenExceptionThrown() throws IOException { + final int port = getPort(); + final InetSocketAddress socketAddress = new InetSocketAddress(port); + + final HttpServer portBlockingServer = HttpServer.create(socketAddress, 0); + portBlockingServer.start(); + + try { + when(configuration.getServerPort()) + .thenReturn(port); + + assertThrows(RuntimeException.class, () -> httpServerProvider.get()); + } catch (final Exception ignored) { + } finally { + portBlockingServer.stop(0); + } + } +} \ No newline at end of file diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandlerTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandlerTest.java index c3fa623f9a..a6f25ff5a6 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandlerTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/ListPipelinesHandlerTest.java @@ -21,7 +21,6 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -29,11 +28,11 @@ class ListPipelinesHandlerTest { @Test public void testGivenNoPipelinesThenResponseWritten() throws IOException { - DataPrepper dataPrepper = mock(DataPrepper.class); - HttpExchange httpExchange = mock(HttpExchange.class); - Headers headers = mock(Headers.class); - OutputStream outputStream = mock(OutputStream.class); - Map transformationPipelines = new HashMap<>(); + final DataPrepper dataPrepper = mock(DataPrepper.class); + final HttpExchange httpExchange = mock(HttpExchange.class); + final Headers headers = mock(Headers.class); + final OutputStream outputStream = mock(OutputStream.class); + final Map transformationPipelines = new HashMap<>(); when(dataPrepper.getTransformationPipelines()) .thenReturn(transformationPipelines); @@ -43,28 +42,28 @@ public void testGivenNoPipelinesThenResponseWritten() throws IOException { when(httpExchange.getResponseBody()) .thenReturn(outputStream); - ListPipelinesHandler handler = new ListPipelinesHandler(dataPrepper); + final ListPipelinesHandler handler = new ListPipelinesHandler(dataPrepper); handler.handle(httpExchange); - verify(headers, times(1)) + verify(headers) .add(eq("Content-Type"), eq("text/plain; charset=UTF-8")); - verify(httpExchange, times(1)) + verify(httpExchange) .sendResponseHeaders(eq(HttpURLConnection.HTTP_OK), anyLong()); - verify(outputStream, times(1)) + verify(outputStream) .write(any(byte[].class)); - verify(outputStream, times(1)) + verify(outputStream) .close(); } @Test public void testGivenPipelinesThenResponseWritten() throws IOException { - DataPrepper dataPrepper = mock(DataPrepper.class); - HttpExchange httpExchange = mock(HttpExchange.class); - Headers headers = mock(Headers.class); - OutputStream outputStream = mock(OutputStream.class); - Pipeline pipeline = mock(Pipeline.class); - Map transformationPipelines = new HashMap<>(); + final DataPrepper dataPrepper = mock(DataPrepper.class); + final HttpExchange httpExchange = mock(HttpExchange.class); + final Headers headers = mock(Headers.class); + final OutputStream outputStream = mock(OutputStream.class); + final Pipeline pipeline = mock(Pipeline.class); + final Map transformationPipelines = new HashMap<>(); transformationPipelines.put("Pipeline A", pipeline); transformationPipelines.put("Pipeline B", pipeline); transformationPipelines.put("Pipeline C", pipeline); @@ -77,34 +76,34 @@ public void testGivenPipelinesThenResponseWritten() throws IOException { when(httpExchange.getResponseBody()) .thenReturn(outputStream); - ListPipelinesHandler handler = new ListPipelinesHandler(dataPrepper); + final ListPipelinesHandler handler = new ListPipelinesHandler(dataPrepper); handler.handle(httpExchange); - verify(headers, times(1)) + verify(headers) .add(eq("Content-Type"), eq("text/plain; charset=UTF-8")); - verify(httpExchange, times(1)) + verify(httpExchange) .sendResponseHeaders(eq(HttpURLConnection.HTTP_OK), anyLong()); - verify(outputStream, times(1)) + verify(outputStream) .write(any(byte[].class)); - verify(outputStream, times(1)) + verify(outputStream) .close(); } @Test public void testGivenExceptionThrownThenErrorResponseWrittern() throws IOException { - HttpExchange httpExchange = mock(HttpExchange.class); - OutputStream outputStream = mock(OutputStream.class); + final HttpExchange httpExchange = mock(HttpExchange.class); + final OutputStream outputStream = mock(OutputStream.class); when(httpExchange.getResponseBody()) .thenReturn(outputStream); - ListPipelinesHandler handler = new ListPipelinesHandler(null); + final ListPipelinesHandler handler = new ListPipelinesHandler(null); handler.handle(httpExchange); - verify(httpExchange, times(1)) + verify(httpExchange) .sendResponseHeaders(eq(HttpURLConnection.HTTP_INTERNAL_ERROR), eq(0L)); - verify(outputStream, times(1)) + verify(outputStream) .close(); } } \ No newline at end of file diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java index 8dcb1b2ba4..fd6eb4753a 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java @@ -9,34 +9,28 @@ import com.amazon.dataprepper.model.configuration.PluginModel; import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.model.plugin.PluginFactory; -import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.pipeline.server.DataPrepperCoreAuthenticationProvider; +import com.amazon.dataprepper.pipeline.server.HttpServerProvider; import com.amazon.dataprepper.pipeline.server.ListPipelinesHandler; +import com.amazon.dataprepper.pipeline.server.PrometheusMetricsHandler; import com.amazon.dataprepper.pipeline.server.ShutdownHandler; import com.sun.net.httpserver.Authenticator; import com.sun.net.httpserver.HttpContext; +import com.sun.net.httpserver.HttpHandler; import com.sun.net.httpserver.HttpServer; -import com.sun.net.httpserver.HttpsConfigurator; -import com.sun.net.httpserver.HttpsServer; +import io.micrometer.prometheus.PrometheusMeterRegistry; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.io.IOException; -import java.net.DatagramSocket; -import java.net.InetSocketAddress; -import java.net.ServerSocket; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Random; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isA; -import static org.hamcrest.Matchers.not; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -47,114 +41,66 @@ @ExtendWith(MockitoExtension.class) class DataPrepperServerConfigurationTest { - private static final Random r = new Random(); - private static final Integer MAX_PORT = 65535; - private static final String P12_KEYSTORE = "src/test/resources/tls/test_keystore.p12"; - - private static boolean available(final int port) { - ServerSocket ss = null; - DatagramSocket ds = null; - try { - ss = new ServerSocket(port); - ss.setReuseAddress(true); - ds = new DatagramSocket(port); - ds.setReuseAddress(true); - return true; - } catch (final IOException ignored) { - } finally { - if (ds != null) { - ds.close(); - } - - if (ss != null) { - try { - ss.close(); - } catch (final IOException ignored) { - } - } - } - - return false; - } + @Mock + private HttpContext context; - private static int getPort() { - final int MAX_RETRIES = 10; - for (int i = 0; i < MAX_RETRIES; i++) { - final int electivePort = r.nextInt(MAX_PORT); - if (available(electivePort)) { - return electivePort; - } - } - throw new RuntimeException("No available port found"); - } + @Mock + private HttpServer httpServer; @Mock - private DataPrepperConfiguration configuration; + private HttpServerProvider httpServerProvider; + + @Mock + private ListPipelinesHandler listPipelinesHandler; private final DataPrepperServerConfiguration serverConfiguration = new DataPrepperServerConfiguration(); @Test - public void testGivenNoSslThenInsecureServerCreated() { - final int expectedPort = getPort(); - when(configuration.ssl()) - .thenReturn(false); - when(configuration.getServerPort()) - .thenReturn(expectedPort); - - final HttpServer server = serverConfiguration.httpServer(configuration); - - assertThat(server, isA(HttpServer.class)); - assertThat(server, not(isA(HttpsServer.class))); - assertThat(server.getAddress().getPort(), is(expectedPort)); + public void testGivenNullPrometheusMeterRegistryAndNullAuthenticatorThenServerIsCreated() { + when(httpServerProvider.get()) + .thenReturn(httpServer); + final HttpServer server = serverConfiguration.httpServer(httpServerProvider, listPipelinesHandler, null, null); + + assertThat(server, is(httpServer)); + verify(server).createContext("/list", listPipelinesHandler); + } @Test - public void testGivenSslConfigThenHttpsServerCreated() { - final int expectedPort = getPort(); + public void testGivenPrometheusMeterRegistryAndNullAuthenticatorThenServerIsCreated() { + final PrometheusMeterRegistry meterRegistry = mock(PrometheusMeterRegistry.class); - when(configuration.ssl()) - .thenReturn(true); - when(configuration.getKeyStoreFilePath()) - .thenReturn(P12_KEYSTORE); - when(configuration.getKeyStorePassword()) - .thenReturn(""); - when(configuration.getPrivateKeyPassword()) - .thenReturn(""); - when(configuration.getServerPort()) - .thenReturn(expectedPort); + when(httpServerProvider.get()) + .thenReturn(httpServer); + when(httpServer.createContext(any(String.class), any(HttpHandler.class))) + .thenReturn(context); - final HttpServer server = serverConfiguration.httpServer(configuration); + final HttpServer server = serverConfiguration.httpServer(httpServerProvider, listPipelinesHandler, meterRegistry, null); - assertThat(server, isA(HttpServer.class)); - assertThat(server, isA(HttpsServer.class)); - assertThat(server.getAddress().getPort(), is(expectedPort)); + assertThat(server, is(httpServer)); + verify(server).createContext(eq("/list"), eq(listPipelinesHandler)); + verify(server).createContext(eq("/metrics/prometheus"), any(PrometheusMetricsHandler.class)); + verify(server).createContext(eq("/metrics/sys"), any(PrometheusMetricsHandler.class)); + verifyNoInteractions(context); + } - final HttpsServer httpsServer = (HttpsServer) server; + @Test + public void testGivenPrometheusMeterRegistryAndAuthenticatorThenServerIsCreated() { + final PrometheusMeterRegistry meterRegistry = mock(PrometheusMeterRegistry.class); + final Authenticator authenticator = mock(Authenticator.class); - assertThat(httpsServer.getHttpsConfigurator(), isA(HttpsConfigurator.class)); + when(httpServerProvider.get()) + .thenReturn(httpServer); + when(httpServer.createContext(any(String.class), any(HttpHandler.class))) + .thenReturn(context); - verify(configuration, times(1)).getKeyStoreFilePath(); - verify(configuration, times(1)).getKeyStorePassword(); - verify(configuration, times(1)).getPrivateKeyPassword(); - } + final HttpServer server = serverConfiguration.httpServer(httpServerProvider, listPipelinesHandler, meterRegistry, authenticator); - @Test - public void testGivenPortInUseThenExceptionThrown() throws IOException { - final int port = getPort(); - final InetSocketAddress socketAddress = new InetSocketAddress(port); - - final HttpServer portBlockingServer = HttpServer.create(socketAddress, 0); - portBlockingServer.start(); - - try { - when(configuration.getServerPort()) - .thenReturn(port); - - assertThrows(RuntimeException.class, () -> serverConfiguration.httpServer(configuration)); - } catch (final Exception ignored) { - } finally { - portBlockingServer.stop(0); - } + assertThat(server, is(httpServer)); + verify(server).createContext(eq("/list"), eq(listPipelinesHandler)); + verify(server).createContext(eq("/metrics/prometheus"), any(PrometheusMetricsHandler.class)); + verify(server).createContext(eq("/metrics/sys"), any(PrometheusMetricsHandler.class)); + verify(context, times(3)).setAuthenticator(eq(authenticator)); } @Test @@ -214,47 +160,32 @@ public void testGivenPluginFactoryAndPluginSettingsThenCreateAuthenticationProvi @Test public void testGivenGetAuthenticatorReturnsValueThenReturnOptionalContainingValue() { final DataPrepperCoreAuthenticationProvider provider = mock(DataPrepperCoreAuthenticationProvider.class); - final Authenticator authenticator = mock(Authenticator.class); + final Authenticator authenticatorMock = mock(Authenticator.class); when(provider.getAuthenticator()) - .thenReturn(authenticator); + .thenReturn(authenticatorMock); - final Optional optional = serverConfiguration.optionalAuthenticator(provider); + final Authenticator authenticator = serverConfiguration.authenticator(provider); - assertThat(optional.isPresent(), is(true)); - assertThat(optional.get(), is(authenticator)); + assertThat(authenticator, is(authenticatorMock)); } @Test public void testGivenValidInputWithNoAuthenticatorThenServerListContextCreated() { final DataPrepper dataPrepper = mock(DataPrepper.class); - final HttpServer server = mock(HttpServer.class); - final HttpContext context = mock(HttpContext.class); - - when(server.createContext(eq("/list"), any(ListPipelinesHandler.class))) - .thenReturn(context); - final ListPipelinesHandler handler = serverConfiguration.listPipelinesHandler(dataPrepper, Optional.empty(), server); + final ListPipelinesHandler handler = serverConfiguration.listPipelinesHandler(dataPrepper); assertThat(handler, isA(ListPipelinesHandler.class)); - verifyNoInteractions(context); } @Test public void testGivenValidInputWithAuthenticatorThenServerListContextCreated() { final DataPrepper dataPrepper = mock(DataPrepper.class); - final Authenticator authenticator = mock(Authenticator.class); - final HttpServer server = mock(HttpServer.class); - final HttpContext context = mock(HttpContext.class); - when(server.createContext(eq("/list"), any(ListPipelinesHandler.class))) - .thenReturn(context); - - final ListPipelinesHandler handler = serverConfiguration.listPipelinesHandler(dataPrepper, Optional.of(authenticator), server); + final ListPipelinesHandler handler = serverConfiguration.listPipelinesHandler(dataPrepper); assertThat(handler, isA(ListPipelinesHandler.class)); - verify(context, times(1)) - .setAuthenticator(eq(authenticator)); } @Test @@ -285,7 +216,7 @@ public void testGivenValidInputWithAuthenticatorThenServerShutdownContextCreated final ShutdownHandler handler = serverConfiguration.shutdownHandler(dataPrepper, Optional.of(authenticator), server); assertThat(handler, isA(ShutdownHandler.class)); - verify(context, times(1)) + verify(context) .setAuthenticator(eq(authenticator)); } } \ No newline at end of file diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java index b9aee70c0b..c6a6b14c35 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java @@ -20,7 +20,6 @@ import static org.hamcrest.Matchers.notNullValue; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -47,15 +46,15 @@ public void testGivenValidServerWhenStartThenShouldCallServerStart() { dataPrepperServer.start(); - verify(server, times(1)).start(); - verify(server, times(1)).getAddress(); - verify(socketAddress, times(1)).getPort(); + verify(server).start(); + verify(server).getAddress(); + verify(socketAddress).getPort(); } @Test public void testGivenValidServerWhenStopThenShouldCallServerStopWithNoDely() { dataPrepperServer.stop(); - verify(server, times(1)).stop(eq(0)); + verify(server).stop(eq(0)); } } From b4a4c1164fd654d7f1756f7e03ba00b235fd15b8 Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Tue, 18 Jan 2022 17:50:00 -0600 Subject: [PATCH 41/43] Added CloudWatchMeterRegistryProvider bean factory method Signed-off-by: Steven Bayer --- .../parser/config/MetricsConfig.java | 21 ++++++- .../CloudWatchMeterRegistryProvider.java | 4 -- .../pipeline/server/HttpServerProvider.java | 4 ++ .../DataPrepperServerConfiguration.java | 25 +++----- .../DataPrepperServerConfigurationTest.java | 59 ++++++++----------- 5 files changed, 57 insertions(+), 56 deletions(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java index 83abb8869d..0d7cfe7685 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/config/MetricsConfig.java @@ -25,10 +25,12 @@ import io.micrometer.prometheus.PrometheusMeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import software.amazon.awssdk.core.exception.SdkClientException; +import javax.annotation.Nullable; import java.util.Collections; import java.util.List; @@ -96,12 +98,29 @@ public MeterRegistry prometheusMeterRegistry(final DataPrepperConfiguration data } } + @Bean + public CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider( + final DataPrepperConfiguration dataPrepperConfiguration + ) { + if (dataPrepperConfiguration.getMetricRegistryTypes().contains(MetricRegistryType.CloudWatch)) { + return new CloudWatchMeterRegistryProvider(); + } + else { + return null; + } + } + @Bean public MeterRegistry cloudWatchMeterRegistry( final DataPrepperConfiguration dataPrepperConfiguration, - final CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider + @Autowired(required = false) @Nullable final CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider ) { if (dataPrepperConfiguration.getMetricRegistryTypes().contains(MetricRegistryType.CloudWatch)) { + if (cloudWatchMeterRegistryProvider == null) { + throw new IllegalStateException( + "configuration required configure cloudwatch meter registry but one could not be configured"); + } + try { final CloudWatchMeterRegistry meterRegistry = cloudWatchMeterRegistryProvider.getCloudWatchMeterRegistry(); configureMetricRegistry(meterRegistry); diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java index 4467f46460..50746c0fbe 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java @@ -12,8 +12,6 @@ import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; -import javax.inject.Inject; -import javax.inject.Named; import java.io.IOException; import java.io.InputStream; import java.util.Properties; @@ -27,14 +25,12 @@ * {@link CloudWatchMeterRegistryProvider} also has a constructor with {@link CloudWatchAsyncClient} that will be used * for communication with Cloudwatch. */ -@Named public class CloudWatchMeterRegistryProvider { private static final String CLOUDWATCH_PROPERTIES = "cloudwatch.properties"; private static final Logger LOG = LoggerFactory.getLogger(CloudWatchMeterRegistryProvider.class); private final CloudWatchMeterRegistry cloudWatchMeterRegistry; - @Inject public CloudWatchMeterRegistryProvider() { this(CLOUDWATCH_PROPERTIES, CloudWatchAsyncClient.create()); } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/HttpServerProvider.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/HttpServerProvider.java index 2943d57f5e..0c94e0b4d9 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/HttpServerProvider.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/HttpServerProvider.java @@ -55,6 +55,10 @@ public void configure(final HttpsParameters params) { return server; } else { + LOG.warn("Creating Data Prepper server without TLS. This is not secure."); + LOG.warn("In order to set up TLS for the Data Prepper server, go here: " + + "https://github.com/opensearch-project/data-prepper/blob/main/docs/configuration.md#server-configuration"); + return HttpServer.create(socketAddress, 0); } } catch (final IOException ex) { diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java index 9782eef395..2c045fc302 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfiguration.java @@ -27,7 +27,6 @@ import javax.annotation.Nullable; import java.util.Collections; -import java.util.Optional; @Configuration public class DataPrepperServerConfiguration { @@ -52,6 +51,7 @@ private void createContext( public HttpServer httpServer( final HttpServerProvider httpServerProvider, final ListPipelinesHandler listPipelinesHandler, + final ShutdownHandler shutdownHandler, @Autowired(required = false) @Nullable final PrometheusMeterRegistry prometheusMeterRegistry, @Autowired(required = false) @Nullable final Authenticator authenticator ) { @@ -59,6 +59,7 @@ public HttpServer httpServer( final HttpServer server = httpServerProvider.get(); createContext(server, listPipelinesHandler, authenticator, "/list"); + createContext(server, shutdownHandler, authenticator, "/shutdown"); if (prometheusMeterRegistry != null) { final PrometheusMetricsHandler prometheusMetricsHandler = new PrometheusMetricsHandler(prometheusMeterRegistry); @@ -75,14 +76,13 @@ private void printInsecurePluginModelWarning() { } @Bean - public PluginSetting pluginSetting(final Optional optionalPluginModel) { - if (optionalPluginModel.isPresent()) { - final PluginModel pluginModel = optionalPluginModel.get(); - final String pluginName = pluginModel.getPluginName(); + public PluginSetting pluginSetting(@Autowired(required = false) final PluginModel authentication) { + if (authentication != null) { + final String pluginName = authentication.getPluginName(); if (pluginName.equals(DataPrepperCoreAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)) { printInsecurePluginModelWarning(); } - return new PluginSetting(pluginName, pluginModel.getPluginSettings()); + return new PluginSetting(pluginName, authentication.getPluginSettings()); } else { printInsecurePluginModelWarning(); @@ -114,16 +114,7 @@ public ListPipelinesHandler listPipelinesHandler(final DataPrepper dataPrepper) } @Bean - public ShutdownHandler shutdownHandler( - final DataPrepper dataPrepper, - final Optional optionalAuthenticator, - final HttpServer server - ) { - final ShutdownHandler shutdownHandler = new ShutdownHandler(dataPrepper); - - final HttpContext context = server.createContext("/shutdown", shutdownHandler); - optionalAuthenticator.ifPresent(context::setAuthenticator); - - return shutdownHandler; + public ShutdownHandler shutdownHandler(final DataPrepper dataPrepper) { + return new ShutdownHandler(dataPrepper); } } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java index fd6eb4753a..c2fa3ceeb6 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/pipeline/server/config/DataPrepperServerConfigurationTest.java @@ -26,7 +26,6 @@ import java.util.HashMap; import java.util.Map; -import java.util.Optional; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; @@ -53,17 +52,20 @@ class DataPrepperServerConfigurationTest { @Mock private ListPipelinesHandler listPipelinesHandler; + @Mock + private ShutdownHandler shutdownHandler; + private final DataPrepperServerConfiguration serverConfiguration = new DataPrepperServerConfiguration(); @Test public void testGivenNullPrometheusMeterRegistryAndNullAuthenticatorThenServerIsCreated() { when(httpServerProvider.get()) .thenReturn(httpServer); - final HttpServer server = serverConfiguration.httpServer(httpServerProvider, listPipelinesHandler, null, null); + final HttpServer server = serverConfiguration.httpServer(httpServerProvider, listPipelinesHandler, shutdownHandler, null, null); assertThat(server, is(httpServer)); verify(server).createContext("/list", listPipelinesHandler); - + verify(server).createContext(eq("/shutdown"), eq(shutdownHandler)); } @Test @@ -75,10 +77,16 @@ public void testGivenPrometheusMeterRegistryAndNullAuthenticatorThenServerIsCrea when(httpServer.createContext(any(String.class), any(HttpHandler.class))) .thenReturn(context); - final HttpServer server = serverConfiguration.httpServer(httpServerProvider, listPipelinesHandler, meterRegistry, null); + final HttpServer server = serverConfiguration.httpServer( + httpServerProvider, + listPipelinesHandler, + shutdownHandler, + meterRegistry, + null); assertThat(server, is(httpServer)); verify(server).createContext(eq("/list"), eq(listPipelinesHandler)); + verify(server).createContext(eq("/shutdown"), eq(shutdownHandler)); verify(server).createContext(eq("/metrics/prometheus"), any(PrometheusMetricsHandler.class)); verify(server).createContext(eq("/metrics/sys"), any(PrometheusMetricsHandler.class)); verifyNoInteractions(context); @@ -94,18 +102,24 @@ public void testGivenPrometheusMeterRegistryAndAuthenticatorThenServerIsCreated( when(httpServer.createContext(any(String.class), any(HttpHandler.class))) .thenReturn(context); - final HttpServer server = serverConfiguration.httpServer(httpServerProvider, listPipelinesHandler, meterRegistry, authenticator); + final HttpServer server = serverConfiguration.httpServer( + httpServerProvider, + listPipelinesHandler, + shutdownHandler, + meterRegistry, + authenticator); assertThat(server, is(httpServer)); verify(server).createContext(eq("/list"), eq(listPipelinesHandler)); + verify(server).createContext(eq("/shutdown"), eq(shutdownHandler)); verify(server).createContext(eq("/metrics/prometheus"), any(PrometheusMetricsHandler.class)); verify(server).createContext(eq("/metrics/sys"), any(PrometheusMetricsHandler.class)); - verify(context, times(3)).setAuthenticator(eq(authenticator)); + verify(context, times(4)).setAuthenticator(eq(authenticator)); } @Test public void testGivingNoConfigThenCreateInsecureSettings() { - final PluginSetting pluginSetting = serverConfiguration.pluginSetting(Optional.empty()); + final PluginSetting pluginSetting = serverConfiguration.pluginSetting(null); assertThat(pluginSetting.getName(), is("unauthenticated")); assertThat(pluginSetting.getSettings().isEmpty(), is(true)); @@ -118,7 +132,7 @@ public void testGivingInsecureConfigThenCreateInsecureSettings() { when(pluginModel.getPluginName()) .thenReturn("unauthenticated"); - final PluginSetting pluginSetting = serverConfiguration.pluginSetting(Optional.of(pluginModel)); + final PluginSetting pluginSetting = serverConfiguration.pluginSetting(pluginModel); assertThat(pluginSetting.getName(), is("unauthenticated")); assertThat(pluginSetting.getSettings().isEmpty(), is(true)); @@ -135,7 +149,7 @@ public void testGivingSecureConfigThenCreateInsecureSettings() { when(pluginModel.getPluginSettings()) .thenReturn(settings); - final PluginSetting pluginSetting = serverConfiguration.pluginSetting(Optional.of(pluginModel)); + final PluginSetting pluginSetting = serverConfiguration.pluginSetting(pluginModel); assertThat(pluginSetting.getName(), is("super secure plugin")); assertThat(pluginSetting.getSettings(), is(settings)); @@ -189,34 +203,11 @@ public void testGivenValidInputWithAuthenticatorThenServerListContextCreated() { } @Test - public void testGivenValidInputWithNoAuthenticatorThenServerShutdownContextCreated() { - final DataPrepper dataPrepper = mock(DataPrepper.class); - final HttpServer server = mock(HttpServer.class); - final HttpContext context = mock(HttpContext.class); - - when(server.createContext(eq("/shutdown"), any(ShutdownHandler.class))) - .thenReturn(context); - - final ShutdownHandler handler = serverConfiguration.shutdownHandler(dataPrepper, Optional.empty(), server); - - assertThat(handler, isA(ShutdownHandler.class)); - verifyNoInteractions(context); - } - - @Test - public void testGivenValidInputWithAuthenticatorThenServerShutdownContextCreated() { + public void testShutdownHandlerIsCreated() { final DataPrepper dataPrepper = mock(DataPrepper.class); - final Authenticator authenticator = mock(Authenticator.class); - final HttpServer server = mock(HttpServer.class); - final HttpContext context = mock(HttpContext.class); - - when(server.createContext(eq("/shutdown"), any(ShutdownHandler.class))) - .thenReturn(context); - final ShutdownHandler handler = serverConfiguration.shutdownHandler(dataPrepper, Optional.of(authenticator), server); + final ShutdownHandler handler = serverConfiguration.shutdownHandler(dataPrepper); assertThat(handler, isA(ShutdownHandler.class)); - verify(context) - .setAuthenticator(eq(authenticator)); } } \ No newline at end of file From 6b5fc42fed4980d7c08d43bc74820bc30561880a Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Wed, 19 Jan 2022 09:45:03 -0600 Subject: [PATCH 42/43] remove unused import Signed-off-by: Steven Bayer --- data-prepper-core/build.gradle | 1 - .../java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index c604603901..6622d2c33d 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -19,7 +19,6 @@ dependencies { testImplementation project(':data-prepper-plugins:common').sourceSets.test.output implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' - implementation "javax.validation:validation-api:2.0.1.Final" implementation "org.reflections:reflections:0.10.2" implementation 'io.micrometer:micrometer-core' implementation 'io.micrometer:micrometer-registry-prometheus' diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java index c8277e96ea..445fad3b54 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java @@ -102,6 +102,7 @@ private PluginArgumentsContext getConstructionContext(final PluginSetting pl return new PluginArgumentsContext.Builder() .withPluginSetting(pluginSetting) .withPluginConfiguration(configuration) + .withPluginSetting(pluginSetting) .withPluginFactory(this) .build(); } From 90d9e804a5cf699b7213f15004db45b2c777f02e Mon Sep 17 00:00:00 2001 From: Steven Bayer Date: Fri, 21 Jan 2022 14:57:03 -0600 Subject: [PATCH 43/43] Removed duplicated call to withPluginSetting Signed-off-by: Steven Bayer --- .../java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java | 1 - 1 file changed, 1 deletion(-) diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java index 445fad3b54..c8277e96ea 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/plugin/DefaultPluginFactory.java @@ -102,7 +102,6 @@ private PluginArgumentsContext getConstructionContext(final PluginSetting pl return new PluginArgumentsContext.Builder() .withPluginSetting(pluginSetting) .withPluginConfiguration(configuration) - .withPluginSetting(pluginSetting) .withPluginFactory(this) .build(); }