From bd9b1802a7379d013780885369c65c2caa22c254 Mon Sep 17 00:00:00 2001 From: Vamsi Manohar Date: Thu, 8 Jun 2023 18:37:05 -0700 Subject: [PATCH] Moved auth interceptors to common module (#1703) --- build.gradle | 7 +- common/build.gradle | 21 +++ .../AwsSigningInterceptor.java | 2 +- .../BasicAuthenticationInterceptor.java | 2 +- .../sql/common/setting/Settings.java | 7 +- .../AwsSigningInterceptorTest.java | 15 +- .../BasicAuthenticationInterceptorTest.java | 11 +- .../settings/DataSourceSettings.java | 26 --- docs/user/ppl/admin/datasources.rst | 11 ++ integ-test/build.gradle | 2 - .../sql/datasource/DataSourceAPIsIT.java | 21 ++- .../executor/AsyncRestExecutorTest.java | 2 + .../unittest/LocalClusterStateTest.java | 2 + .../unittest/planner/QueryPlannerTest.java | 4 +- .../setting/OpenSearchSettings.java | 53 ++++++ .../setting/OpenSearchSettingsTest.java | 42 +++++ plugin/build.gradle | 2 - .../org/opensearch/sql/plugin/SQLPlugin.java | 8 +- prometheus/build.gradle | 11 +- .../storage/PrometheusStorageFactory.java | 54 ++++-- .../storage/PrometheusStorageFactoryTest.java | 157 +++++++++++++++--- 21 files changed, 362 insertions(+), 98 deletions(-) rename {prometheus/src/main/java/org/opensearch/sql/prometheus => common/src/main/java/org/opensearch/sql/common}/authinterceptors/AwsSigningInterceptor.java (97%) rename {prometheus/src/main/java/org/opensearch/sql/prometheus => common/src/main/java/org/opensearch/sql/common}/authinterceptors/BasicAuthenticationInterceptor.java (93%) rename {prometheus/src/test/java/org/opensearch/sql/prometheus => common/src/test/java/org/opensearch/sql/common}/authinterceptors/AwsSigningInterceptorTest.java (87%) rename {prometheus/src/test/java/org/opensearch/sql/prometheus => common/src/test/java/org/opensearch/sql/common}/authinterceptors/BasicAuthenticationInterceptorTest.java (83%) delete mode 100644 datasources/src/main/java/org/opensearch/sql/datasources/settings/DataSourceSettings.java diff --git a/build.gradle b/build.gradle index ec4ca1e3e5..052204ef00 100644 --- a/build.gradle +++ b/build.gradle @@ -76,6 +76,7 @@ repositories { mavenLocal() maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available + maven { url 'https://jitpack.io' } } allprojects { @@ -91,6 +92,10 @@ allprojects { plugins.withId('java') { sourceCompatibility = targetCompatibility = "11" } + configurations.all { + resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" + resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" + } } subprojects { @@ -98,8 +103,8 @@ subprojects { mavenLocal() maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() - // todo. remove this when lucene 9.4.0 is released maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/" } + maven { url 'https://jitpack.io' } } } diff --git a/common/build.gradle b/common/build.gradle index 369a649cde..992135e06d 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -36,9 +36,30 @@ dependencies { api group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' api group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1' api group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' + api group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3' + implementation 'com.github.babbel:okhttp-aws-signer:1.0.2' + api group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.12.1' + api group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: '1.12.1' testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.9.1' testImplementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') + testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.12.4' + testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.12.4' + testImplementation group: 'com.squareup.okhttp3', name: 'mockwebserver', version: '4.9.3' } + + +configurations.all { + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.9.3" + resolutionStrategy.force "org.apache.httpcomponents:httpcore:4.4.13" + resolutionStrategy.force "joda-time:joda-time:2.10.12" + resolutionStrategy.force "org.slf4j:slf4j-api:1.7.36" +} \ No newline at end of file diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptor.java b/common/src/main/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptor.java similarity index 97% rename from prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptor.java rename to common/src/main/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptor.java index 56e66431fd..6c65c69c31 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptor.java +++ b/common/src/main/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptor.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.prometheus.authinterceptors; +package org.opensearch.sql.common.authinterceptors; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptor.java b/common/src/main/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptor.java similarity index 93% rename from prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptor.java rename to common/src/main/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptor.java index 6151018567..34634d1580 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptor.java +++ b/common/src/main/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptor.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.prometheus.authinterceptors; +package org.opensearch.sql.common.authinterceptors; import java.io.IOException; import lombok.NonNull; diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index bb68296626..3b0eba157d 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -39,8 +39,13 @@ public enum Key { */ QUERY_MEMORY_LIMIT("plugins.query.memory_limit"), QUERY_SIZE_LIMIT("plugins.query.size_limit"), + ENCYRPTION_MASTER_KEY("plugins.query.datasources.encryption.masterkey"), + DATASOURCES_URI_ALLOWHOSTS("plugins.query.datasources.uri.allowhosts"), + METRICS_ROLLING_WINDOW("plugins.query.metrics.rolling_window"), - METRICS_ROLLING_INTERVAL("plugins.query.metrics.rolling_interval"); + METRICS_ROLLING_INTERVAL("plugins.query.metrics.rolling_interval"), + + CLUSTER_NAME("cluster.name"); @Getter private final String keyValue; diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptorTest.java b/common/src/test/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptorTest.java similarity index 87% rename from prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptorTest.java rename to common/src/test/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptorTest.java index 5d5471edc0..894f3974ce 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptorTest.java +++ b/common/src/test/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptorTest.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.prometheus.authinterceptors; +package org.opensearch.sql.common.authinterceptors; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -15,7 +15,6 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; -import com.amazonaws.auth.STSSessionCredentialsProvider; import lombok.SneakyThrows; import okhttp3.Interceptor; import okhttp3.Request; @@ -25,7 +24,9 @@ import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.common.authinterceptors.AwsSigningInterceptor; @ExtendWith(MockitoExtension.class) public class AwsSigningInterceptorTest { @@ -54,7 +55,7 @@ void testConstructors() { @Test @SneakyThrows void testIntercept() { - when(chain.request()).thenReturn(new Request.Builder() + Mockito.when(chain.request()).thenReturn(new Request.Builder() .url("http://localhost:9090") .build()); AwsSigningInterceptor awsSigningInterceptor @@ -62,7 +63,7 @@ void testIntercept() { getStaticAWSCredentialsProvider("testAccessKey", "testSecretKey"), "us-east-1", "aps"); awsSigningInterceptor.intercept(chain); - verify(chain).proceed(requestArgumentCaptor.capture()); + Mockito.verify(chain).proceed(requestArgumentCaptor.capture()); Request request = requestArgumentCaptor.getValue(); Assertions.assertNotNull(request.headers("Authorization")); Assertions.assertNotNull(request.headers("x-amz-date")); @@ -73,16 +74,16 @@ void testIntercept() { @Test @SneakyThrows void testSTSCredentialsProviderInterceptor() { - when(chain.request()).thenReturn(new Request.Builder() + Mockito.when(chain.request()).thenReturn(new Request.Builder() .url("http://localhost:9090") .build()); - when(stsAssumeRoleSessionCredentialsProvider.getCredentials()) + Mockito.when(stsAssumeRoleSessionCredentialsProvider.getCredentials()) .thenReturn(getAWSSessionCredentials()); AwsSigningInterceptor awsSigningInterceptor = new AwsSigningInterceptor(stsAssumeRoleSessionCredentialsProvider, "us-east-1", "aps"); awsSigningInterceptor.intercept(chain); - verify(chain).proceed(requestArgumentCaptor.capture()); + Mockito.verify(chain).proceed(requestArgumentCaptor.capture()); Request request = requestArgumentCaptor.getValue(); Assertions.assertNotNull(request.headers("Authorization")); Assertions.assertNotNull(request.headers("x-amz-date")); diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptorTest.java b/common/src/test/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptorTest.java similarity index 83% rename from prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptorTest.java rename to common/src/test/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptorTest.java index b5b5acd457..596894da6d 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptorTest.java +++ b/common/src/test/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptorTest.java @@ -5,10 +5,7 @@ * */ -package org.opensearch.sql.prometheus.authinterceptors; - -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +package org.opensearch.sql.common.authinterceptors; import java.util.Collections; import lombok.SneakyThrows; @@ -21,7 +18,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; -import org.mockito.internal.matchers.Null; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -45,13 +42,13 @@ void testConstructors() { @Test @SneakyThrows void testIntercept() { - when(chain.request()).thenReturn(new Request.Builder() + Mockito.when(chain.request()).thenReturn(new Request.Builder() .url("http://localhost:9090") .build()); BasicAuthenticationInterceptor basicAuthenticationInterceptor = new BasicAuthenticationInterceptor("testAdmin", "testPassword"); basicAuthenticationInterceptor.intercept(chain); - verify(chain).proceed(requestArgumentCaptor.capture()); + Mockito.verify(chain).proceed(requestArgumentCaptor.capture()); Request request = requestArgumentCaptor.getValue(); Assertions.assertEquals( Collections.singletonList(Credentials.basic("testAdmin", "testPassword")), diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/settings/DataSourceSettings.java b/datasources/src/main/java/org/opensearch/sql/datasources/settings/DataSourceSettings.java deleted file mode 100644 index 3a92181636..0000000000 --- a/datasources/src/main/java/org/opensearch/sql/datasources/settings/DataSourceSettings.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.datasources.settings; - -import java.io.InputStream; -import org.opensearch.common.settings.SecureSetting; -import org.opensearch.common.settings.Setting; - -public class DataSourceSettings { - - // we are keeping this to not break upgrades if the config is already present. - // This will be completely removed in 3.0. - public static final Setting DATASOURCE_CONFIG = SecureSetting.secureFile( - "plugins.query.federation.datasources.config", - null, - Setting.Property.Deprecated); - - public static final Setting DATASOURCE_MASTER_SECRET_KEY = Setting.simpleString( - "plugins.query.datasources.encryption.masterkey", - "0000000000000000", - Setting.Property.NodeScope, - Setting.Property.Dynamic); -} diff --git a/docs/user/ppl/admin/datasources.rst b/docs/user/ppl/admin/datasources.rst index ea8190d50f..6a5871a9e9 100644 --- a/docs/user/ppl/admin/datasources.rst +++ b/docs/user/ppl/admin/datasources.rst @@ -136,6 +136,17 @@ Master Key config for encrypting credential information # Print the master key print("Generated master key:", master_key) +Datasource Allow Hosts Config +======================================================== +* In the OpenSearch configuration file (opensearch.yml), the parameter "plugins.query.datasources.uri.allowhosts" can be utilized to control the permitted hosts within the datasource URI configuration. +* By default, the value is set to `.*`, which allows any domain to be accepted. +* For instance, if you set the value to `dummy.*.com`, the following URIs are some examples that would be allowed in the datasource configuration: + - https://dummy.prometheus.com:9080 + - http://dummy.prometheus.com + +Note: The mentioned URIs are just examples to illustrate the concept. + + Using a datasource in PPL command ==================================== Datasource is referred in source command as show in the code block below. diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 88a7055fdc..35c030d909 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -78,8 +78,6 @@ configurations.all { resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" - resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" - resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.9.3" resolutionStrategy.force "org.apache.httpcomponents:httpcore:4.4.13" resolutionStrategy.force "joda-time:joda-time:2.10.12" diff --git a/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java b/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java index 0dbc03d5a2..ac6949e77e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java @@ -22,6 +22,7 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.opensearch.action.update.UpdateRequest; import org.opensearch.client.Request; import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; @@ -89,7 +90,7 @@ public void updateDataSourceAPITest() { //update datasource DataSourceMetadata updateDSM = new DataSourceMetadata("update_prometheus", DataSourceType.PROMETHEUS, - ImmutableList.of(), ImmutableMap.of("prometheus.uri", "https://randomtest:9090")); + ImmutableList.of(), ImmutableMap.of("prometheus.uri", "https://randomtest.com:9090")); Request updateRequest = getUpdateDataSourceRequest(updateDSM); Response updateResponse = client().performRequest(updateRequest); Assert.assertEquals(200, updateResponse.getStatusLine().getStatusCode()); @@ -99,6 +100,22 @@ public void updateDataSourceAPITest() { //Datasource is not immediately updated. so introducing a sleep of 2s. Thread.sleep(2000); + //update datasource with invalid URI + updateDSM = + new DataSourceMetadata("update_prometheus", DataSourceType.PROMETHEUS, + ImmutableList.of(), ImmutableMap.of("prometheus.uri", "https://randomtest:9090")); + final Request illFormedUpdateRequest + = getUpdateDataSourceRequest(updateDSM); + ResponseException updateResponseException + = Assert.assertThrows(ResponseException.class, () -> client().performRequest(illFormedUpdateRequest)); + Assert.assertEquals(400, updateResponseException.getResponse().getStatusLine().getStatusCode()); + updateResponseString = getResponseBody(updateResponseException.getResponse()); + JsonObject errorMessage = new Gson().fromJson(updateResponseString, JsonObject.class); + Assert.assertEquals("Invalid hostname in the uri: https://randomtest:9090", + errorMessage.get("error").getAsJsonObject().get("details").getAsString()); + + Thread.sleep(2000); + //get datasource to validate the modification. //get datasource Request getRequest = getFetchDataSourceRequest("update_prometheus"); @@ -107,7 +124,7 @@ public void updateDataSourceAPITest() { String getResponseString = getResponseBody(getResponse); DataSourceMetadata dataSourceMetadata = new Gson().fromJson(getResponseString, DataSourceMetadata.class); - Assert.assertEquals("https://randomtest:9090", + Assert.assertEquals("https://randomtest.com:9090", dataSourceMetadata.getProperties().get("prometheus.uri")); } diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/executor/AsyncRestExecutorTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/executor/AsyncRestExecutorTest.java index 24d1814067..b26e171ce7 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/executor/AsyncRestExecutorTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/executor/AsyncRestExecutorTest.java @@ -25,6 +25,7 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterName; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.rest.RestChannel; import org.opensearch.sql.legacy.esdomain.LocalClusterState; @@ -62,6 +63,7 @@ public class AsyncRestExecutorTest { public void setUp() { when(client.threadPool()).thenReturn(mock(ThreadPool.class)); when(action.getSqlRequest()).thenReturn(SqlRequest.NULL); + when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); OpenSearchSettings settings = spy(new OpenSearchSettings(clusterSettings)); doReturn(emptyList()).when(settings).getSettings(); diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java index b23b24413a..4149fd8328 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java @@ -26,6 +26,7 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterStateListener; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; @@ -181,6 +182,7 @@ public void getMappingFromCache() throws IOException { @Test public void getDefaultValueForQuerySlowLog() { + when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); OpenSearchSettings settings = new OpenSearchSettings(clusterSettings); assertEquals(Integer.valueOf(2), settings.getSettingValue(Settings.Key.SQL_SLOWLOG)); } diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java index 98e067a68a..a894ccedc2 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java @@ -38,6 +38,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchScrollRequestBuilder; import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterName; import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -105,8 +106,9 @@ public static void initLogger() { @Before public void init() { MockitoAnnotations.initMocks(this); - + when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); OpenSearchSettings settings = spy(new OpenSearchSettings(clusterSettings)); + // Force return empty list to avoid ClusterSettings be invoked which is a final class and hard to mock. // In this case, default value in Setting will be returned all the time. doReturn(emptyList()).when(settings).getSettings(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index accd356041..671f4113be 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -7,17 +7,21 @@ package org.opensearch.sql.opensearch.setting; import static org.opensearch.common.settings.Settings.EMPTY; +import static org.opensearch.sql.common.setting.Settings.Key.ENCYRPTION_MASTER_KEY; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.io.InputStream; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; +import org.opensearch.cluster.ClusterName; import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.SecureSetting; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.MemorySizeValue; import org.opensearch.sql.common.setting.LegacySettings; @@ -98,6 +102,25 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + // we are keeping this to not break upgrades if the config is already present. + // This will be completely removed in 3.0. + public static final Setting DATASOURCE_CONFIG = SecureSetting.secureFile( + "plugins.query.federation.datasources.config", + null, + Setting.Property.Deprecated); + + public static final Setting DATASOURCE_MASTER_SECRET_KEY = Setting.simpleString( + ENCYRPTION_MASTER_KEY.getKeyValue(), + "0000000000000000", + Setting.Property.NodeScope, + Setting.Property.Final); + + public static final Setting DATASOURCE_URI_ALLOW_HOSTS = Setting.simpleString( + Key.DATASOURCES_URI_ALLOWHOSTS.getKeyValue(), + ".*", + Setting.Property.NodeScope, + Setting.Property.Dynamic); + /** * Construct OpenSearchSetting. * The OpenSearchSetting must be singleton. @@ -123,6 +146,10 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { METRICS_ROLLING_WINDOW_SETTING, new Updater(Key.METRICS_ROLLING_WINDOW)); register(settingBuilder, clusterSettings, Key.METRICS_ROLLING_INTERVAL, METRICS_ROLLING_INTERVAL_SETTING, new Updater(Key.METRICS_ROLLING_INTERVAL)); + register(settingBuilder, clusterSettings, Key.DATASOURCES_URI_ALLOWHOSTS, + DATASOURCE_URI_ALLOW_HOSTS, new Updater(Key.DATASOURCES_URI_ALLOWHOSTS)); + registerNonDynamicSettings(settingBuilder, clusterSettings, Key.CLUSTER_NAME, + ClusterName.CLUSTER_NAME_SETTING); defaultSettings = settingBuilder.build(); } @@ -139,11 +166,26 @@ private void register(ImmutableMap.Builder> settingBuilder, ClusterSettings clusterSettings, Settings.Key key, Setting setting, Consumer updater) { + if (clusterSettings.get(setting) != null) { + latestSettings.put(key, clusterSettings.get(setting)); + } settingBuilder.put(key, setting); clusterSettings .addSettingsUpdateConsumer(setting, updater); } + /** + * Register Non Dynamic Settings without consumer. + */ + private void registerNonDynamicSettings( + ImmutableMap.Builder> settingBuilder, + ClusterSettings clusterSettings, Settings.Key key, + Setting setting) { + settingBuilder.put(key, setting); + latestSettings.put(key, clusterSettings.get(setting)); + } + + /** * Add the inner class only for UT coverage purpose. * Lambda could be much elegant solution. But which is hard to test. @@ -174,6 +216,17 @@ public static List> pluginSettings() { .add(QUERY_SIZE_LIMIT_SETTING) .add(METRICS_ROLLING_WINDOW_SETTING) .add(METRICS_ROLLING_INTERVAL_SETTING) + .add(DATASOURCE_URI_ALLOW_HOSTS) + .build(); + } + + /** + * Init Non Dynamic Plugin Settings. + */ + public static List> pluginNonDynamicSettings() { + return new ImmutableList.Builder>() + .add(DATASOURCE_MASTER_SECRET_KEY) + .add(DATASOURCE_CONFIG) .build(); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index fb97065099..923021f501 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -10,14 +10,23 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.AdditionalMatchers.not; +import static org.mockito.AdditionalMatchers.or; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; import static org.opensearch.common.unit.TimeValue.timeValueMinutes; +import static org.opensearch.sql.opensearch.setting.LegacyOpenDistroSettings.PPL_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.LegacyOpenDistroSettings.legacySettings; import java.util.List; +import java.util.Set; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.AdditionalMatchers; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.cluster.ClusterName; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.ByteSizeValue; @@ -33,12 +42,29 @@ class OpenSearchSettingsTest { @Test void getSettingValue() { + when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); + when(clusterSettings.get(not((eq(ClusterName.CLUSTER_NAME_SETTING))))) + .thenReturn(null); OpenSearchSettings settings = new OpenSearchSettings(clusterSettings); ByteSizeValue sizeValue = settings.getSettingValue(Settings.Key.QUERY_MEMORY_LIMIT); assertNotNull(sizeValue); } + @Test + void getSettingValueWithPresetValuesInYml() { + when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); + when(clusterSettings + .get((Setting) OpenSearchSettings.QUERY_MEMORY_LIMIT_SETTING)) + .thenReturn(new ByteSizeValue(20)); + when(clusterSettings.get(not(or(eq(ClusterName.CLUSTER_NAME_SETTING), + eq((Setting) OpenSearchSettings.QUERY_MEMORY_LIMIT_SETTING))))) + .thenReturn(null); + OpenSearchSettings settings = new OpenSearchSettings(clusterSettings); + ByteSizeValue sizeValue = settings.getSettingValue(Settings.Key.QUERY_MEMORY_LIMIT); + assertEquals(sizeValue, new ByteSizeValue(20)); + } + @Test void pluginSettings() { List> settings = OpenSearchSettings.pluginSettings(); @@ -46,14 +72,27 @@ void pluginSettings() { assertFalse(settings.isEmpty()); } + @Test + void pluginNonDynamicSettings() { + List> settings = OpenSearchSettings.pluginNonDynamicSettings(); + + assertFalse(settings.isEmpty()); + } + @Test void getSettings() { + when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); + when(clusterSettings.get(not((eq(ClusterName.CLUSTER_NAME_SETTING))))) + .thenReturn(null); OpenSearchSettings settings = new OpenSearchSettings(clusterSettings); assertFalse(settings.getSettings().isEmpty()); } @Test void update() { + when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); + when(clusterSettings.get(not((eq(ClusterName.CLUSTER_NAME_SETTING))))) + .thenReturn(null); OpenSearchSettings settings = new OpenSearchSettings(clusterSettings); ByteSizeValue oldValue = settings.getSettingValue(Settings.Key.QUERY_MEMORY_LIMIT); OpenSearchSettings.Updater updater = @@ -67,6 +106,9 @@ void update() { @Test void settingsFallback() { + when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); + when(clusterSettings.get(not((eq(ClusterName.CLUSTER_NAME_SETTING))))) + .thenReturn(null); OpenSearchSettings settings = new OpenSearchSettings(clusterSettings); assertEquals( settings.getSettingValue(Settings.Key.SQL_ENABLED), diff --git a/plugin/build.gradle b/plugin/build.gradle index 4a6c175d61..807e015597 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -102,8 +102,6 @@ configurations.all { resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" - resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" - resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.9.3" resolutionStrategy.force "joda-time:joda-time:2.10.12" resolutionStrategy.force "org.slf4j:slf4j-api:1.7.36" diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index fcb66e2e43..36986c9afc 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -57,7 +57,6 @@ import org.opensearch.sql.datasources.rest.RestDataSourceQueryAction; import org.opensearch.sql.datasources.service.DataSourceMetadataStorage; import org.opensearch.sql.datasources.service.DataSourceServiceImpl; -import org.opensearch.sql.datasources.settings.DataSourceSettings; import org.opensearch.sql.datasources.storage.OpenSearchDataSourceMetadataStorage; import org.opensearch.sql.datasources.transport.TransportCreateDataSourceAction; import org.opensearch.sql.datasources.transport.TransportDeleteDataSourceAction; @@ -200,8 +199,7 @@ public List> getSettings() { return new ImmutableList.Builder>() .addAll(LegacyOpenDistroSettings.legacySettings()) .addAll(OpenSearchSettings.pluginSettings()) - .add(DataSourceSettings.DATASOURCE_CONFIG) - .add(DataSourceSettings.DATASOURCE_MASTER_SECRET_KEY) + .addAll(OpenSearchSettings.pluginNonDynamicSettings()) .build(); } @@ -211,7 +209,7 @@ public ScriptEngine getScriptEngine(Settings settings, Collection() .add(new OpenSearchDataSourceFactory( new OpenSearchNodeClient(this.client), pluginSettings)) - .add(new PrometheusStorageFactory()) + .add(new PrometheusStorageFactory(pluginSettings)) .build(), dataSourceMetadataStorage, dataSourceUserAuthorizationHelper); diff --git a/prometheus/build.gradle b/prometheus/build.gradle index f6b97e66d3..e98dfd83e4 100644 --- a/prometheus/build.gradle +++ b/prometheus/build.gradle @@ -11,21 +11,20 @@ plugins { repositories { mavenCentral() - maven { url 'https://jitpack.io' } } dependencies { api project(':core') implementation project(':datasources') + + implementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation "io.github.resilience4j:resilience4j-retry:1.5.0" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${versions.jackson_databind}" implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${versions.jackson}" - implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3' - implementation 'com.github.babbel:okhttp-aws-signer:1.0.2' - implementation group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.12.1' - implementation group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: '1.12.1' implementation group: 'org.json', name: 'json', version: '20230227' + // https://mvnrepository.com/artifact/commons-validator/commons-validator + implementation group: 'commons-validator', name: 'commons-validator', version: '1.7' testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' @@ -43,8 +42,6 @@ test { } configurations.all { - resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" - resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" resolutionStrategy.force 'junit:junit:4.13.2' } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java index 4a0f52f4a5..b3ecd25af3 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java @@ -17,18 +17,24 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import lombok.RequiredArgsConstructor; import okhttp3.OkHttpClient; +import org.apache.commons.validator.routines.DomainValidator; +import org.opensearch.sql.common.authinterceptors.AwsSigningInterceptor; +import org.opensearch.sql.common.authinterceptors.BasicAuthenticationInterceptor; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; import org.opensearch.sql.datasources.auth.AuthenticationType; -import org.opensearch.sql.prometheus.authinterceptors.AwsSigningInterceptor; -import org.opensearch.sql.prometheus.authinterceptors.BasicAuthenticationInterceptor; import org.opensearch.sql.prometheus.client.PrometheusClient; import org.opensearch.sql.prometheus.client.PrometheusClientImpl; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.sql.storage.StorageEngine; +@RequiredArgsConstructor public class PrometheusStorageFactory implements DataSourceFactory { public static final String URI = "prometheus.uri"; @@ -38,9 +44,10 @@ public class PrometheusStorageFactory implements DataSourceFactory { public static final String REGION = "prometheus.auth.region"; public static final String ACCESS_KEY = "prometheus.auth.access_key"; public static final String SECRET_KEY = "prometheus.auth.secret_key"; - private static final Integer MAX_LENGTH_FOR_CONFIG_PROPERTY = 1000; + private final Settings settings; + @Override public DataSourceType getDataSourceType() { return DataSourceType.PROMETHEUS; @@ -51,36 +58,39 @@ public DataSource createDataSource(DataSourceMetadata metadata) { return new DataSource( metadata.getName(), DataSourceType.PROMETHEUS, - getStorageEngine(metadata.getName(), metadata.getProperties())); + getStorageEngine(metadata.getProperties())); } - private void validateDataSourceConfigProperties(Map dataSourceMetadataConfig) { + //Need to refactor to a separate Validator class. + private void validateDataSourceConfigProperties(Map dataSourceMetadataConfig) + throws URISyntaxException { if (dataSourceMetadataConfig.get(AUTH_TYPE) != null) { AuthenticationType authenticationType = AuthenticationType.get(dataSourceMetadataConfig.get(AUTH_TYPE)); if (AuthenticationType.BASICAUTH.equals(authenticationType)) { - validateFields(dataSourceMetadataConfig, Set.of(URI, USERNAME, PASSWORD)); + validateMissingFields(dataSourceMetadataConfig, Set.of(URI, USERNAME, PASSWORD)); } else if (AuthenticationType.AWSSIGV4AUTH.equals(authenticationType)) { - validateFields(dataSourceMetadataConfig, Set.of(URI, ACCESS_KEY, SECRET_KEY, + validateMissingFields(dataSourceMetadataConfig, Set.of(URI, ACCESS_KEY, SECRET_KEY, REGION)); } } else { - validateFields(dataSourceMetadataConfig, Set.of(URI)); + validateMissingFields(dataSourceMetadataConfig, Set.of(URI)); } + validateURI(dataSourceMetadataConfig); } - StorageEngine getStorageEngine(String catalogName, Map requiredConfig) { - validateDataSourceConfigProperties(requiredConfig); + StorageEngine getStorageEngine(Map requiredConfig) { PrometheusClient prometheusClient; prometheusClient = AccessController.doPrivileged((PrivilegedAction) () -> { try { + validateDataSourceConfigProperties(requiredConfig); return new PrometheusClientImpl(getHttpClient(requiredConfig), new URI(requiredConfig.get(URI))); } catch (URISyntaxException e) { - throw new RuntimeException( - String.format("Prometheus Client creation failed due to: %s", e.getMessage())); + throw new IllegalArgumentException( + String.format("Invalid URI in prometheus properties: %s", e.getMessage())); } }); return new PrometheusStorageEngine(prometheusClient); @@ -110,7 +120,7 @@ private OkHttpClient getHttpClient(Map config) { return okHttpClient.build(); } - private void validateFields(Map config, Set fields) { + private void validateMissingFields(Map config, Set fields) { Set missingFields = new HashSet<>(); Set invalidLengthFields = new HashSet<>(); for (String field : fields) { @@ -135,5 +145,23 @@ private void validateFields(Map config, Set fields) { } } + private void validateURI(Map config) throws URISyntaxException { + URI uri = new URI(config.get(URI)); + String host = uri.getHost(); + if (host == null || (!(DomainValidator.getInstance().isValid(host) + || DomainValidator.getInstance().isValidLocalTld(host)))) { + throw new IllegalArgumentException( + String.format("Invalid hostname in the uri: %s", config.get(URI))); + } else { + Pattern allowHostsPattern = + Pattern.compile(settings.getSettingValue(Settings.Key.DATASOURCES_URI_ALLOWHOSTS)); + Matcher matcher = allowHostsPattern.matcher(host); + if (!matcher.matches()) { + throw new IllegalArgumentException( + String.format("Disallowed hostname in the uri: %s. Validate with %s config", + config.get(URI), Settings.Key.DATASOURCES_URI_ALLOWHOSTS.getKeyValue())); + } + } + } } diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactoryTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactoryTest.java index 36f7e5b5f1..d6a934a015 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactoryTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactoryTest.java @@ -7,13 +7,18 @@ package org.opensearch.sql.prometheus.storage; +import static org.mockito.Mockito.when; + import java.util.HashMap; import lombok.SneakyThrows; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.cluster.ClusterName; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; @@ -22,9 +27,12 @@ @ExtendWith(MockitoExtension.class) public class PrometheusStorageFactoryTest { + @Mock + private Settings settings; + @Test void testGetConnectorType() { - PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(settings); Assertions.assertEquals( DataSourceType.PROMETHEUS, prometheusStorageFactory.getDataSourceType()); } @@ -32,29 +40,31 @@ void testGetConnectorType() { @Test @SneakyThrows void testGetStorageEngineWithBasicAuth() { - PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + when(settings.getSettingValue(Settings.Key.DATASOURCES_URI_ALLOWHOSTS)).thenReturn(".*"); + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(settings); HashMap properties = new HashMap<>(); - properties.put("prometheus.uri", "http://dummyprometheus:9090"); + properties.put("prometheus.uri", "http://dummyprometheus.com:9090"); properties.put("prometheus.auth.type", "basicauth"); properties.put("prometheus.auth.username", "admin"); properties.put("prometheus.auth.password", "admin"); StorageEngine storageEngine - = prometheusStorageFactory.getStorageEngine("my_prometheus", properties); + = prometheusStorageFactory.getStorageEngine(properties); Assertions.assertTrue(storageEngine instanceof PrometheusStorageEngine); } @Test @SneakyThrows void testGetStorageEngineWithAWSSigV4Auth() { - PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + when(settings.getSettingValue(Settings.Key.DATASOURCES_URI_ALLOWHOSTS)).thenReturn(".*"); + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(settings); HashMap properties = new HashMap<>(); - properties.put("prometheus.uri", "http://dummyprometheus:9090"); + properties.put("prometheus.uri", "http://dummyprometheus.com:9090"); properties.put("prometheus.auth.type", "awssigv4"); properties.put("prometheus.auth.region", "us-east-1"); properties.put("prometheus.auth.secret_key", "accessKey"); properties.put("prometheus.auth.access_key", "secretKey"); StorageEngine storageEngine - = prometheusStorageFactory.getStorageEngine("my_prometheus", properties); + = prometheusStorageFactory.getStorageEngine(properties); Assertions.assertTrue(storageEngine instanceof PrometheusStorageEngine); } @@ -62,14 +72,14 @@ void testGetStorageEngineWithAWSSigV4Auth() { @Test @SneakyThrows void testGetStorageEngineWithMissingURI() { - PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(settings); HashMap properties = new HashMap<>(); properties.put("prometheus.auth.type", "awssigv4"); properties.put("prometheus.auth.region", "us-east-1"); properties.put("prometheus.auth.secret_key", "accessKey"); properties.put("prometheus.auth.access_key", "secretKey"); IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, - () -> prometheusStorageFactory.getStorageEngine("my_prometheus", properties)); + () -> prometheusStorageFactory.getStorageEngine(properties)); Assertions.assertEquals("Missing [prometheus.uri] fields " + "in the Prometheus connector properties.", exception.getMessage()); @@ -78,14 +88,14 @@ void testGetStorageEngineWithMissingURI() { @Test @SneakyThrows void testGetStorageEngineWithMissingRegionInAWS() { - PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(settings); HashMap properties = new HashMap<>(); properties.put("prometheus.uri", "http://dummyprometheus:9090"); properties.put("prometheus.auth.type", "awssigv4"); properties.put("prometheus.auth.secret_key", "accessKey"); properties.put("prometheus.auth.access_key", "secretKey"); IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, - () -> prometheusStorageFactory.getStorageEngine("my_prometheus", properties)); + () -> prometheusStorageFactory.getStorageEngine(properties)); Assertions.assertEquals("Missing [prometheus.auth.region] fields in the " + "Prometheus connector properties.", exception.getMessage()); @@ -95,14 +105,14 @@ void testGetStorageEngineWithMissingRegionInAWS() { @Test @SneakyThrows void testGetStorageEngineWithLongConfigProperties() { - PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(settings); HashMap properties = new HashMap<>(); properties.put("prometheus.uri", RandomStringUtils.random(1001)); properties.put("prometheus.auth.type", "awssigv4"); properties.put("prometheus.auth.secret_key", "accessKey"); properties.put("prometheus.auth.access_key", "secretKey"); IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, - () -> prometheusStorageFactory.getStorageEngine("my_prometheus", properties)); + () -> prometheusStorageFactory.getStorageEngine(properties)); Assertions.assertEquals("Missing [prometheus.auth.region] fields in the " + "Prometheus connector properties." + "Fields [prometheus.uri] exceeds more than 1000 characters.", @@ -112,7 +122,8 @@ void testGetStorageEngineWithLongConfigProperties() { @Test @SneakyThrows void testGetStorageEngineWithWrongAuthType() { - PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + when(settings.getSettingValue(Settings.Key.DATASOURCES_URI_ALLOWHOSTS)).thenReturn(".*"); + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(settings); HashMap properties = new HashMap<>(); properties.put("prometheus.uri", "https://test.com"); properties.put("prometheus.auth.type", "random"); @@ -120,7 +131,7 @@ void testGetStorageEngineWithWrongAuthType() { properties.put("prometheus.auth.secret_key", "accessKey"); properties.put("prometheus.auth.access_key", "secretKey"); IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, - () -> prometheusStorageFactory.getStorageEngine("my_prometheus", properties)); + () -> prometheusStorageFactory.getStorageEngine(properties)); Assertions.assertEquals("AUTH Type : random is not supported with Prometheus Connector", exception.getMessage()); } @@ -129,31 +140,68 @@ void testGetStorageEngineWithWrongAuthType() { @Test @SneakyThrows void testGetStorageEngineWithNONEAuthType() { - PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + when(settings.getSettingValue(Settings.Key.DATASOURCES_URI_ALLOWHOSTS)).thenReturn(".*"); + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(settings); HashMap properties = new HashMap<>(); properties.put("prometheus.uri", "https://test.com"); StorageEngine storageEngine - = prometheusStorageFactory.getStorageEngine("my_prometheus", properties); + = prometheusStorageFactory.getStorageEngine(properties); Assertions.assertTrue(storageEngine instanceof PrometheusStorageEngine); } @Test @SneakyThrows void testGetStorageEngineWithInvalidURISyntax() { - PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(settings); HashMap properties = new HashMap<>(); - properties.put("prometheus.uri", "http://dummyprometheus:9090? param"); + properties.put("prometheus.uri", "http://dummyprometheus.com:9090? param"); properties.put("prometheus.auth.type", "basicauth"); properties.put("prometheus.auth.username", "admin"); properties.put("prometheus.auth.password", "admin"); RuntimeException exception = Assertions.assertThrows(RuntimeException.class, - () -> prometheusStorageFactory.getStorageEngine("my_prometheus", properties)); + () -> prometheusStorageFactory.getStorageEngine(properties)); Assertions.assertTrue( - exception.getMessage().contains("Prometheus Client creation failed due to:")); + exception.getMessage().contains("Invalid URI in prometheus properties: ")); } @Test void createDataSourceSuccess() { + when(settings.getSettingValue(Settings.Key.DATASOURCES_URI_ALLOWHOSTS)).thenReturn(".*"); + HashMap properties = new HashMap<>(); + properties.put("prometheus.uri", "http://dummyprometheus.com:9090"); + properties.put("prometheus.auth.type", "basicauth"); + properties.put("prometheus.auth.username", "admin"); + properties.put("prometheus.auth.password", "admin"); + + DataSourceMetadata metadata = new DataSourceMetadata(); + metadata.setName("prometheus"); + metadata.setConnector(DataSourceType.PROMETHEUS); + metadata.setProperties(properties); + + DataSource dataSource = new PrometheusStorageFactory(settings).createDataSource(metadata); + Assertions.assertTrue(dataSource.getStorageEngine() instanceof PrometheusStorageEngine); + } + + @Test + void createDataSourceSuccessWithLocalhost() { + when(settings.getSettingValue(Settings.Key.DATASOURCES_URI_ALLOWHOSTS)).thenReturn(".*"); + HashMap properties = new HashMap<>(); + properties.put("prometheus.uri", "http://localhost:9090"); + properties.put("prometheus.auth.type", "basicauth"); + properties.put("prometheus.auth.username", "admin"); + properties.put("prometheus.auth.password", "admin"); + + DataSourceMetadata metadata = new DataSourceMetadata(); + metadata.setName("prometheus"); + metadata.setConnector(DataSourceType.PROMETHEUS); + metadata.setProperties(properties); + + DataSource dataSource = new PrometheusStorageFactory(settings).createDataSource(metadata); + Assertions.assertTrue(dataSource.getStorageEngine() instanceof PrometheusStorageEngine); + } + + @Test + void createDataSourceWithInvalidHostname() { HashMap properties = new HashMap<>(); properties.put("prometheus.uri", "http://dummyprometheus:9090"); properties.put("prometheus.auth.type", "basicauth"); @@ -165,8 +213,73 @@ void createDataSourceSuccess() { metadata.setConnector(DataSourceType.PROMETHEUS); metadata.setProperties(properties); - DataSource dataSource = new PrometheusStorageFactory().createDataSource(metadata); + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(settings); + RuntimeException exception = Assertions.assertThrows(RuntimeException.class, + () -> prometheusStorageFactory.createDataSource(metadata)); + Assertions.assertTrue( + exception.getMessage().contains("Invalid hostname in the uri: http://dummyprometheus:9090")); + } + + @Test + void createDataSourceWithInvalidIp() { + HashMap properties = new HashMap<>(); + properties.put("prometheus.uri", "http://231.54.11.987:9090"); + properties.put("prometheus.auth.type", "basicauth"); + properties.put("prometheus.auth.username", "admin"); + properties.put("prometheus.auth.password", "admin"); + + DataSourceMetadata metadata = new DataSourceMetadata(); + metadata.setName("prometheus"); + metadata.setConnector(DataSourceType.PROMETHEUS); + metadata.setProperties(properties); + + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(settings); + RuntimeException exception = Assertions.assertThrows(RuntimeException.class, + () -> prometheusStorageFactory.createDataSource(metadata)); + Assertions.assertTrue( + exception.getMessage().contains("Invalid hostname in the uri: http://231.54.11.987:9090")); + } + + @Test + void createDataSourceWithHostnameNotMatchingWithAllowHostsConfig() { + when(settings.getSettingValue(Settings.Key.DATASOURCES_URI_ALLOWHOSTS)) + .thenReturn("^dummy.*.com$"); + HashMap properties = new HashMap<>(); + properties.put("prometheus.uri", "http://localhost.com:9090"); + properties.put("prometheus.auth.type", "basicauth"); + properties.put("prometheus.auth.username", "admin"); + properties.put("prometheus.auth.password", "admin"); + + DataSourceMetadata metadata = new DataSourceMetadata(); + metadata.setName("prometheus"); + metadata.setConnector(DataSourceType.PROMETHEUS); + metadata.setProperties(properties); + + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(settings); + RuntimeException exception = Assertions.assertThrows(RuntimeException.class, + () -> prometheusStorageFactory.createDataSource(metadata)); + Assertions.assertTrue( + exception.getMessage().contains("Disallowed hostname in the uri: http://localhost.com:9090. " + + "Validate with plugins.query.datasources.uri.allowhosts config")); + } + + @Test + void createDataSourceSuccessWithHostnameRestrictions() { + when(settings.getSettingValue(Settings.Key.DATASOURCES_URI_ALLOWHOSTS)) + .thenReturn("^dummy.*.com$"); + HashMap properties = new HashMap<>(); + properties.put("prometheus.uri", "http://dummy.prometheus.com:9090"); + properties.put("prometheus.auth.type", "basicauth"); + properties.put("prometheus.auth.username", "admin"); + properties.put("prometheus.auth.password", "admin"); + + DataSourceMetadata metadata = new DataSourceMetadata(); + metadata.setName("prometheus"); + metadata.setConnector(DataSourceType.PROMETHEUS); + metadata.setProperties(properties); + DataSource dataSource = new PrometheusStorageFactory(settings).createDataSource(metadata); Assertions.assertTrue(dataSource.getStorageEngine() instanceof PrometheusStorageEngine); } + }