From 7f737117f8737419dc831f6a9b22a04c62c6c212 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 25 Jul 2024 00:45:18 +0000 Subject: [PATCH] Fix integration test failures for 2.x (#46) * fix integration test failures Signed-off-by: Chenyang Ji * add security related config in gradle and integ test Signed-off-by: Chenyang Ji --------- Signed-off-by: Chenyang Ji (cherry picked from commit ccd270f063b691fd2558aae2fc115d5604a1c53d) Signed-off-by: github-actions[bot] --- build.gradle | 356 ++++++++++++++---- .../top_queries/TopQueriesRestIT.java | 146 ++++++- 2 files changed, 424 insertions(+), 78 deletions(-) diff --git a/build.gradle b/build.gradle index 0a89fc35..d3427b30 100644 --- a/build.gradle +++ b/build.gradle @@ -1,10 +1,52 @@ import org.opensearch.gradle.test.RestIntegTestTask +import org.opensearch.gradle.testclusters.OpenSearchCluster + +import groovy.xml.XmlParser +import java.nio.file.Paths +import java.util.concurrent.Callable +import java.util.stream.Collectors + +buildscript { + ext { + opensearch_version = System.getProperty("opensearch.version", "2.16.0-SNAPSHOT") + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + buildVersionQualifier = System.getProperty("build.version_qualifier", "") + version_tokens = opensearch_version.tokenize('-') + opensearch_build = version_tokens[0] + '.0' + if (buildVersionQualifier) { + opensearch_build += "-${buildVersionQualifier}" + } + opensearch_build_snapshot = opensearch_build + '-SNAPSHOT' + if (isSnapshot) { + opensearch_build += "-SNAPSHOT" + } + } + + repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } + } + + dependencies { + classpath "org.opensearch.gradle:build-tools:${opensearch_version}" + } +} + + +plugins { + id "de.undercouch.download" version "5.3.0" + id 'com.diffplug.spotless' version '6.25.0' +} apply plugin: 'java' apply plugin: 'idea' apply plugin: 'eclipse' apply plugin: 'opensearch.opensearchplugin' apply plugin: 'opensearch.pluginzip' +apply plugin: 'opensearch.rest-test' +apply plugin: 'opensearch.repositories' def pluginName = 'query-insights' def pluginDescription = 'OpenSearch Query Insights plugin' @@ -12,36 +54,40 @@ def projectPath = 'org.opensearch' def pathToPlugin = 'plugin.insights' def pluginClassName = 'QueryInsightsPlugin' +configurations { + zipArchive +} + publishing { - publications { - pluginZip(MavenPublication) { publication -> - pom { - name = pluginName - description = pluginDescription - groupId = "org.opensearch.plugin" - licenses { - license { - name = "The Apache License, Version 2.0" - url = "http://www.apache.org/licenses/LICENSE-2.0.txt" - } - } - developers { - developer { - name = "OpenSearch" - url = "https://github.com/opensearch-project/opensearch-plugin-template-java" - } - } - } + publications { + pluginZip(MavenPublication) { publication -> + pom { + name = pluginName + description = pluginDescription + groupId = "org.opensearch.plugin" + licenses { + license { + name = "The Apache License, Version 2.0" + url = "http://www.apache.org/licenses/LICENSE-2.0.txt" + } } + developers { + developer { + name = "OpenSearch" + url = "https://github.com/opensearch-project/opensearch-plugin-template-java" + } + } + } } + } } opensearchplugin { - name pluginName - description pluginDescription - classname "${projectPath}.${pathToPlugin}.${pluginClassName}" - licenseFile rootProject.file('LICENSE.txt') - noticeFile rootProject.file('NOTICE.txt') + name pluginName + description pluginDescription + classname "${projectPath}.${pathToPlugin}.${pluginClassName}" + licenseFile rootProject.file('LICENSE.txt') + noticeFile rootProject.file('NOTICE.txt') } // This requires an additional Jar not published as part of build-tools @@ -50,80 +96,246 @@ loggerUsageCheck.enabled = false // No need to validate pom, as we do not upload to maven/sonatype validateNebulaPom.enabled = false -buildscript { - ext { - opensearch_version = System.getProperty("opensearch.version", "2.16.0-SNAPSHOT") - isSnapshot = "true" == System.getProperty("build.snapshot", "true") - buildVersionQualifier = System.getProperty("build.version_qualifier", "") +allprojects { + group 'org.opensearch' + version = opensearch_version.tokenize('-')[0] + '.0' + if (buildVersionQualifier) { + version += "-${buildVersionQualifier}" + } + if (isSnapshot) { + version += "-SNAPSHOT" + } +} + +repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } +} + +ext { + getSecurityPluginDownloadLink = { -> + var repo = "https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/plugin/" + + "opensearch-security/$opensearch_build_snapshot/" + var metadataFile = Paths.get(projectDir.toString(), "build", "maven-metadata.xml").toAbsolutePath().toFile() + download.run { + src repo + "maven-metadata.xml" + dest metadataFile } + def metadata = new XmlParser().parse(metadataFile) + def securitySnapshotVersion = metadata.versioning.snapshotVersions[0].snapshotVersion[0].value[0].text() + + return repo + "opensearch-security-${securitySnapshotVersion}.zip" + } + + var projectAbsPath = projectDir.getAbsolutePath() + File downloadedSecurityPlugin = Paths.get(projectAbsPath, 'bin', 'opensearch-security-snapshot.zip').toFile() - repositories { - mavenLocal() - maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } - mavenCentral() - maven { url "https://plugins.gradle.org/m2/" } + configureSecurityPlugin = { OpenSearchCluster cluster -> + + cluster.getNodes().forEach { node -> + var creds = node.getCredentials() + if (creds.isEmpty()) { + creds.add(Map.of('useradd', 'admin', '-p', 'admin')) + } else { + creds.get(0).putAll(Map.of('useradd', 'admin', '-p', 'admin')) + } } - dependencies { - classpath "org.opensearch.gradle:build-tools:${opensearch_version}" + // add a check to avoid re-downloading multiple times during single test run + if (!downloadedSecurityPlugin.exists()) { + download.run { + src getSecurityPluginDownloadLink() + dest downloadedSecurityPlugin + } + } else { + println "Security Plugin File Already Exists" } -} -allprojects { - group 'org.opensearch' - version = opensearch_version.tokenize('-')[0] + '.0' - if (buildVersionQualifier) { - version += "-${buildVersionQualifier}" + // Config below including files are copied from security demo configuration + ['esnode.pem', 'esnode-key.pem', 'root-ca.pem'].forEach { file -> + File local = Paths.get(projectAbsPath, 'bin', file).toFile() + download.run { + src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file + dest local + overwrite false + } + cluster.extraConfigFile file, local } - if (isSnapshot) { - version += "-SNAPSHOT" + [ + // config copied from security plugin demo configuration + 'plugins.security.ssl.transport.pemcert_filepath' : 'esnode.pem', + 'plugins.security.ssl.transport.pemkey_filepath' : 'esnode-key.pem', + 'plugins.security.ssl.transport.pemtrustedcas_filepath' : 'root-ca.pem', + 'plugins.security.ssl.transport.enforce_hostname_verification' : 'false', + // https is disabled to simplify test debugging + 'plugins.security.ssl.http.enabled' : 'false', + 'plugins.security.ssl.http.pemcert_filepath' : 'esnode.pem', + 'plugins.security.ssl.http.pemkey_filepath' : 'esnode-key.pem', + 'plugins.security.ssl.http.pemtrustedcas_filepath' : 'root-ca.pem', + 'plugins.security.allow_unsafe_democertificates' : 'true', + 'plugins.security.unsupported.inject_user.enabled': 'true', + 'plugins.security.allow_default_init_securityindex' : 'true', + 'plugins.security.authcz.admin_dn' : 'CN=kirk,OU=client,O=client,L=test,C=de', + 'plugins.security.audit.type' : 'internal_opensearch', + 'plugins.security.enable_snapshot_restore_privilege' : 'true', + 'plugins.security.check_snapshot_restore_write_privileges' : 'true', + 'plugins.security.restapi.roles_enabled' : '["all_access", "security_rest_api_access", "query_insights_full_access"]', + 'plugins.security.system_indices.enabled' : 'true' + ].forEach { name, value -> + cluster.setting name, value } -} -repositories { - mavenLocal() - maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } - mavenCentral() - maven { url "https://plugins.gradle.org/m2/" } + cluster.plugin provider((Callable) (() -> (RegularFile) (() -> downloadedSecurityPlugin))) + } } test { - include '**/*Tests.class' + include '**/*Tests.class' } -task integTest(type: RestIntegTestTask) { - description = "Run tests against a cluster" - testClassesDirs = sourceSets.test.output.classesDirs - classpath = sourceSets.test.runtimeClasspath -} tasks.named("check").configure { dependsOn(integTest) } integTest { - // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable - if (System.getProperty("test.debug") != null) { - jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } + systemProperty "cluster.names", + getClusters().stream().map(cluster -> cluster.getName()).collect(Collectors.joining(",")) + + dependsOn project.tasks.bundlePlugin + testLogging { + events "passed", "skipped", "failed" + } + afterTest { desc, result -> + logger.quiet "${desc.className}.${desc.name}: ${result.resultType} ${(result.getEndTime() - result.getStartTime())/1000}s" + } + systemProperty 'tests.security.manager', 'false' + systemProperty 'project.root', project.projectDir.absolutePath + // Set default query size limit + systemProperty 'defaultQuerySizeLimit', '10000' + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for + // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. + doFirst { + systemProperty 'cluster.debug', getDebug() + getClusters().forEach { cluster -> + + String allTransportSocketURI = cluster.nodes.stream().flatMap { node -> + node.getAllTransportPortURI().stream() + }.collect(Collectors.joining(",")) + String allHttpSocketURI = cluster.nodes.stream().flatMap { node -> + node.getAllHttpSocketURI().stream() + }.collect(Collectors.joining(",")) + + systemProperty "tests.rest.${cluster.name}.http_hosts", "${-> allHttpSocketURI}" + systemProperty "tests.rest.${cluster.name}.transport_hosts", "${-> allTransportSocketURI}" + } + + systemProperty "https", "false" + } + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } + // NOTE: this IT config discovers only junit5 (jupiter) tests. + // https://github.com/opensearch-project/sql/issues/1974 + filter { + includeTestsMatching 'org.opensearch.plugin.insights.rules.resthandler.top_queries.TopQueriesRestIT' + } + if (System.getProperty("security.enabled") == "true") { + useCluster testClusters.integTestWithSecurity + getClusters().forEach { cluster -> + configureSecurityPlugin(cluster) } + systemProperty "user", "admin" + systemProperty "password", "admin" + } else { + useCluster testClusters.integTest + } } -testClusters.integTest { - testDistribution = "INTEG_TEST" +tasks.named("integTest").configure { + it.dependsOn(project.tasks.named("bundlePlugin")) +} - // This installs our plugin into the testClusters - plugin(project.tasks.bundlePlugin.archiveFile) +testClusters.all { + testDistribution = "INTEG_TEST" + // This installs our plugin into the testClusters + plugin(project.tasks.bundlePlugin.archiveFile) } run { - useCluster testClusters.integTest + useCluster testClusters.integTest } -// updateVersion: Task to auto update version to the next development iteration -task updateVersion { - onlyIf { System.getProperty('newVersion') } - doLast { - ext.newVersion = System.getProperty('newVersion') - println "Setting version to ${newVersion}." - // String tokenization to support -SNAPSHOT - ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true) +check.dependsOn spotlessCheck +check.dependsOn jacocoTestReport + +task integTestWithSecurity(type: RestIntegTestTask) { + useCluster testClusters.integTestWithSecurity + + systemProperty "cluster.names", + getClusters().stream().map(cluster -> cluster.getName()).collect(Collectors.joining(",")) + + getClusters().forEach { cluster -> + configureSecurityPlugin(cluster) + } + + dependsOn project.tasks.bundlePlugin + testLogging { + events "passed", "skipped", "failed" + } + afterTest { desc, result -> + logger.quiet "${desc.className}.${desc.name}: ${result.resultType} ${(result.getEndTime() - result.getStartTime())/1000}s" + } + + systemProperty 'tests.security.manager', 'false' + systemProperty 'project.root', project.projectDir.absolutePath + + // Set default query size limit + systemProperty 'defaultQuerySizeLimit', '10000' + + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for + // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. + doFirst { + systemProperty 'cluster.debug', getDebug() + getClusters().forEach { cluster -> + + String allTransportSocketURI = cluster.nodes.stream().flatMap { node -> + node.getAllTransportPortURI().stream() + }.collect(Collectors.joining(",")) + String allHttpSocketURI = cluster.nodes.stream().flatMap { node -> + node.getAllHttpSocketURI().stream() + }.collect(Collectors.joining(",")) + + systemProperty "tests.rest.${cluster.name}.http_hosts", "${-> allHttpSocketURI}" + systemProperty "tests.rest.${cluster.name}.transport_hosts", "${-> allTransportSocketURI}" } + + systemProperty "https", "false" + systemProperty "user", "admin" + systemProperty "password", "admin" + } + + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } + + // NOTE: this IT config discovers only junit5 (jupiter) tests. + // https://github.com/opensearch-project/sql/issues/1974 + filter { + includeTestsMatching 'org.opensearch.plugin.insights.rules.resthandler.top_queries.TopQueriesRestIT' + } } +// updateVersion: Task to auto update version to the next development iteration +task updateVersion { + onlyIf { System.getProperty('newVersion') } + doLast { + ext.newVersion = System.getProperty('newVersion') + println "Setting version to ${newVersion}." + // String tokenization to support -SNAPSHOT + ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true) + } +} diff --git a/src/test/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/TopQueriesRestIT.java b/src/test/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/TopQueriesRestIT.java index 33f7f71a..9fe6bc3d 100644 --- a/src/test/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/TopQueriesRestIT.java +++ b/src/test/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/TopQueriesRestIT.java @@ -8,27 +8,160 @@ package org.opensearch.plugin.insights.rules.resthandler.top_queries; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.message.BasicHeader; +import org.apache.http.ssl.SSLContextBuilder; +import org.junit.After; +import org.junit.Assert; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.plugin.insights.settings.QueryInsightsSettings; import org.opensearch.test.rest.OpenSearchRestTestCase; -import org.junit.Assert; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Map; /** * Rest Action tests for Query Insights */ public class TopQueriesRestIT extends OpenSearchRestTestCase { + private static String QUERY_INSIGHTS_INDICES_PREFIX = "top_queries"; + + protected boolean isHttps() { + return Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); + } + + @Override + protected String getProtocol() { + return isHttps() ? "https" : "http"; + } + + @Override + protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { + RestClientBuilder builder = RestClient.builder(hosts); + if (isHttps()) { + configureHttpsClient(builder, settings); + } else { + configureClient(builder, settings); + } + + builder.setStrictDeprecationMode(false); + return builder.build(); + } + + protected static void configureClient(RestClientBuilder builder, Settings settings) throws IOException { + String userName = System.getProperty("user"); + String password = System.getProperty("password"); + if (userName != null && password != null) { + builder.setHttpClientConfigCallback(httpClientBuilder -> { + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(new AuthScope(null, -1), new UsernamePasswordCredentials(userName, password)); + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + }); + } + OpenSearchRestTestCase.configureClient(builder, settings); + } + + protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) throws IOException { + // Similar to client configuration with OpenSearch: + // https://github.com/opensearch-project/OpenSearch/blob/2.15.1/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java#L841-L863 + builder.setHttpClientConfigCallback(httpClientBuilder -> { + String userName = Optional.ofNullable(System.getProperty("user")) + .orElseThrow(() -> new RuntimeException("user name is missing")); + String password = Optional.ofNullable(System.getProperty("password")) + .orElseThrow(() -> new RuntimeException("password is missing")); + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + final AuthScope anyScope = new AuthScope(null, -1); + credentialsProvider.setCredentials(anyScope, new UsernamePasswordCredentials(userName, password)); + try { + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) + // disable the certificate since our testing cluster just uses the default security configuration + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSSLContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + Map headers = ThreadContext.buildDefaultHeaders(settings); + Header[] defaultHeaders = new Header[headers.size()]; + int i = 0; + for (Map.Entry entry : headers.entrySet()) { + defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); + } + builder.setDefaultHeaders(defaultHeaders); + final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); + final TimeValue socketTimeout = TimeValue.parseTimeValue( + socketTimeoutString == null ? "60s" : socketTimeoutString, + CLIENT_SOCKET_TIMEOUT + ); + builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); + if (settings.hasValue(CLIENT_PATH_PREFIX)) { + builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); + } + } + + /** + * wipeAllIndices won't work since it cannot delete security index. Use + * wipeAllQueryInsightsIndices instead. + */ + @Override + protected boolean preserveIndicesUponCompletion() { + return true; + } + + @SuppressWarnings("unchecked") + @After + public void wipeAllQueryInsightsIndices() throws Exception { + Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); + MediaType xContentType = MediaType.fromMediaType(response.getEntity().getContentType().getValue()); + try ( + XContentParser parser = xContentType.xContent() + .createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + response.getEntity().getContent() + ) + ) { + XContentParser.Token token = parser.nextToken(); + List> parserList = null; + if (token == XContentParser.Token.START_ARRAY) { + parserList = parser.listOrderedMap().stream().map(obj -> (Map) obj).collect(Collectors.toList()); + } else { + parserList = Collections.singletonList(parser.mapOrdered()); + } + + for (Map index : parserList) { + final String indexName = (String) index.get("index"); + if (indexName.startsWith(QUERY_INSIGHTS_INDICES_PREFIX)) { + adminClient().performRequest(new Request("DELETE", "/" + indexName)); + } + } + } + } /** * test Query Insights is installed + * * @throws IOException IOException */ @SuppressWarnings("unchecked") @@ -47,6 +180,7 @@ public void testQueryInsightsPluginInstalled() throws IOException { /** * test enabling top queries + * * @throws IOException IOException */ public void testTopQueriesResponses() throws IOException, InterruptedException {