Skip to content

Commit

Permalink
Fix cluster default initialization Part 1 (opensearch-project#4002)
Browse files Browse the repository at this point in the history 
Signed-off-by: Andrey Pleskach <[email protected]>
(cherry picked from commit b0d26dd)
  • Loading branch information
@willyborankin
willyborankin committed Apr 15, 2024
1 parent c0c4529 commit efd7637
Show file tree
Hide file tree
Showing 20 changed files with 1,979 additions and 136 deletions.
95 changes: 51 additions & 44 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,23 +16,23 @@ import groovy.json.JsonBuilder

buildscript {
ext {
opensearch_version = System.getProperty("opensearch.version", "2.14.0-SNAPSHOT")
opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
buildVersionQualifier = System.getProperty("build.version_qualifier", "")

// 2.0.0-rc1-SNAPSHOT -> 2.0.0.0-rc1-SNAPSHOT
version_tokens = opensearch_version.tokenize('-')
opensearch_build = version_tokens[0] + '.0'

common_utils_version = System.getProperty("common_utils.version", '2.9.0.0-SNAPSHOT')
common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT')
kafka_version = '3.7.0'
apache_cxf_version = '4.0.3'
open_saml_version = '4.3.0'
apache_cxf_version = '4.0.4'
open_saml_version = '4.3.1'
one_login_java_saml = '2.9.0'
jjwt_version = '0.12.5'
guava_version = '32.1.3-jre'
jaxb_version = '2.3.9'
spring_version = '5.3.34'
spring_version = '5.3.33'

if (buildVersionQualifier) {
opensearch_build += "-${buildVersionQualifier}"
Expand Down Expand Up @@ -96,6 +96,7 @@ spotbugsTest {
java.sourceCompatibility = JavaVersion.VERSION_11
java.targetCompatibility = JavaVersion.VERSION_11


compileJava {
options.compilerArgs = [
'-Xlint:auxiliaryclass',
Expand Down Expand Up @@ -152,6 +153,7 @@ compileTestJava {
options.encoding = 'UTF-8'
}


licenseHeaders.enabled = true

// The following check that have never be enabled in security
Expand Down Expand Up @@ -384,9 +386,18 @@ jacocoTestReport {

checkstyle {
toolVersion "10.3.3"
showViolations true
configDirectory.set(rootProject.file("checkstyle/"))
}

tasks.withType(Checkstyle) {
dependsOn(':precommit')
reports {
ignoreFailures = false
}
}


opensearchplugin {
name 'opensearch-security'
description 'Provide access control related features for OpenSearch'
Expand Down Expand Up @@ -439,20 +450,6 @@ repositories {
maven { url "https://build.shibboleth.net/nexus/content/repositories/releases" }
}

tasks.withType(Checkstyle) {
showViolations true
reports {
ignoreFailures = false
}
}

tasks.withType(JavaCompile) {
configure(options) {
options.encoding = 'UTF-8'
options.compilerArgs << '-Xlint:removal' << '-Werror'
}
}

tasks.test.finalizedBy(jacocoTestReport) // report is always generated after tests run

allprojects {
Expand Down Expand Up @@ -492,19 +489,14 @@ configurations {
force "org.apache.commons:commons-lang3:${versions.commonslang}"

// for spotless transitive dependency CVE
force "org.eclipse.platform:org.eclipse.core.runtime:3.30.0"
force "org.eclipse.platform:org.eclipse.core.runtime:3.31.0"

// For integrationTest
force "org.apache.httpcomponents:httpclient-cache:4.5.14"
force "org.apache.httpcomponents:httpclient:4.5.13"
force "org.apache.httpcomponents:fluent-hc:4.5.13"
force "org.apache.httpcomponents:httpclient:4.5.14"
force "org.apache.httpcomponents:httpcore:4.4.16"
force "org.apache.httpcomponents:httpcore-nio:4.4.16"
force "org.apache.httpcomponents:httpasyncclient:4.1.5"
force 'org.checkerframework:checker-qual:3.40.0'
force "com.google.errorprone:error_prone_annotations:2.26.1"
force "org.checkerframework:checker-qual:3.42.0"
force "ch.qos.logback:logback-classic:1.5.5"
force "ch.qos.logback:logback-classic:1.5.3"
}
}

Expand Down Expand Up @@ -553,6 +545,21 @@ task integrationTest(type: Test) {
classpath = sourceSets.integrationTest.runtimeClasspath
//run the integrationTest task after the test task
shouldRunAfter test
jacoco {
excludes = [
"com.sun.jndi.dns.*",
"com.sun.security.sasl.gsskerb.*",
"java.sql.*",
"javax.script.*",
"org.jcp.xml.dsig.internal.dom.*",
"sun.nio.cs.ext.*",
"sun.security.ec.*",
"sun.security.jgss.*",
"sun.security.pkcs11.*",
"sun.security.smartcardio.*",
"sun.util.resources.provider.*"
]
}
}

tasks.integrationTest.finalizedBy(jacocoTestReport) // report is always generated after integration tests run
Expand All @@ -563,14 +570,14 @@ check.dependsOn integrationTest
dependencies {
implementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}"
implementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}"
implementation "org.apache.httpcomponents:httpclient-cache:${versions.httpclient}"
implementation "org.apache.httpcomponents.client5:httpclient5-cache:${versions.httpclient5}"
implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}"
implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}"
implementation "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
implementation "com.google.guava:guava:${guava_version}"
implementation 'org.greenrobot:eventbus-java:3.3.1'
implementation 'commons-cli:commons-cli:1.6.0'
implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}"
implementation "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}"
implementation 'org.ldaptive:ldaptive:1.2.3'
implementation 'com.nimbusds:nimbus-jose-jwt:9.37.3'

Expand All @@ -587,26 +594,26 @@ dependencies {

implementation "org.apache.kafka:kafka-clients:${kafka_version}"

runtimeOnly 'net.minidev:accessors-smart:2.5.0'
runtimeOnly 'net.minidev:accessors-smart:2.5.1'

runtimeOnly "org.apache.cxf:cxf-core:${apache_cxf_version}"
implementation "org.apache.cxf:cxf-rt-rs-json-basic:${apache_cxf_version}"
runtimeOnly "org.apache.cxf:cxf-rt-security:${apache_cxf_version}"

runtimeOnly 'com.sun.activation:jakarta.activation:1.2.2'
runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5'
runtimeOnly 'commons-codec:commons-codec:1.16.0'
runtimeOnly 'commons-codec:commons-codec:1.16.1'
runtimeOnly 'org.cryptacular:cryptacular:1.2.6'
compileOnly 'com.google.errorprone:error_prone_annotations:2.26.1'
runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0'
runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2'
runtimeOnly 'org.ow2.asm:asm:9.6'
runtimeOnly 'org.ow2.asm:asm:9.7'

testImplementation 'org.apache.camel:camel-xmlsecurity:3.22.1'

//OpenSAML
implementation 'net.shibboleth.utilities:java-support:8.4.2'
runtimeOnly "io.dropwizard.metrics:metrics-core:4.2.25"
implementation 'net.shibboleth.utilities:java-support:8.4.1'
runtimeOnly "io.dropwizard.metrics:metrics-core:4.2.15"
implementation "com.onelogin:java-saml:${one_login_java_saml}"
implementation "com.onelogin:java-saml-core:${one_login_java_saml}"
implementation "org.opensaml:opensaml-core:${open_saml_version}"
Expand All @@ -624,26 +631,25 @@ dependencies {
runtimeOnly "org.opensaml:opensaml-soap-impl:${open_saml_version}"
implementation "org.opensaml:opensaml-storage-api:${open_saml_version}"

implementation "com.nulab-inc:zxcvbn:1.9.0"
implementation "com.nulab-inc:zxcvbn:1.8.2"

runtimeOnly 'com.google.guava:failureaccess:1.0.2'
runtimeOnly 'org.apache.commons:commons-text:1.11.0'
runtimeOnly "org.glassfish.jaxb:jaxb-runtime:${jaxb_version}"
runtimeOnly 'com.google.j2objc:j2objc-annotations:2.8'
compileOnly 'com.google.code.findbugs:jsr305:3.0.2'
runtimeOnly 'org.lz4:lz4-java:1.8.0'
runtimeOnly 'io.dropwizard.metrics:metrics-core:4.2.25'
runtimeOnly 'org.slf4j:slf4j-api:1.7.36'
runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.5'
runtimeOnly 'org.codehaus.woodstox:stax2-api:4.2.2'
runtimeOnly "org.glassfish.jaxb:txw2:${jaxb_version}"
runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.6.2'
runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.6.1'
runtimeOnly 'org.apache.ws.xmlschema:xmlschema-core:2.3.1'
runtimeOnly 'org.apache.santuario:xmlsec:2.3.4'
runtimeOnly "com.github.luben:zstd-jni:${versions.zstd}"
runtimeOnly 'org.checkerframework:checker-qual:3.42.0'
runtimeOnly "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}"
runtimeOnly "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
runtimeOnly 'org.scala-lang.modules:scala-java8-compat_3:1.0.2'


Expand All @@ -663,7 +669,8 @@ dependencies {
testImplementation 'com.github.stephenc.jcip:jcip-annotations:1.0-1'
testImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14'
testImplementation 'javax.servlet:servlet-api:2.5'
testImplementation 'org.apache.httpcomponents:fluent-hc:4.5.13'
testImplementation 'org.apache.httpcomponents:fluent-hc:4.5.14'
testImplementation "org.apache.httpcomponents.client5:httpclient5-fluent:${versions.httpclient5}"
testImplementation "org.apache.kafka:kafka_2.13:${kafka_version}"
testImplementation "org.apache.kafka:kafka-server:${kafka_version}"
testImplementation "org.apache.kafka:kafka-server-common:${kafka_version}"
Expand Down Expand Up @@ -692,12 +699,12 @@ dependencies {
exclude(group:'org.springframework', module: 'spring-jcl' )
}
testRuntimeOnly 'org.scala-lang:scala-library:2.13.13'
testRuntimeOnly 'com.yammer.metrics:metrics-core:2.2.0'
testRuntimeOnly 'com.typesafe.scala-logging:scala-logging_3:3.9.5'
testRuntimeOnly('org.apache.zookeeper:zookeeper:3.9.2') {
exclude(group:'ch.qos.logback', module: 'logback-classic' )
exclude(group:'ch.qos.logback', module: 'logback-core' )
}
testRuntimeOnly 'com.yammer.metrics:metrics-core:2.2.0'
testRuntimeOnly "org.apache.kafka:kafka-metadata:${kafka_version}"
testRuntimeOnly "org.apache.kafka:kafka-storage:${kafka_version}"

Expand All @@ -713,12 +720,12 @@ dependencies {
integrationTestImplementation 'junit:junit:4.13.2'
integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
integrationTestImplementation 'commons-io:commons-io:2.16.1'
integrationTestImplementation 'commons-io:commons-io:2.15.1'
integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"
integrationTestImplementation 'org.hamcrest:hamcrest:2.2'
integrationTestImplementation "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}"
integrationTestImplementation "org.bouncycastle:bcutil-jdk15to18:${versions.bouncycastle}"
integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${versions.bouncycastle}"
integrationTestImplementation('org.awaitility:awaitility:4.2.1') {
exclude(group: 'org.hamcrest', module: 'hamcrest')
}
Expand All @@ -730,7 +737,7 @@ dependencies {
integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5"

//spotless
implementation('com.google.googlejavaformat:google-java-format:1.22.0') {
implementation('com.google.googlejavaformat:google-java-format:1.21.0') {
exclude group: 'com.google.guava'
}
}
Expand Down
88 changes: 53 additions & 35 deletions ...h/security/DefaultConfigurationTests.java → ...ty/AbstractDefaultConfigurationTests.java
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
*/
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
*/
package org.opensearch.security;

import java.io.IOException;
Expand All @@ -19,47 +19,39 @@
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.fasterxml.jackson.databind.JsonNode;
import org.apache.commons.io.FileUtils;
import org.apache.http.HttpStatus;
import org.awaitility.Awaitility;
import org.junit.AfterClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.runner.RunWith;

import org.opensearch.test.framework.TestSecurityConfig.User;
import org.opensearch.test.framework.cluster.ClusterManager;
import org.opensearch.security.state.SecurityMetadata;
import org.opensearch.test.framework.TestSecurityConfig;
import org.opensearch.test.framework.cluster.LocalCluster;
import org.opensearch.test.framework.cluster.TestRestClient;
import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.aMapWithSize;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
public class DefaultConfigurationTests {

private final static Path configurationFolder = ConfigurationFiles.createConfigurationDirectory();
private static final User ADMIN_USER = new User("admin");
private static final User NEW_USER = new User("new-user");
private static final User LIMITED_USER = new User("limited-user");

@ClassRule
public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE)
.nodeSettings(
Map.of(
"plugins.security.allow_default_init_securityindex",
true,
"plugins.security.restapi.roles_enabled",
List.of("user_admin__all_access")
)
)
.defaultConfigurationInitDirectory(configurationFolder.toString())
.loadConfigurationIntoIndex(false)
.build();
public abstract class AbstractDefaultConfigurationTests {
public final static Path configurationFolder = ConfigurationFiles.createConfigurationDirectory();
private static final TestSecurityConfig.User ADMIN_USER = new TestSecurityConfig.User("admin");
private static final TestSecurityConfig.User NEW_USER = new TestSecurityConfig.User("new-user");
private static final TestSecurityConfig.User LIMITED_USER = new TestSecurityConfig.User("limited-user");

private final LocalCluster cluster;

protected AbstractDefaultConfigurationTests(LocalCluster cluster) {
this.cluster = cluster;
}

@AfterClass
public static void cleanConfigurationDirectory() throws IOException {
Expand All @@ -73,18 +65,43 @@ public void shouldLoadDefaultConfiguration() {
}
try (TestRestClient client = cluster.getRestClient(ADMIN_USER)) {
client.confirmCorrectCredentials(ADMIN_USER.getName());
HttpResponse response = client.get("_plugins/_security/api/internalusers");
response.assertStatusCode(200);
TestRestClient.HttpResponse response = client.get("_plugins/_security/api/internalusers");
response.assertStatusCode(HttpStatus.SC_OK);
Map<String, Object> users = response.getBodyAs(Map.class);
assertThat(
response.getBody(),
users,
allOf(aMapWithSize(3), hasKey(ADMIN_USER.getName()), hasKey(NEW_USER.getName()), hasKey(LIMITED_USER.getName()))
);
}
}

void assertClusterState(final TestRestClient client) {
if (cluster.node().settings().getAsBoolean("plugins.security.allow_default_init_securityindex.use_cluster_state", false)) {
final TestRestClient.HttpResponse response = client.get("_cluster/state");
response.assertStatusCode(HttpStatus.SC_OK);
final var clusterState = response.getBodyAs(Map.class);
assertTrue(response.getBody(), clusterState.containsKey(SecurityMetadata.TYPE));
@SuppressWarnings("unchecked")
final var securityClusterState = (Map<String, Object>) clusterState.get(SecurityMetadata.TYPE);
@SuppressWarnings("unchecked")
final var securityConfiguration = (Map<String, Object>) ((Map<?, ?>) clusterState.get(SecurityMetadata.TYPE)).get(
"configuration"
);
assertTrue(response.getBody(), securityClusterState.containsKey("created"));
assertNotNull(response.getBody(), securityClusterState.get("created"));

for (final var k : securityConfiguration.keySet()) {
@SuppressWarnings("unchecked")
final var sc = (Map<String, Object>) securityConfiguration.get(k);
assertTrue(response.getBody(), sc.containsKey("hash"));
assertTrue(response.getBody(), sc.containsKey("last_modified"));
}
}
}

@Test
public void securityRolesUgrade() throws Exception {
public void securityRolesUpgrade() throws Exception {
try (var client = cluster.getRestClient(ADMIN_USER)) {
// Setup: Make sure the config is ready before starting modifications
Awaitility.await().alias("Load default configuration").until(() -> client.getAuthInfo().getStatusCode(), equalTo(200));
Expand Down Expand Up @@ -159,4 +176,5 @@ private Set<String> extractFieldNames(final JsonNode json) {
json.fieldNames().forEachRemaining(set::add);
return set;
}

}
Loading

0 comments on commit efd7637

Please sign in to comment.