Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow Integ Tests to run in a FIPS-140 JVM #31989

Merged
merged 13 commits into from
Jul 24, 2018
Merged

Allow Integ Tests to run in a FIPS-140 JVM #31989

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
c9008db
Complete changes for running IT in a fips JVM
jkakavas Jul 6, 2018
32a9b89
unused import and identation
jkakavas Jul 12, 2018
740259d
remove print and migrate to ml-native-multi-node-tests
jkakavas Jul 12, 2018
86ba696
Address feedback
jkakavas Jul 13, 2018
ced0e72
Revert "Address feedback"
jkakavas Jul 23, 2018
86bdddd
Partially addresses feedback
jkakavas Jul 23, 2018
7e1550f
Merge remote-tracking branch 'origin/master' into fips-IT
jkakavas Jul 23, 2018
dd61b4b
Address feedback
jkakavas Jul 23, 2018
8f020b1
Remove unused import
jkakavas Jul 23, 2018
7660385
Merge remote-tracking branch 'origin/master' into fips-IT
jkakavas Jul 23, 2018
12b7ccb
Address feedback
jkakavas Jul 23, 2018
a44d734
add link to SSL sql cli issue
jkakavas Jul 23, 2018
b5753bb
Fix key password parameter
jkakavas Jul 24, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 45 additions & 45 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,39 +219,39 @@ subprojects {
them as external dependencies so the build plugin that we use can be used
to build elasticsearch plugins outside of the elasticsearch source tree. */
ext.projectSubstitutions = [
"org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
"org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
"org.elasticsearch:elasticsearch:${version}": ':server',
"org.elasticsearch:elasticsearch-cli:${version}": ':libs:elasticsearch-cli',
"org.elasticsearch:elasticsearch-core:${version}": ':libs:core',
"org.elasticsearch:elasticsearch-nio:${version}": ':libs:nio',
"org.elasticsearch:elasticsearch-x-content:${version}": ':libs:x-content',
"org.elasticsearch:elasticsearch-secure-sm:${version}": ':libs:secure-sm',
"org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
"org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
"org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',
"org.elasticsearch.client:test:${version}": ':client:test',
"org.elasticsearch.client:transport:${version}": ':client:transport',
"org.elasticsearch.test:framework:${version}": ':test:framework',
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
"org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip',
"org.elasticsearch.distribution.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
"org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:archives:tar',
"org.elasticsearch.distribution.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
"org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
"org.elasticsearch.distribution.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
"org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:packages:deb',
"org.elasticsearch.distribution.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
"org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
"org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
// for transport client
"org.elasticsearch.plugin:transport-netty4-client:${version}": ':modules:transport-netty4',
"org.elasticsearch.plugin:reindex-client:${version}": ':modules:reindex',
"org.elasticsearch.plugin:lang-mustache-client:${version}": ':modules:lang-mustache',
"org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
"org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
"org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
"org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
"org.elasticsearch.gradle:build-tools:${version}" : ':build-tools',
"org.elasticsearch:rest-api-spec:${version}" : ':rest-api-spec',
"org.elasticsearch:elasticsearch:${version}" : ':server',
"org.elasticsearch:elasticsearch-cli:${version}" : ':libs:elasticsearch-cli',
"org.elasticsearch:elasticsearch-core:${version}" : ':libs:core',
"org.elasticsearch:elasticsearch-nio:${version}" : ':libs:nio',
"org.elasticsearch:elasticsearch-x-content:${version}" : ':libs:x-content',
"org.elasticsearch:elasticsearch-secure-sm:${version}" : ':libs:secure-sm',
"org.elasticsearch.client:elasticsearch-rest-client:${version}" : ':client:rest',
"org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}" : ':client:sniffer',
"org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',
"org.elasticsearch.client:test:${version}" : ':client:test',
"org.elasticsearch.client:transport:${version}" : ':client:transport',
"org.elasticsearch.test:framework:${version}" : ':test:framework',
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}" : ':distribution:archives:integ-test-zip',
"org.elasticsearch.distribution.zip:elasticsearch:${version}" : ':distribution:archives:zip',
"org.elasticsearch.distribution.zip:elasticsearch-oss:${version}" : ':distribution:archives:oss-zip',
"org.elasticsearch.distribution.tar:elasticsearch:${version}" : ':distribution:archives:tar',
"org.elasticsearch.distribution.tar:elasticsearch-oss:${version}" : ':distribution:archives:oss-tar',
"org.elasticsearch.distribution.rpm:elasticsearch:${version}" : ':distribution:packages:rpm',
"org.elasticsearch.distribution.rpm:elasticsearch-oss:${version}" : ':distribution:packages:oss-rpm',
"org.elasticsearch.distribution.deb:elasticsearch:${version}" : ':distribution:packages:deb',
"org.elasticsearch.distribution.deb:elasticsearch-oss:${version}" : ':distribution:packages:oss-deb',
"org.elasticsearch.test:logger-usage:${version}" : ':test:logger-usage',
"org.elasticsearch.xpack.test:feature-aware:${version}" : ':x-pack:test:feature-aware',
// for transport client
"org.elasticsearch.plugin:transport-netty4-client:${version}" : ':modules:transport-netty4',
"org.elasticsearch.plugin:reindex-client:${version}" : ':modules:reindex',
"org.elasticsearch.plugin:lang-mustache-client:${version}" : ':modules:lang-mustache',
"org.elasticsearch.plugin:parent-join-client:${version}" : ':modules:parent-join',
"org.elasticsearch.plugin:aggs-matrix-stats-client:${version}" : ':modules:aggs-matrix-stats',
"org.elasticsearch.plugin:percolator-client:${version}" : ':modules:percolator',
"org.elasticsearch.plugin:rank-eval-client:${version}" : ':modules:rank-eval',
]

bwcVersions.snapshotProjectNames.each { snapshotName ->
Expand Down Expand Up @@ -339,18 +339,18 @@ subprojects {
}
boolean hasShadow = project.plugins.hasPlugin(ShadowPlugin)
project.configurations.compile.dependencies
.findAll()
.toSorted(sortClosure)
.each({ c -> depJavadocClosure(hasShadow, c) })
.findAll()
.toSorted(sortClosure)
.each({ c -> depJavadocClosure(hasShadow, c) })
project.configurations.compileOnly.dependencies
.findAll()
.toSorted(sortClosure)
.each({ c -> depJavadocClosure(hasShadow, c) })
.findAll()
.toSorted(sortClosure)
.each({ c -> depJavadocClosure(hasShadow, c) })
if (hasShadow) {
project.configurations.shadow.dependencies
.findAll()
.toSorted(sortClosure)
.each({ c -> depJavadocClosure(false, c) })
.findAll()
.toSorted(sortClosure)
.each({ c -> depJavadocClosure(false, c) })
}
}
}
Expand Down Expand Up @@ -574,7 +574,7 @@ subprojects { project ->
commandLine "${->new File(rootProject.compilerJavaHome, 'bin/jar')}",
'xf', "${-> jarTask.outputs.files.singleFile}", 'META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'
workingDir destination
onlyIf {jarTask.enabled}
onlyIf { jarTask.enabled }
doFirst {
project.delete(destination)
Files.createDirectories(destination)
Expand All @@ -583,7 +583,7 @@ subprojects { project ->

final Task checkNotice = project.task("verify${jarTask.name.capitalize()}Notice") {
dependsOn extract
onlyIf {jarTask.enabled}
onlyIf { jarTask.enabled }
doLast {
final List<String> noticeLines = Files.readAllLines(project.noticeFile.toPath())
final Path noticePath = extract.destination.resolve('META-INF/NOTICE.txt')
Expand All @@ -594,7 +594,7 @@ subprojects { project ->

final Task checkLicense = project.task("verify${jarTask.name.capitalize()}License") {
dependsOn extract
onlyIf {jarTask.enabled}
onlyIf { jarTask.enabled }
doLast {
final List<String> licenseLines = Files.readAllLines(project.licenseFile.toPath())
final Path licensePath = extract.destination.resolve('META-INF/LICENSE.txt')
Expand Down
11 changes: 10 additions & 1 deletion buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,9 @@ class BuildPlugin implements Plugin<Project> {
runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, runtimeJavaHome))
}

String inFipsJvmScript = 'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));'
boolean inFipsJvm = Boolean.parseBoolean(runJavascript(project, runtimeJavaHome, inFipsJvmScript))

// Build debugging info
println '======================================='
println 'Elasticsearch Build Hamster says Hello!'
Expand Down Expand Up @@ -202,6 +205,7 @@ class BuildPlugin implements Plugin<Project> {
project.rootProject.ext.buildChecksDone = true
project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion
project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion
project.rootProject.ext.inFipsJvm = inFipsJvm
}

project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion
Expand All @@ -213,6 +217,7 @@ class BuildPlugin implements Plugin<Project> {
project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion
project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
project.ext.javaVersions = project.rootProject.ext.javaVersions
project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm
}

private static String findCompilerJavaHome() {
Expand Down Expand Up @@ -770,7 +775,11 @@ class BuildPlugin implements Plugin<Project> {
systemProperty property.getKey(), property.getValue()
}
}

// Set the system keystore/truststore password if we're running tests in a FIPS-140 JVM
if (project.inFipsJvm) {
systemProperty 'javax.net.ssl.trustStorePassword', 'password'
systemProperty 'javax.net.ssl.keyStorePassword', 'password'
}
boolean assertionsEnabled = Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))
enableSystemAssertions assertionsEnabled
enableAssertions assertionsEnabled
Expand Down
30 changes: 0 additions & 30 deletions plugins/discovery-gce/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,36 +22,6 @@ dependencies {
compile "commons-codec:commons-codec:${versions.commonscodec}"
}


// needed to be consistent with ssl host checking
String host = InetAddress.getLoopbackAddress().getHostAddress();

// location of keystore and files to generate it
File keystore = new File(project.buildDir, 'keystore/test-node.jks')

// generate the keystore
task createKey(type: LoggedExec) {
doFirst {
project.delete(keystore.parentFile)
keystore.parentFile.mkdirs()
}
executable = new File(project.runtimeJavaHome, 'bin/keytool')
standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
args '-genkey',
'-alias', 'test-node',
'-keystore', keystore,
'-keyalg', 'RSA',
'-keysize', '2048',
'-validity', '712',
'-dname', 'CN=' + host,
'-keypass', 'keypass',
'-storepass', 'keypass'
}

// add keystore to test classpath: it expects it there
sourceSets.test.resources.srcDir(keystore.parentFile)
processTestResources.dependsOn(createKey)

dependencyLicenses {
mapping from: /google-.*/, to: 'google'
}
Expand Down
9 changes: 8 additions & 1 deletion server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,14 @@ public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {
assertThat(nodesMap.size(), equalTo(cluster().size()));
for (final NodesReloadSecureSettingsResponse.NodeResponse nodeResponse : nodesReloadResponse.getNodes()) {
assertThat(nodeResponse.reloadException(), notNullValue());
assertThat(nodeResponse.reloadException(), instanceOf(IOException.class));
// Running in a JVM with a BouncyCastle FIPS Security Provider, decrypting the Keystore with the wrong
// password returns a SecurityException if the DataInputStream can't be fully consumed
if (inFipsJvm()) {
assertThat(nodeResponse.reloadException(), instanceOf(SecurityException.class));
} else {
assertThat(nodeResponse.reloadException(), instanceOf(IOException.class));
}

}
} catch (final AssertionError e) {
reloadSettingsError.set(e);
Expand Down
4 changes: 4 additions & 0 deletions test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.Security;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
Expand Down Expand Up @@ -2364,4 +2365,7 @@ protected void assertSeqNos() throws Exception {
});
}

public static boolean inFipsJvm() {
return Security.getProviders()[0].getName().toLowerCase(Locale.ROOT).contains("fips");
}
}
47 changes: 18 additions & 29 deletions x-pack/plugin/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,39 +104,26 @@ integTestRunner {
systemProperty 'tests.rest.blacklist', blacklist.join(',')
}

// location of generated keystores and certificates
// location for keys and certificates
File keystoreDir = new File(project.buildDir, 'keystore')

// Generate the node's keystore
File nodeKeystore = new File(keystoreDir, 'test-node.jks')
task createNodeKeyStore(type: LoggedExec) {
doFirst {
if (nodeKeystore.parentFile.exists() == false) {
nodeKeystore.parentFile.mkdirs()
}
if (nodeKeystore.exists()) {
delete nodeKeystore
File nodeKey = file("$keystoreDir/testnode.pem")
File nodeCert = file("$keystoreDir/testnode.crt")

// Add key and certs to test classpath: it expects them there
// User cert and key PEM files instead of a JKS Keystore for the cluster's trust material so that
// it can run in a FIPS 140 JVM
task copyKeyCerts(type: Copy) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you please add a comment with a TODO and link to the PR that will make this not use cross project references?

from(project(':x-pack:plugin:core').file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/')) {
include 'testnode.crt', 'testnode.pem'
}
}
executable = new File(project.runtimeJavaHome, 'bin/keytool')
standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
args '-genkey',
'-alias', 'test-node',
'-keystore', nodeKeystore,
'-keyalg', 'RSA',
'-keysize', '2048',
'-validity', '712',
'-dname', 'CN=smoke-test-plugins-ssl',
'-keypass', 'keypass',
'-storepass', 'keypass'
into keystoreDir
}

// Add keystores to test classpath: it expects it there
sourceSets.test.resources.srcDir(keystoreDir)
processTestResources.dependsOn(createNodeKeyStore)
processTestResources.dependsOn(copyKeyCerts)

integTestCluster {
dependsOn createNodeKeyStore
dependsOn copyKeyCerts
setting 'xpack.ml.enabled', 'true'
setting 'xpack.security.enabled', 'true'
setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE'
Expand All @@ -145,17 +132,19 @@ integTestCluster {
setting 'xpack.monitoring.exporters._local.enabled', 'false'
setting 'xpack.security.authc.token.enabled', 'true'
setting 'xpack.security.transport.ssl.enabled', 'true'
setting 'xpack.security.transport.ssl.keystore.path', nodeKeystore.name
setting 'xpack.security.transport.ssl.key', nodeKey.name
setting 'xpack.security.transport.ssl.certificate', nodeCert.name
setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
setting 'xpack.security.audit.enabled', 'true'
setting 'xpack.license.self_generated.type', 'trial'
keystoreSetting 'bootstrap.password', 'x-pack-test-password'
keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'testnode'
distribution = 'zip' // this is important since we use the reindex module in ML

setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'

extraConfigFile nodeKeystore.name, nodeKeystore
extraConfigFile nodeKey.name, nodeKey
extraConfigFile nodeCert.name, nodeCert

waitCondition = { NodeInfo node, AntBuilder ant ->
File tmpFile = new File(node.cwd, 'wait.success')
Expand Down
5 changes: 2 additions & 3 deletions x-pack/plugin/src/test/resources/rest-api-spec/test/ssl/10_basic.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
xpack.ssl.certificates: {}

- length: { $body: 1 }
- match: { $body.0.path: "test-node.jks" }
- match: { $body.0.format: "jks" }
- match: { $body.0.alias: "test-node" }
- match: { $body.0.path: "testnode.crt" }
- match: { $body.0.format: "PEM" }
- match: { $body.0.has_private_key: true }
8 changes: 4 additions & 4 deletions x-pack/qa/full-cluster-restart/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,10 +124,10 @@ subprojects {
}

String output = "${buildDir}/generated-resources/${project.name}"
task copyTestNodeKeystore(type: Copy) {
from project(xpackModule('core'))
.file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
into outputDir
task copyTestNodeKeystore(type: Copy) {
from project(':x-pack:plugin:core')
.file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
into outputDir
}

for (Version version : bwcVersions.indexCompatible) {
Expand Down
Loading