[WIP] Add BouncyCastleFipsProvider as test dependency

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -152,9 +152,8 @@ class BuildPlugin implements Plugin<Project> {
                 runtimeJavaVersionDetails = findJavaVersionDetails(project, runtimeJavaHome)
                 runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, runtimeJavaHome))
             }
-
-            String inFipsJvmScript = 'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));'
-            boolean inFipsJvm = Boolean.parseBoolean(runJavaAsScript(project, runtimeJavaHome, inFipsJvmScript))
+            // Java home name checking is fragile, but we control the environment
+            boolean inFipsJvm = runtimeJavaHome.contains("fips")
 
             // Build debugging info
             println '======================================='
@@ -906,6 +905,16 @@ class BuildPlugin implements Plugin<Project> {
             File heapdumpDir = new File(project.buildDir, 'heapdump')
 
             project.tasks.withType(Test) { Test test ->
+                RepositoryHandler repos = project.repositories
+                if (project.ext.inFipsJvm) {
+                    repos.ivy {
+                        url "https://downloads.bouncycastle.org"
+                        patternLayout {
+                            artifact 'fips-java/[module]-[revision].[ext]'
+                        }
+                    }
+                    project.dependencies.add('testRuntimeOnly', "org.bouncycastle:bc-fips:1.0.1:jar")
+                }
                 File testOutputDir = new File(test.reports.junitXml.getDestination(), "output")
 
                 doFirst {

distribution/tools/plugin-cli/build.gradle

@@ -40,8 +40,7 @@ test {
   systemProperty 'tests.security.manager', 'false'
 }
 
-if (project.inFipsJvm) {
-  // FIPS JVM includes manny classes from bouncycastle which count as jar hell for the third party audit,
-  // rather than provide a long list of exclusions, disable the check on FIPS.
-  thirdPartyAudit.enabled = false
-}
+if (inFipsJvm){
+  // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+  jarHell.enabled = false
+}

modules/transport-netty4/build.gradle

@@ -88,6 +88,7 @@ thirdPartyAudit {
         'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
         'org.bouncycastle.jce.provider.BouncyCastleProvider',
         'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+        'org.bouncycastle.asn1.x500.X500Name',
 
         // from io.netty.handler.ssl.JettyNpnSslEngine (netty)
         'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',
@@ -169,12 +170,4 @@ thirdPartyAudit {
         'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess',
         'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator'
     )
-}
-
-if (project.inFipsJvm == false) {
-    // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in
-    // a FIPS JVM with BouncyCastleFIPS Provider
-    thirdPartyAudit.ignoreMissingClasses (
-            'org.bouncycastle.asn1.x500.X500Name'
-    )
-}
+}

plugins/discovery-azure-classic/build.gradle

@@ -48,7 +48,7 @@ dependencies {
   compile 'org.codehaus.jackson:jackson-core-asl:1.9.2'
   compile 'org.codehaus.jackson:jackson-mapper-asl:1.9.2'
   compile 'org.codehaus.jackson:jackson-jaxrs:1.9.2'
-  compile 'org.codehaus.jackson:jackson-xc:1.9.2'  
+  compile 'org.codehaus.jackson:jackson-xc:1.9.2'
 
   // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
   // and whitelist this hack in JarHell 
@@ -63,6 +63,7 @@ File keystore = new File(project.buildDir, 'keystore/test-node.jks')
 
 // generate the keystore
 task createKey(type: LoggedExec) {
+  onlyIf { inFipsJvm == false }
   doFirst {
     project.delete(keystore.parentFile)
     keystore.parentFile.mkdirs()
@@ -133,3 +134,8 @@ thirdPartyAudit.ignoreMissingClasses (
   'com.sun.xml.fastinfoset.stax.StAXDocumentParser', 
   'com.sun.xml.fastinfoset.stax.StAXDocumentSerializer'
 )
+
+if (inFipsJvm) {
+  // We do not run integ tests in FIPS mode as these use a JKS keystore for the azure settings
+  integTest.enabled = false
+}

plugins/ingest-attachment/build.gradle

@@ -50,7 +50,10 @@ dependencies {
   compile "org.apache.pdfbox:jempbox:1.8.16"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "org.bouncycastle:bcmail-jdk15on:${versions.bouncycastle}"
-  compile "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}"
+  if (inFipsJvm == false) {
+    // In FIPS JVMs we add bc fips provider as a testRuntimeOnly dependency
+    compile "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}"
+  }
   compile "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}"
   // OpenOffice
   compile "org.apache.poi:poi-ooxml:${versions.poi}"
@@ -85,7 +88,6 @@ thirdPartyAudit{
 }
 
 if (project.inFipsJvm) {
-    // FIPS JVM includes manny classes from bouncycastle which count as jar hell for the third party audit,
-    // rather than provide a long list of exclusions, disable the check on FIPS.
-    thirdPartyAudit.enabled = false
+  // Unused license bcprov-jdk15on-LICENSE.txt when not adding bcprov as compile dependency
+  dependencyLicenses.enabled = false
 }

plugins/transport-nio/build.gradle

@@ -59,6 +59,7 @@ thirdPartyAudit {
         'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
         'org.bouncycastle.jce.provider.BouncyCastleProvider',
         'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+        'org.bouncycastle.asn1.x500.X500Name',
 
         // from io.netty.handler.ssl.JettyNpnSslEngine (netty)
         'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',
@@ -147,11 +148,4 @@ thirdPartyAudit {
 
         'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator'
     )
-}
-if (project.inFipsJvm == false) {
-    // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in
-    // a FIPS JVM with BouncyCastleFIPS Provider
-    thirdPartyAudit.ignoreMissingClasses (
-            'org.bouncycastle.asn1.x500.X500Name'
-    )
-}
+}

x-pack/plugin/ml/build.gradle

@@ -114,3 +114,7 @@ gradle.projectsEvaluated {
 task icTest {
   dependsOn internalClusterTest
 }
+if (inFipsJvm){
+    // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+    jarHell.enabled = false
+}

x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle

@@ -70,3 +70,7 @@ integTestCluster {
         return tmpFile.exists()
     }
 }
+if (inFipsJvm){
+    // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+    jarHell.enabled = false
+}

x-pack/plugin/security/build.gradle

@@ -327,3 +327,8 @@ gradle.projectsEvaluated {
             .each { check.dependsOn it.check }
 }
 
+if (inFipsJvm){
+    // Disable jarHell in FIPS as we bundle the BC FIPS Provider for the tests and there are libraries (opensaml) that
+    // depend on the plain BC Provider
+    jarHell.enabled = false
+}

x-pack/plugin/security/cli/build.gradle

@@ -23,15 +23,14 @@ dependencyLicenses {
 }
 
 if (project.inFipsJvm) {
+    // Disable jarHell in FIPS as we bundle the BC FIPS Provider for the tests and this project depends on the the normal
+    // BouncyCastle Provider
+    jarHell.enabled = false
     test.enabled = false
     testingConventions.enabled = false
     // Forbiden APIs non-portable checks fail because bouncy castle classes being used from the FIPS JDK since those are
     // not part of the Java specification - all of this is as designed, so we have to relax this check for FIPS.
     tasks.withType(CheckForbiddenApis) {
         bundledSignatures -= "jdk-non-portable"
     }
-    // FIPS JVM includes many classes from bouncycastle which count as jar hell for the third party audit,
-    // rather than provide a long list of exclusions, disable the check on FIPS.
-    thirdPartyAudit.enabled = false
-
 }

x-pack/plugin/security/qa/tls-basic/build.gradle

@@ -46,3 +46,7 @@ integTestCluster {
   }
 }
 
+if (inFipsJvm){
+  // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+  jarHell.enabled = false
+}

x-pack/plugin/sql/build.gradle

@@ -147,3 +147,7 @@ task regen {
         }
     }
 }
+if (inFipsJvm){
+    // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+    jarHell.enabled = false
+}

x-pack/plugin/sql/qa/security/with-ssl/build.gradle

@@ -208,8 +208,7 @@ integTestCluster {
   }
 }
 Closure notRunningFips = {
-  Boolean.parseBoolean(BuildPlugin.runJavaAsScript(project, project.runtimeJavaHome,
-          'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));')) == false
+  inFipsJvm == false
 }
 
 // Do not attempt to form a cluster in a FIPS JVM, as doing so with a JKS keystore will fail.
@@ -219,6 +218,12 @@ tasks.matching({ it.name == "integTestCluster#init" }).all { onlyIf notRunningFi
 tasks.matching({ it.name == "integTestCluster#start" }).all { onlyIf notRunningFips }
 tasks.matching({ it.name == "integTestCluster#wait" }).all { onlyIf notRunningFips }
 tasks.matching({ it.name == "integTestRunner" }).all { onlyIf notRunningFips }
+tasks.matching({ it.name == "createNodeKeyStore" }).all { onlyIf notRunningFips }
+tasks.matching({ it.name == "createClientKeyStore" }).all { onlyIf notRunningFips }
+tasks.matching({ it.name == "exportNodeCertificate" }).all { onlyIf notRunningFips }
+tasks.matching({ it.name == "exportClientCertificate" }).all { onlyIf  notRunningFips }
+tasks.matching({ it.name == "importNodeCertificateInClientKeyStore" }).all { onlyIf notRunningFips}
+tasks.matching({ it.name == "importClientCertificateInNodeKeyStore" }).all { onlyIf notRunningFips}
 
 /** A lazy evaluator to find the san to use for certificate generation. */
 class SanEvaluator {

x-pack/qa/core-rest-tests-with-security/build.gradle

@@ -44,3 +44,8 @@ integTestCluster {
     return tmpFile.exists()
   }
 }
+
+if (inFipsJvm){
+  // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+  jarHell.enabled = false
+}

x-pack/qa/evil-tests/build.gradle

@@ -9,3 +9,8 @@ test {
     systemProperty 'tests.security.manager', 'false'
     include '**/*Tests.class'
 }
+
+if (inFipsJvm){
+    // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+    jarHell.enabled = false
+}

x-pack/qa/full-cluster-restart/build.gradle

@@ -233,3 +233,8 @@ task copyXPackPluginProps(type: Copy) {
     into outputDir
 }
 project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)
+
+if (inFipsJvm){
+    // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+    jarHell.enabled = false
+}

x-pack/qa/kerberos-tests/build.gradle

@@ -75,3 +75,7 @@ task copyKeytabToGeneratedResources(type: Copy) {
 }
 project.sourceSets.test.output.dir(generatedResources, builtBy:copyKeytabToGeneratedResources)
 
+if (inFipsJvm){
+    // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+    jarHell.enabled = false
+}

x-pack/qa/oidc-op-tests/build.gradle

@@ -81,4 +81,9 @@ integTestCluster {
     }
 }
 
-thirdPartyAudit.enabled = false
+thirdPartyAudit.enabled = false
+
+if (inFipsJvm){
+    // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+    jarHell.enabled = false
+}

x-pack/qa/openldap-tests/build.gradle

@@ -18,3 +18,8 @@ task copyIdpTrust(type: Copy) {
     into outputDir
 }
 project.sourceSets.test.output.dir(outputDir, builtBy: copyIdpTrust)
+
+if (inFipsJvm){
+    // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+    jarHell.enabled = false
+}

x-pack/qa/reindex-tests-with-security/build.gradle

@@ -59,3 +59,8 @@ integTestCluster {
     return http.wait(5000)
   }
 }
+
+if (inFipsJvm){
+  // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+  jarHell.enabled = false
+}

x-pack/qa/saml-idp-tests/build.gradle

@@ -112,3 +112,8 @@ thirdPartyAudit {
        'com.ibm.icu.lang.UCharacter'
    )
 }
+
+if (inFipsJvm){
+    // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+    jarHell.enabled = false
+}

x-pack/qa/security-tools-tests/build.gradle

@@ -21,3 +21,8 @@ forbiddenPatterns {
 
 // these are just tests, no need to audit
 thirdPartyAudit.enabled = false
+
+if (inFipsJvm){
+  // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+  jarHell.enabled = false
+}

x-pack/qa/third-party/active-directory/build.gradle

@@ -32,3 +32,7 @@ test {
   include '**/*Tests.class'
 }
 
+if (inFipsJvm){
+  // Disable jarHell in FIPS as we add the BC FIPS Provider as a dependency for the tests
+  jarHell.enabled = false
+}