Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Require JDK 9 for compilation #28071

Merged
merged 19 commits into from
Jan 16, 2018
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,14 @@ Contributing to the Elasticsearch codebase

**Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)

JDK 9 is required to build Elasticsearch. You must have a JDK 9 installation
with the environment variable `JAVA_HOME` referencing the path to Java home for
your JDK 9 installation. By default, tests use the same runtime as `JAVA_HOME`.
However, since Elasticsearch, supports JDK 8 the build supports compiling with
JDK 9 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
pointing to the Java home of a JDK 8 installation. Note that this mechanism can
be used to test against other JDKs as well, this is not only limited to JDK 8.

Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
using the wrapper via the `gradlew` script in the root of the repository.

Expand Down
107 changes: 52 additions & 55 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@ import java.time.ZonedDateTime
*/
class BuildPlugin implements Plugin<Project> {

static final JavaVersion minimumJava = JavaVersion.VERSION_1_8
static final JavaVersion minimumRuntimeVersion = JavaVersion.VERSION_1_8
static final JavaVersion minimumCompilerVersion = JavaVersion.VERSION_1_9

@Override
void apply(Project project) {
Expand Down Expand Up @@ -93,20 +94,26 @@ class BuildPlugin implements Plugin<Project> {
/** Performs checks on the build environment and prints information about the build environment. */
static void globalBuildInfo(Project project) {
if (project.rootProject.ext.has('buildChecksDone') == false) {
String javaHome = findJavaHome()
String compilerJavaHome = findCompilerJavaHome()
String runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome)
File gradleJavaHome = Jvm.current().javaHome
String javaVendor = System.getProperty('java.vendor')
String javaVersion = System.getProperty('java.version')
String gradleJavaVersionDetails = "${javaVendor} ${javaVersion}" +
" [${System.getProperty('java.vm.name')} ${System.getProperty('java.vm.version')}]"

String javaVersionDetails = gradleJavaVersionDetails
JavaVersion javaVersionEnum = JavaVersion.current()
if (new File(javaHome).canonicalPath != gradleJavaHome.canonicalPath) {
javaVersionDetails = findJavaVersionDetails(project, javaHome)
javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, javaHome))
javaVendor = findJavaVendor(project, javaHome)
javaVersion = findJavaVersion(project, javaHome)
String compilerJavaVersionDetails = gradleJavaVersionDetails
JavaVersion compilerJavaVersionEnum = JavaVersion.current()
if (new File(compilerJavaHome).canonicalPath != gradleJavaHome.canonicalPath) {
compilerJavaVersionDetails = findJavaVersionDetails(project, compilerJavaHome)
compilerJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, compilerJavaHome))
}

String runtimeJavaVersionDetails = gradleJavaVersionDetails
JavaVersion runtimeJavaVersionEnum = JavaVersion.current()
if (new File(runtimeJavaHome).canonicalPath != gradleJavaHome.canonicalPath) {
runtimeJavaVersionDetails = findJavaVersionDetails(project, runtimeJavaHome)
runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, runtimeJavaHome))
}

// Build debugging info
Expand All @@ -115,11 +122,13 @@ class BuildPlugin implements Plugin<Project> {
println '======================================='
println " Gradle Version : ${project.gradle.gradleVersion}"
println " OS Info : ${System.getProperty('os.name')} ${System.getProperty('os.version')} (${System.getProperty('os.arch')})"
if (gradleJavaVersionDetails != javaVersionDetails) {
if (gradleJavaVersionDetails != compilerJavaVersionDetails || gradleJavaVersionDetails != runtimeJavaVersionDetails) {
println " JDK Version (gradle) : ${gradleJavaVersionDetails}"
println " JAVA_HOME (gradle) : ${gradleJavaHome}"
println " JDK Version (compile) : ${javaVersionDetails}"
println " JAVA_HOME (compile) : ${javaHome}"
println " JDK Version (compile) : ${compilerJavaVersionDetails}"
println " JAVA_HOME (compile) : ${compilerJavaHome}"
println " JDK Version (runtime) : ${runtimeJavaVersionDetails}"
println " JAVA_HOME (runtime) : ${runtimeJavaHome}"
} else {
println " JDK Version : ${gradleJavaVersionDetails}"
println " JAVA_HOME : ${gradleJavaHome}"
Expand All @@ -135,54 +144,47 @@ class BuildPlugin implements Plugin<Project> {
}

// enforce Java version
if (javaVersionEnum < minimumJava) {
throw new GradleException("Java ${minimumJava} or above is required to build Elasticsearch")
if (compilerJavaVersionEnum < minimumCompilerVersion) {
throw new GradleException("Java ${minimumCompilerVersion} or above is required to build Elasticsearch")
}

// this block of code detecting buggy JDK 8 compiler versions can be removed when minimum Java version is incremented
assert minimumJava == JavaVersion.VERSION_1_8 : "Remove JDK compiler bug detection only applicable to JDK 8"
if (javaVersionEnum == JavaVersion.VERSION_1_8) {
if (Objects.equals("Oracle Corporation", javaVendor)) {
def matcher = javaVersion =~ /1\.8\.0(?:_(\d+))?/
if (matcher.matches()) {
int update;
if (matcher.group(1) == null) {
update = 0
} else {
update = matcher.group(1).toInteger()
}
if (update < 40) {
throw new GradleException("JDK ${javaVendor} ${javaVersion} has compiler bug JDK-8052388, update your JDK to at least 8u40")
}
}
}
if (runtimeJavaVersionEnum < minimumRuntimeVersion) {
throw new GradleException("Java ${minimumRuntimeVersion} or above is required to run Elasticsearch")
}

project.rootProject.ext.javaHome = javaHome
project.rootProject.ext.javaVersion = javaVersionEnum
project.rootProject.ext.compilerJavaHome = compilerJavaHome
project.rootProject.ext.runtimeJavaHome = runtimeJavaHome
project.rootProject.ext.compilerJavaVersion = compilerJavaVersionEnum
project.rootProject.ext.runtimeJavaVersion = runtimeJavaVersionEnum
project.rootProject.ext.buildChecksDone = true
}
project.targetCompatibility = minimumJava
project.sourceCompatibility = minimumJava
project.targetCompatibility = minimumRuntimeVersion
project.sourceCompatibility = minimumRuntimeVersion
// set java home for each project, so they dont have to find it in the root project
project.ext.javaHome = project.rootProject.ext.javaHome
project.ext.javaVersion = project.rootProject.ext.javaVersion
project.ext.compilerJavaHome = project.rootProject.ext.compilerJavaHome
project.ext.runtimeJavaHome = project.rootProject.ext.runtimeJavaHome
project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion
project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
}

/** Finds and enforces JAVA_HOME is set */
private static String findJavaHome() {
String javaHome = System.getenv('JAVA_HOME')
private static String findCompilerJavaHome() {
final String javaHome = System.getenv('JAVA_HOME')
if (javaHome == null) {
if (System.getProperty("idea.active") != null || System.getProperty("eclipse.launcher") != null) {
// intellij doesn't set JAVA_HOME, so we use the jdk gradle was run with
javaHome = Jvm.current().javaHome
// IntelliJ does not set JAVA_HOME, so we use the JDK that Gradle was run with
return Jvm.current().javaHome
} else {
throw new GradleException('JAVA_HOME must be set to build Elasticsearch')
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why was this exception changed to an assertion? Is it moved somewhere else I'm missing?

Copy link
Member Author

@jasontedor jasontedor Jan 16, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am glad that you caught this. This was inadvertent, it arose in a refactoring I did in 95475f9 which refactored two basically identical methods from an earlier version of this pull request (findCompilerJavaHome and findRuntimeJavaHome) into one (findJavaHome). Before the refactoring, the second method did not need such a line because by the time it had executed the first one would have already thrown that exception (hence the assert false). When I refactored, I must have based by work on findRuntimeJavaHome inadvertently losing this exception. I have pushed: bcee920.

throw new GradleException("JAVA_HOME must be set to build Elasticsearch")
}
}
return javaHome
}

private static String findRuntimeJavaHome(final String compilerJavaHome) {
assert compilerJavaHome != null
return System.getenv('RUNTIME_JAVA_HOME') ?: compilerJavaHome
}

/** Finds printable java version of the given JAVA_HOME */
private static String findJavaVersionDetails(Project project, String javaHome) {
String versionInfoScript = 'print(' +
Expand Down Expand Up @@ -412,7 +414,7 @@ class BuildPlugin implements Plugin<Project> {

/** Adds compiler settings to the project */
static void configureCompile(Project project) {
if (project.javaVersion < JavaVersion.VERSION_1_10) {
if (project.compilerJavaVersion < JavaVersion.VERSION_1_10) {
project.ext.compactProfile = 'compact3'
} else {
project.ext.compactProfile = 'full'
Expand All @@ -422,7 +424,7 @@ class BuildPlugin implements Plugin<Project> {
File gradleJavaHome = Jvm.current().javaHome
// we fork because compiling lots of different classes in a shared jvm can eventually trigger GC overhead limitations
options.fork = true
options.forkOptions.executable = new File(project.javaHome, 'bin/javac')
options.forkOptions.javaHome = new File(project.compilerJavaHome)
options.forkOptions.memoryMaximumSize = "1g"
if (project.targetCompatibility >= JavaVersion.VERSION_1_8) {
// compile with compact 3 profile by default
Expand All @@ -447,22 +449,17 @@ class BuildPlugin implements Plugin<Project> {

options.encoding = 'UTF-8'
options.incremental = true

if (project.javaVersion == JavaVersion.VERSION_1_9) {
// hack until gradle supports java 9's new "--release" arg
assert minimumJava == JavaVersion.VERSION_1_8
options.compilerArgs << '--release' << '8'
}
options.compilerArgs << '--release' << project.targetCompatibility.majorVersion
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a comment to this line linking the relevant gradle issue for adding builtin support for this: gradle/gradle#2510

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I pushed 9434717.

}
}
}

static void configureJavadoc(Project project) {
project.tasks.withType(Javadoc) {
executable = new File(project.javaHome, 'bin/javadoc')
executable = new File(project.compilerJavaHome, 'bin/javadoc')
}
configureJavadocJar(project)
if (project.javaVersion == JavaVersion.VERSION_1_10) {
if (project.compilerJavaVersion == JavaVersion.VERSION_1_10) {
project.tasks.withType(Javadoc) { it.enabled = false }
project.tasks.getByName('javadocJar').each { it.enabled = false }
}
Expand Down Expand Up @@ -508,7 +505,7 @@ class BuildPlugin implements Plugin<Project> {
'X-Compile-Lucene-Version': VersionProperties.lucene,
'X-Compile-Elasticsearch-Snapshot': isSnapshot,
'Build-Date': ZonedDateTime.now(ZoneOffset.UTC),
'Build-Java-Version': project.javaVersion)
'Build-Java-Version': project.compilerJavaVersion)
if (jarTask.manifest.attributes.containsKey('Change') == false) {
logger.warn('Building without git revision id.')
jarTask.manifest.attributes('Change': 'Unknown')
Expand Down Expand Up @@ -545,7 +542,7 @@ class BuildPlugin implements Plugin<Project> {
/** Returns a closure of common configuration shared by unit and integration tests. */
static Closure commonTestConfig(Project project) {
return {
jvm "${project.javaHome}/bin/java"
jvm "${project.runtimeJavaHome}/bin/java"
parallelism System.getProperty('tests.jvms', 'auto')
ifNoTests 'fail'
onNonEmptyWorkDirectory 'wipe'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ public class PluginBuildPlugin extends BuildPlugin {
Files.copy(jarFile.resolveSibling(sourcesFileName), jarFile.resolveSibling(clientSourcesFileName),
StandardCopyOption.REPLACE_EXISTING)

if (project.javaVersion < JavaVersion.VERSION_1_10) {
if (project.compilerJavaVersion < JavaVersion.VERSION_1_10) {
String javadocFileName = jarFile.fileName.toString().replace('.jar', '-javadoc.jar')
String clientJavadocFileName = clientFileName.replace('.jar', '-javadoc.jar')
Files.copy(jarFile.resolveSibling(javadocFileName), jarFile.resolveSibling(clientJavadocFileName),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public class JarHellTask extends LoggedExec {
inputs.files(classpath)
dependsOn(classpath)
description = "Runs CheckJarHell on ${classpath}"
executable = new File(project.javaHome, 'bin/java')
executable = new File(project.runtimeJavaHome, 'bin/java')
doFirst({
/* JarHell doesn't like getting directories that don't exist but
gradle isn't especially careful about that. So we have to do it
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public class LoggerUsageTask extends LoggedExec {
project.afterEvaluate {
dependsOn(classpath)
description = "Runs LoggerUsageCheck on ${classDirectories}"
executable = new File(project.javaHome, 'bin/java')
executable = new File(project.runtimeJavaHome, 'bin/java')
if (classDirectories == null) {
// Default to main and test class files
List files = []
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ public class NamingConventionsTask extends LoggedExec {
FileCollection classpath = project.sourceSets.test.runtimeClasspath
inputs.files(classpath)
description = "Tests that test classes aren't misnamed or misplaced"
executable = new File(project.javaHome, 'bin/java')
executable = new File(project.runtimeJavaHome, 'bin/java')
if (false == checkForTestsInMain) {
/* This task is created by default for all subprojects with this
* setting and there is no point in running it if the files don't
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -655,7 +655,7 @@ class ClusterFormationTasks {
String pid = node.pidFile.getText('UTF-8')
ByteArrayOutputStream output = new ByteArrayOutputStream()
project.exec {
commandLine = ["${project.javaHome}/bin/jstack", pid]
commandLine = ["${project.runtimeJavaHome}/bin/jstack", pid]
standardOutput = output
}
output.toString('UTF-8').eachLine { line -> logger.error("| ${line}") }
Expand Down Expand Up @@ -699,7 +699,7 @@ class ClusterFormationTasks {
}

private static File getJpsExecutableByName(Project project, String jpsExecutableName) {
return Paths.get(project.javaHome.toString(), "bin/" + jpsExecutableName).toFile()
return Paths.get(project.runtimeJavaHome.toString(), "bin/" + jpsExecutableName).toFile()
}

/** Adds a task to kill an elasticsearch node with the given pidfile */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ class NodeInfo {
args.add("${esScript}")
}

env = ['JAVA_HOME': project.javaHome]
env = ['JAVA_HOME': project.runtimeJavaHome]
args.addAll("-E", "node.portsfile=true")
String collectedSystemProperties = config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
String esJavaOpts = config.jvmArgs.isEmpty() ? collectedSystemProperties : collectedSystemProperties + " " + config.jvmArgs
Expand Down
23 changes: 13 additions & 10 deletions distribution/bwc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.Version

import java.util.regex.Matcher

/**
Expand Down Expand Up @@ -118,29 +119,31 @@ if (project.hasProperty('bwcVersion')) {
task buildBwcVersion(type: Exec) {
dependsOn checkoutBwcBranch, writeBuildMetadata
workingDir = checkoutDir
if (project.rootProject.ext.runtimeJavaVersion == JavaVersion.VERSION_1_8 && ["5.6", "6.1"].contains(bwcBranch)) {
/*
* If runtime Java home is set to JDK 8 and we are building branches that are officially built with JDK 8, push this to JAVA_HOME for
* these builds.
*/
environment('JAVA_HOME', System.getenv('RUNTIME_JAVA_HOME'))
}
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
executable 'cmd'
args '/C', 'call', new File(checkoutDir, 'gradlew').toString()
} else {
executable = new File(checkoutDir, 'gradlew').toString()
executable new File(checkoutDir, 'gradlew').toString()
}
final ArrayList<String> commandLineArgs = [
":distribution:deb:assemble",
":distribution:rpm:assemble",
":distribution:zip:assemble",
"-Dbuild.snapshot=${System.getProperty('build.snapshot') ?: 'true'}"]
args ":distribution:deb:assemble", ":distribution:rpm:assemble", ":distribution:zip:assemble", "-Dbuild.snapshot=${System.getProperty('build.snapshot') ?: 'true'}"
final LogLevel logLevel = gradle.startParameter.logLevel
if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
commandLineArgs << "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
}
final String showStacktraceName = gradle.startParameter.showStacktrace.name()
assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
if (showStacktraceName.equals("ALWAYS")) {
commandLineArgs << "--stacktrace"
args "--stacktrace"
} else if (showStacktraceName.equals("ALWAYS_FULL")) {
commandLineArgs << "--full-stacktrace"
args "--full-stacktrace"
}
args commandLineArgs
doLast {
List missing = [bwcDeb, bwcRpm, bwcZip].grep { file ->
false == file.exists()
Expand Down
2 changes: 1 addition & 1 deletion plugins/discovery-azure-classic/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ task createKey(type: LoggedExec) {
project.delete(keystore.parentFile)
keystore.parentFile.mkdirs()
}
executable = new File(project.javaHome, 'bin/keytool')
executable = new File(project.runtimeJavaHome, 'bin/keytool')
standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
args '-genkey',
'-alias', 'test-node',
Expand Down
6 changes: 2 additions & 4 deletions plugins/discovery-ec2/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,11 @@ thirdPartyAudit.excludes = [
'software.amazon.ion.system.IonSystemBuilder',
'software.amazon.ion.system.IonTextWriterBuilder',
'software.amazon.ion.system.IonWriterBuilder',
'javax.xml.bind.DatatypeConverter',
'javax.xml.bind.JAXBContext',
'javax.servlet.ServletContextEvent',
'javax.servlet.ServletContextListener',
'org.apache.avalon.framework.logger.Logger',
'org.apache.log.Hierarchy',
'org.apache.log.Logger',
]

if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += ['javax.xml.bind.DatatypeConverter']
}
2 changes: 1 addition & 1 deletion plugins/discovery-gce/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ task createKey(type: LoggedExec) {
project.delete(keystore.parentFile)
keystore.parentFile.mkdirs()
}
executable = new File(project.javaHome, 'bin/keytool')
executable = new File(project.runtimeJavaHome, 'bin/keytool')
standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
args '-genkey',
'-alias', 'test-node',
Expand Down
Loading