From 0766a2c58e6e473461d8623030b8f469fac53c84 Mon Sep 17 00:00:00 2001 From: Chad Wilson Date: Fri, 29 Oct 2021 21:28:26 +0800 Subject: [PATCH] Use JLink to produce JREs for Windows, Linux and OSX - Adoptium / Eclipse Temurin no longer produces JREs, unfortunately https://github.com/adoptium/temurin-build/issues/2683 This means we need to download a JDK and produce a JRE for each platform - They recommend you use JLink to produce your own JREs https://blog.adoptium.net/2021/10/jlink-to-produce-own-runtime/ - It seems possible to do this cross-platform, use the Gradle Toolchain JVM with a JDK for target platform to produce JRE for target platform - This currently includes all modules and can be cut down further to use fewer modules, similar to that within the old AdoptOpenJDK JREs --- .../go/build/AdoptiumJreTask.groovy | 76 +++++++++++++++++++ .../go/build/AdoptiumUrlHelper.groovy | 30 ++++++-- installers/build.gradle | 3 + installers/linux.gradle | 50 ++++++------ installers/osx.gradle | 57 +++++++------- installers/windows.gradle | 52 ++++++------- 6 files changed, 179 insertions(+), 89 deletions(-) create mode 100644 buildSrc/src/main/groovy/com/thoughtworks/go/build/AdoptiumJreTask.groovy diff --git a/buildSrc/src/main/groovy/com/thoughtworks/go/build/AdoptiumJreTask.groovy b/buildSrc/src/main/groovy/com/thoughtworks/go/build/AdoptiumJreTask.groovy new file mode 100644 index 000000000000..353b95b3797a --- /dev/null +++ b/buildSrc/src/main/groovy/com/thoughtworks/go/build/AdoptiumJreTask.groovy @@ -0,0 +1,76 @@ +/* + * Copyright 2021 ThoughtWorks, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.thoughtworks.go.build + +import org.gradle.api.DefaultTask +import org.gradle.api.file.RelativePath +import org.gradle.api.tasks.* +import org.gradle.jvm.toolchain.JavaLanguageVersion + +class AdoptiumJreTask extends DefaultTask { + @Input AdoptiumVersion version + @InputFile File sourceJdkArchive + + @Internal + File getOutputJdkDir() { + new File("${project.buildDir}/jdks-tmp/${version.operatingSystem}") + } + + @OutputDirectory + File getOutputDir() { + new File("${project.buildDir}/jres/${version.operatingSystem}") + } + + @TaskAction + def perform() { + unpackJdk() + jlinkJre() + } + + private void unpackJdk() { + project.delete(outputJdkDir) + def archiveTree = OperatingSystem.windows == version.operatingSystem ? project.zipTree(sourceJdkArchive) : project.tarTree(sourceJdkArchive) + project.copy { + from(archiveTree) { + includeEmptyDirs = false + eachFile { fcd -> // Drop leading directory (package name, which differes version to version) + fcd.relativePath = new RelativePath(true, fcd.relativePath.segments.drop(1)) + } + } + into outputJdkDir + } + } + + private void jlinkJre() { + project.delete(outputDir) + + // use Gradle toolchains to find a JVM of the appropriate major version for our target JRE + def jvm = project.javaToolchains.compilerFor { + languageVersion = JavaLanguageVersion.of(version.featureVersion) + } + project.exec { + commandLine = ["${jvm.get().metadata.installationPath.asFile.absolutePath}/bin/jlink", + '--add-modules', 'ALL-MODULE-PATH', // TODO make modules configurable input (with default?) + '--module-path', "${outputJdkDir}/${version.javaModulesRelativePath()}", + '--strip-debug', '--no-man-pages', '--no-header-files', '--compress=2', + '--output', "${outputDir}/jre"] + standardOutput = System.out + errorOutput = System.err + } + } + +} \ No newline at end of file diff --git a/buildSrc/src/main/groovy/com/thoughtworks/go/build/AdoptiumUrlHelper.groovy b/buildSrc/src/main/groovy/com/thoughtworks/go/build/AdoptiumUrlHelper.groovy index b21cc595819b..545a23037574 100644 --- a/buildSrc/src/main/groovy/com/thoughtworks/go/build/AdoptiumUrlHelper.groovy +++ b/buildSrc/src/main/groovy/com/thoughtworks/go/build/AdoptiumUrlHelper.groovy @@ -17,14 +17,34 @@ package com.thoughtworks.go.build +class AdoptiumVersion implements Serializable { + OperatingSystem operatingSystem + Integer featureVersion + Integer interimVersion + Integer updateVersion + Integer buildVersion + + AdoptiumVersion(OperatingSystem operatingSystem, Integer featureVersion, Integer interimVersion, Integer updateVersion, Integer buildVersion) { + this.operatingSystem = operatingSystem + this.featureVersion = featureVersion + this.interimVersion = interimVersion + this.updateVersion = updateVersion + this.buildVersion = buildVersion + } + + String javaModulesRelativePath() { + OperatingSystem.mac == operatingSystem ? "Contents/Home/jmods" : "jmods" + } +} + class AdoptiumUrlHelper { - static String downloadURL(OperatingSystem operatingSystem, Integer featureVersion, Integer interimVersion, Integer updateVersion, Integer buildVersion) { - String versionComponent = [featureVersion, interimVersion, updateVersion].findAll({ it != null }).join('.') + static String downloadURL(AdoptiumVersion v) { + String versionComponent = [v.featureVersion, v.interimVersion, v.updateVersion].findAll({ it != null }).join('.') - "https://github.com/adoptium/temurin${featureVersion}-binaries/releases/download/jdk-${versionComponent}%2B${buildVersion}/OpenJDK${featureVersion}-jdk_x64_${operatingSystem.adoptiumAlias()}_hotspot_${versionComponent}_${buildVersion}.${operatingSystem.extension}" + "https://github.com/adoptium/temurin${v.featureVersion}-binaries/releases/download/jdk-${versionComponent}%2B${v.buildVersion}/OpenJDK${v.featureVersion}-jdk_x64_${v.operatingSystem.adoptiumAlias()}_hotspot_${versionComponent}_${v.buildVersion}.${v.operatingSystem.extension}" } - static String sha256sumURL(OperatingSystem operatingSystem, Integer featureVersion, Integer interimVersion, Integer updateVersion, Integer buildVersion) { - "${downloadURL(operatingSystem, featureVersion, interimVersion, updateVersion, buildVersion)}.sha256.txt" + static String sha256sumURL(AdoptiumVersion v) { + "${downloadURL(v)}.sha256.txt" } } diff --git a/installers/build.gradle b/installers/build.gradle index ab5280c15ff1..9285577b88a2 100644 --- a/installers/build.gradle +++ b/installers/build.gradle @@ -17,6 +17,9 @@ import groovy.json.JsonOutput description = 'GoCD Installers' apply plugin: 'base' +apply plugin: 'java' + +jar.enabled = false configurations { agentBootstrapperJar { diff --git a/installers/linux.gradle b/installers/linux.gradle index f5ecc617ee06..da4c996f85bc 100644 --- a/installers/linux.gradle +++ b/installers/linux.gradle @@ -15,7 +15,9 @@ */ +import com.thoughtworks.go.build.AdoptiumJreTask import com.thoughtworks.go.build.AdoptiumUrlHelper +import com.thoughtworks.go.build.AdoptiumVersion import com.thoughtworks.go.build.DownloadFile import com.thoughtworks.go.build.InstallerType import groovy.json.JsonOutput @@ -104,37 +106,38 @@ private File destFile(String url) { new File(gradle.gradleUserHomeDir, "download-cache/${DigestUtils.md5Hex(url)}/${new File(new URL(url).path).name}") } -task downloadLinuxJreChecksum(type: DownloadFile) { - def srcUrl = AdoptiumUrlHelper.sha256sumURL( - com.thoughtworks.go.build.OperatingSystem.linux, - project.packaging.adoptOpenjdk.featureVersion, - project.packaging.adoptOpenjdk.interimVersion, - project.packaging.adoptOpenjdk.updateVersion, - project.packaging.adoptOpenjdk.buildVersion - ) +def adoptiumVersion = new AdoptiumVersion( + com.thoughtworks.go.build.OperatingSystem.linux, + project.packaging.adoptOpenjdk.featureVersion, + project.packaging.adoptOpenjdk.interimVersion, + project.packaging.adoptOpenjdk.updateVersion, + project.packaging.adoptOpenjdk.buildVersion +) + +task downloadLinuxJdkChecksum(type: DownloadFile) { + def srcUrl = AdoptiumUrlHelper.sha256sumURL(adoptiumVersion) src srcUrl dest destFile(srcUrl) } -task downloadLinuxJre(type: DownloadFile) { - - dependsOn downloadLinuxJreChecksum - def srcUrl = AdoptiumUrlHelper.downloadURL( - com.thoughtworks.go.build.OperatingSystem.linux, - project.packaging.adoptOpenjdk.featureVersion, - project.packaging.adoptOpenjdk.interimVersion, - project.packaging.adoptOpenjdk.updateVersion, - project.packaging.adoptOpenjdk.buildVersion - ) +task downloadLinuxJdk(type: DownloadFile) { + dependsOn downloadLinuxJdkChecksum + def srcUrl = AdoptiumUrlHelper.downloadURL(adoptiumVersion) src srcUrl dest destFile(srcUrl) - checksum = { downloadLinuxJreChecksum.outputs.files.singleFile.getText("utf-8").trim().split(" ").first() } + checksum = { downloadLinuxJdkChecksum.outputs.files.singleFile.getText("utf-8").trim().split(" ").first() } +} + +task buildLinuxJre(type: AdoptiumJreTask) { + dependsOn downloadLinuxJdk + version = adoptiumVersion + sourceJdkArchive = downloadLinuxJdk.outputs.files.singleFile } def configureLinuxPackage(DefaultTask packageTask, InstallerType installerType, OsType osType, File buildRoot, Configuration configuration, File destDir) { packageTask.with { dependsOn configuration - dependsOn downloadLinuxJre + dependsOn buildLinuxJre dependsOn extractDeltaPack doFirst { @@ -308,12 +311,8 @@ def configureLinuxPackage(DefaultTask packageTask, InstallerType installerType, } project.copy { - into "${buildRoot}/usr/share/${installerType.baseName}/jre" - from(tarTree(downloadLinuxJre.outputs.files.singleFile)) { + from(buildLinuxJre.outputDir) { eachFile { FileCopyDetails fcd -> - List segments = fcd.relativePath.segments - segments.remove(0) - fcd.relativePath = new RelativePath(!fcd.isDirectory(), (segments as String[])) // Remove write access of JRE files for user group `others`. // Perform `&` operation on existing mode with `0775`. // eg. if existing mode is `0777`, the mode will be changed to `0775` after following operation @@ -322,6 +321,7 @@ def configureLinuxPackage(DefaultTask packageTask, InstallerType installerType, } includeEmptyDirs = false } + into "${buildRoot}/usr/share/${installerType.baseName}" } } diff --git a/installers/osx.gradle b/installers/osx.gradle index 3dd737df07b6..a98801272cc4 100644 --- a/installers/osx.gradle +++ b/installers/osx.gradle @@ -1,8 +1,3 @@ -import com.thoughtworks.go.build.AdoptiumUrlHelper -import com.thoughtworks.go.build.DownloadFile -import com.thoughtworks.go.build.InstallerType -import org.apache.commons.codec.digest.DigestUtils - /* * Copyright 2021 ThoughtWorks, Inc. * @@ -19,34 +14,39 @@ import org.apache.commons.codec.digest.DigestUtils * limitations under the License. */ +import com.thoughtworks.go.build.* +import org.apache.commons.codec.digest.DigestUtils + private File destFile(String url) { new File(gradle.gradleUserHomeDir, "download-cache/${DigestUtils.md5Hex(url)}/${new File(new URL(url).path).name}") } -task downloadOsxJreChecksum(type: DownloadFile) { - def srcUrl = AdoptiumUrlHelper.sha256sumURL( - com.thoughtworks.go.build.OperatingSystem.mac, - project.packaging.adoptOpenjdk.featureVersion, - project.packaging.adoptOpenjdk.interimVersion, - project.packaging.adoptOpenjdk.updateVersion, - project.packaging.adoptOpenjdk.buildVersion - ) +def adoptiumVersion = new AdoptiumVersion( + OperatingSystem.mac, + project.packaging.adoptOpenjdk.featureVersion, + project.packaging.adoptOpenjdk.interimVersion, + project.packaging.adoptOpenjdk.updateVersion, + project.packaging.adoptOpenjdk.buildVersion +) + +task downloadOsxJdkChecksum(type: DownloadFile) { + def srcUrl = AdoptiumUrlHelper.sha256sumURL(adoptiumVersion) src srcUrl dest destFile(srcUrl) } -task downloadOsxJre(type: DownloadFile) { - dependsOn downloadOsxJreChecksum - def srcUrl = AdoptiumUrlHelper.downloadURL( - com.thoughtworks.go.build.OperatingSystem.mac, - project.packaging.adoptOpenjdk.featureVersion, - project.packaging.adoptOpenjdk.interimVersion, - project.packaging.adoptOpenjdk.updateVersion, - project.packaging.adoptOpenjdk.buildVersion - ) +task downloadOsxJdk(type: DownloadFile) { + dependsOn downloadOsxJdkChecksum + def srcUrl = AdoptiumUrlHelper.downloadURL(adoptiumVersion) src srcUrl dest destFile(srcUrl) - checksum = { downloadOsxJreChecksum.outputs.files.singleFile.getText("utf-8").trim().split(" ").first() } + checksum = { downloadOsxJdkChecksum.outputs.files.singleFile.getText("utf-8").trim().split(" ").first() } +} + +task buildOsxJre(type: AdoptiumJreTask) { + dependsOn downloadOsxJdk + version = adoptiumVersion + sourceJdkArchive = downloadOsxJdk.outputs.files.singleFile } def configureOSXZip(Zip zipTask, InstallerType installerType, Zip genericZipTask) { @@ -54,7 +54,7 @@ def configureOSXZip(Zip zipTask, InstallerType installerType, Zip genericZipTask group project.name description "Build the ${installerType.baseName} osx (zip) installer" - dependsOn downloadOsxJre + dependsOn buildOsxJre dependsOn genericZipTask destinationDirectory = file("${project.buildDir.path}/${project.distsDirName}/osx") archiveBaseName = installerType.baseName @@ -80,19 +80,14 @@ def configureOSXZip(Zip zipTask, InstallerType installerType, Zip genericZipTask include "${installerType.baseName}-${project.goVersion}/config/wrapper.conf" filter { String eachLine -> if (eachLine == 'wrapper.java.command=java') { - eachLine = 'wrapper.java.command=jre/Contents/Home/bin/java' + eachLine = 'wrapper.java.command=jre/bin/java' } eachLine } } // puts the content of the tar under `go-{agent,server}-XXX/jre` - from(tarTree(downloadOsxJre.outputs.files.singleFile)) { - eachFile { FileCopyDetails fcd -> - List segments = fcd.relativePath.segments - segments.set(1, "jre") - fcd.relativePath = new RelativePath(!fcd.isDirectory(), (segments as String[])) - } + from(buildOsxJre.outputDir) { into("${installerType.baseName}-${project.goVersion}") } } diff --git a/installers/windows.gradle b/installers/windows.gradle index 20e48819dd5d..191b5fc9afa9 100644 --- a/installers/windows.gradle +++ b/installers/windows.gradle @@ -14,8 +14,9 @@ * limitations under the License. */ - +import com.thoughtworks.go.build.AdoptiumJreTask import com.thoughtworks.go.build.AdoptiumUrlHelper +import com.thoughtworks.go.build.AdoptiumVersion import com.thoughtworks.go.build.DownloadFile import com.thoughtworks.go.build.InstallerType import org.apache.commons.codec.digest.DigestUtils @@ -25,30 +26,32 @@ private File destFile(String url) { new File(gradle.gradleUserHomeDir, "download-cache/${DigestUtils.md5Hex(url)}/${new File(new URL(url).path).name}") } -task downloadWindowsJreChecksum(type: DownloadFile) { - def srcUrl = AdoptiumUrlHelper.sha256sumURL( - com.thoughtworks.go.build.OperatingSystem.windows, - project.packaging.adoptOpenjdk.featureVersion, - project.packaging.adoptOpenjdk.interimVersion, - project.packaging.adoptOpenjdk.updateVersion, - project.packaging.adoptOpenjdk.buildVersion - ) +def adoptiumVersion = new AdoptiumVersion( + com.thoughtworks.go.build.OperatingSystem.windows, + project.packaging.adoptOpenjdk.featureVersion, + project.packaging.adoptOpenjdk.interimVersion, + project.packaging.adoptOpenjdk.updateVersion, + project.packaging.adoptOpenjdk.buildVersion +) + +task downloadWindowsJdkChecksum(type: DownloadFile) { + def srcUrl = AdoptiumUrlHelper.sha256sumURL(adoptiumVersion) src srcUrl dest destFile(srcUrl) } -task downloadWindowsJre(type: DownloadFile) { - dependsOn downloadWindowsJreChecksum - def srcUrl = AdoptiumUrlHelper.downloadURL( - com.thoughtworks.go.build.OperatingSystem.windows, - project.packaging.adoptOpenjdk.featureVersion, - project.packaging.adoptOpenjdk.interimVersion, - project.packaging.adoptOpenjdk.updateVersion, - project.packaging.adoptOpenjdk.buildVersion - ) +task downloadWindowsJdk(type: DownloadFile) { + dependsOn downloadWindowsJdkChecksum + def srcUrl = AdoptiumUrlHelper.downloadURL(adoptiumVersion) src srcUrl dest destFile(srcUrl) - checksum = { downloadWindowsJreChecksum.outputs.files.singleFile.getText("utf-8").trim().split(" ").first() } + checksum = { downloadWindowsJdkChecksum.outputs.files.singleFile.getText("utf-8").trim().split(" ").first() } +} + +task buildWindowsJre(type: AdoptiumJreTask) { + dependsOn downloadWindowsJdk + version = adoptiumVersion + sourceJdkArchive = downloadWindowsJdk.outputs.files.singleFile } def configureWindowsFilesystem(Task windowsInstallerTask, InstallerType installerType, Zip genericZipTask, File buildRoot) { @@ -56,7 +59,7 @@ def configureWindowsFilesystem(Task windowsInstallerTask, InstallerType installe group project.name description "Build the ${installerType.baseName} windows (exe) installer" - dependsOn downloadWindowsJre + dependsOn buildWindowsJre dependsOn genericZipTask doFirst { @@ -105,14 +108,7 @@ def configureWindowsFilesystem(Task windowsInstallerTask, InstallerType installe filter(FixCrLfFilter.class, eol: FixCrLfFilter.CrLf.DOS) } - from(zipTree(downloadWindowsJre.outputs.files.singleFile)) { - eachFile { FileCopyDetails fcd -> - List segments = fcd.relativePath.segments - segments.set(1, "jre") - fcd.relativePath = new RelativePath(!fcd.isDirectory(), (segments as String[])) - } - includeEmptyDirs = false - // relative to parent copy task's `into` + from(buildWindowsJre.outputDir) { into "${installerType.baseName}-${project.goVersion}" } }