Add gradle plugin for downloading jdk (elastic#41461)

We currently download 3 variants of the same version of the jdk for
bundling into the distributions. Additionally, the vagrant images do
their own downloading. This commit moves the jdk downloading into a
utility gradle plugin. This will be used in a future PR by the packaging
tests.

The new plugin exposes a "jdks" project extension which allows creating
named jdks. Once the jdk version and platform are set for a named jdk,
the jdk object may be used as a lazy String for the jdk home path, or a
file collection for copying.

buildSrc/build.gradle

@@ -124,6 +124,7 @@ dependencies {
   compile 'com.avast.gradle:gradle-docker-compose-plugin:0.8.12'
   testCompile "junit:junit:${props.getProperty('junit')}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
+  testCompile 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
 }
 
 /*****************************************************************************

buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java

@@ -0,0 +1,112 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.gradle.api.Buildable;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.TaskDependency;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.regex.Pattern;
+
+public class Jdk implements Buildable, Iterable<File> {
+
+    static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+)(@([a-f0-9]{32}))?");
+    private static final List<String> ALLOWED_PLATFORMS = Collections.unmodifiableList(Arrays.asList("linux", "windows", "darwin"));
+
+    private final String name;
+    private final Configuration configuration;
+
+    private final Property<String> version;
+    private final Property<String> platform;
+
+
+    Jdk(String name, Project project) {
+        this.name = name;
+        this.configuration = project.getConfigurations().create("jdk_" + name);
+        this.version = project.getObjects().property(String.class);
+        this.platform = project.getObjects().property(String.class);
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getVersion() {
+        return version.get();
+    }
+
+    public void setVersion(String version) {
+        if (VERSION_PATTERN.matcher(version).matches() == false) {
+            throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]");
+        }
+        this.version.set(version);
+    }
+
+    public String getPlatform() {
+        return platform.get();
+    }
+
+    public void setPlatform(String platform) {
+        if (ALLOWED_PLATFORMS.contains(platform) == false) {
+            throw new IllegalArgumentException(
+                "unknown platform [" + platform + "] for jdk [" + name + "], must be one of " + ALLOWED_PLATFORMS);
+        }
+        this.platform.set(platform);
+    }
+
+    // pkg private, for internal use
+    Configuration getConfiguration() {
+        return configuration;
+    }
+
+    @Override
+    public String toString() {
+        return configuration.getSingleFile().toString();
+    }
+
+    @Override
+    public TaskDependency getBuildDependencies() {
+        return configuration.getBuildDependencies();
+    }
+
+    // internal, make this jdks configuration unmodifiable
+    void finalizeValues() {
+        if (version.isPresent() == false) {
+            throw new IllegalArgumentException("version not specified for jdk [" + name + "]");
+        }
+        if (platform.isPresent() == false) {
+            throw new IllegalArgumentException("platform not specified for jdk [" + name + "]");
+        }
+        version.finalizeValue();
+        platform.finalizeValue();
+    }
+
+    @Override
+    public Iterator<File> iterator() {
+        return configuration.iterator();
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java

@@ -0,0 +1,170 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.gradle.api.Action;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.UnknownTaskException;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.ConfigurationContainer;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
+import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
+import org.gradle.api.file.CopySpec;
+import org.gradle.api.file.FileTree;
+import org.gradle.api.file.RelativePath;
+import org.gradle.api.tasks.Copy;
+import org.gradle.api.tasks.TaskProvider;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.function.Supplier;
+import java.util.regex.Matcher;
+
+public class JdkDownloadPlugin implements Plugin<Project> {
+
+    @Override
+    public void apply(Project project) {
+        NamedDomainObjectContainer<Jdk> jdksContainer = project.container(Jdk.class, name ->
+            new Jdk(name, project)
+        );
+        project.getExtensions().add("jdks", jdksContainer);
+
+        project.afterEvaluate(p -> {
+            for (Jdk jdk : jdksContainer) {
+                jdk.finalizeValues();
+                String version = jdk.getVersion();
+                String platform = jdk.getPlatform();
+
+                // depend on the jdk directory "artifact" from the root project
+                DependencyHandler dependencies = project.getDependencies();
+                Map<String, Object> depConfig = new HashMap<>();
+                depConfig.put("path", ":"); // root project
+                depConfig.put("configuration", configName("extracted_jdk", version, platform));
+                dependencies.add(jdk.getConfiguration().getName(), dependencies.project(depConfig));
+
+                // ensure a root level jdk download task exists
+                setupRootJdkDownload(project.getRootProject(), platform, version);
+            }
+        });
+    }
+
+    private static void setupRootJdkDownload(Project rootProject, String platform, String version) {
+        String extractTaskName = "extract" + capitalize(platform) + "Jdk" + version;
+        // NOTE: this is *horrendous*, but seems to be the only way to check for the existence of a registered task
+        try {
+            rootProject.getTasks().named(extractTaskName);
+            // already setup this version
+            return;
+        } catch (UnknownTaskException e) {
+            // fall through: register the task
+        }
+
+        // decompose the bundled jdk version, broken into elements as: [feature, interim, update, build]
+        // Note the "patch" version is not yet handled here, as it has not yet been used by java.
+        Matcher jdkVersionMatcher = Jdk.VERSION_PATTERN.matcher(version);
+        if (jdkVersionMatcher.matches() == false) {
+            throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
+        }
+        String jdkVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : "");
+        String jdkMajor = jdkVersionMatcher.group(1);
+        String jdkBuild = jdkVersionMatcher.group(3);
+        String hash = jdkVersionMatcher.group(5);
+
+        // add fake ivy repo for jdk url
+        String repoName = "jdk_repo_" + version;
+        if (rootProject.getRepositories().findByName(repoName) == null) {
+            // simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
+            rootProject.getRepositories().ivy(ivyRepo -> {
+                ivyRepo.setName(repoName);
+                ivyRepo.setUrl("https://download.oracle.com");
+                ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
+                ivyRepo.patternLayout(layout ->
+                    layout.artifact("java/GA/jdk" + jdkMajor + "/" + jdkBuild + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]"));
+                ivyRepo.content(content -> content.includeGroup("jdk"));
+            });
+            // current pattern since 12.0.1
+            rootProject.getRepositories().ivy(ivyRepo -> {
+                ivyRepo.setName(repoName + "_with_hash");
+                ivyRepo.setUrl("https://download.oracle.com");
+                ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
+                ivyRepo.patternLayout(layout -> layout.artifact(
+                    "java/GA/jdk" + jdkVersion + "/" + hash + "/" + jdkBuild + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]"));
+                ivyRepo.content(content -> content.includeGroup("jdk"));
+            });
+        }
+
+        // add the jdk as a "dependency"
+        final ConfigurationContainer configurations = rootProject.getConfigurations();
+        String remoteConfigName = configName("openjdk", version, platform);
+        String localConfigName = configName("extracted_jdk", version, platform);
+        Configuration jdkConfig = configurations.findByName(remoteConfigName);
+        if (jdkConfig == null) {
+            jdkConfig = configurations.create(remoteConfigName);
+            configurations.create(localConfigName);
+        }
+        String extension = platform.equals("windows") ? "zip" : "tar.gz";
+        String jdkDep = "jdk:" + (platform.equals("darwin") ? "osx" : platform) + ":" + jdkVersion + "@" + extension;
+        rootProject.getDependencies().add(configName("openjdk", version, platform), jdkDep);
+
+        // add task for extraction
+        // TODO: look into doing this as an artifact transform, which are cacheable starting in gradle 5.3
+        int rootNdx = platform.equals("darwin") ? 2 : 1;
+        Action<CopySpec> removeRootDir = copy -> {
+            // remove extra unnecessary directory levels
+            copy.eachFile(details -> {
+                String[] pathSegments = details.getRelativePath().getSegments();
+                String[] newPathSegments = Arrays.copyOfRange(pathSegments, rootNdx, pathSegments.length);
+                details.setRelativePath(new RelativePath(true, newPathSegments));
+            });
+            copy.setIncludeEmptyDirs(false);
+        };
+        // delay resolving jdkConfig until runtime
+        Supplier<File> jdkArchiveGetter = jdkConfig::getSingleFile;
+        final Callable<FileTree> fileGetter;
+        if (extension.equals("zip")) {
+            fileGetter = () -> rootProject.zipTree(jdkArchiveGetter.get());
+        } else {
+            fileGetter = () -> rootProject.tarTree(rootProject.getResources().gzip(jdkArchiveGetter.get()));
+        }
+        String extractDir = rootProject.getBuildDir().toPath().resolve("jdks/openjdk-" + jdkVersion + "_" + platform).toString();
+        TaskProvider<Copy> extractTask = rootProject.getTasks().register(extractTaskName, Copy.class, copyTask -> {
+            copyTask.doFirst(t -> rootProject.delete(extractDir));
+            copyTask.into(extractDir);
+            copyTask.from(fileGetter, removeRootDir);
+        });
+        rootProject.getArtifacts().add(localConfigName,
+            rootProject.getLayout().getProjectDirectory().dir(extractDir),
+            artifact -> artifact.builtBy(extractTask));
+    }
+
+    private static String configName(String prefix, String version, String platform) {
+        return prefix + "_" + version + "_" + platform;
+    }
+
+    private static String capitalize(String s) {
+        return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1);
+    }
+}

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.jdk-download.properties

@@ -0,0 +1 @@
+implementation-class=org.elasticsearch.gradle.JdkDownloadPlugin