Include buildtool support for bundling reachability metadata into jar…

… files (#322)

* Add DirectoryConfiguration.copy method

Update `DirectoryConfiguration` with a new `copy` method that
can be used to copy matadata files into `META-INF/native-image`
directories.

The `DirectoryConfiguration` class now has group, artifact and
version information so that it can write files into the
appropriate folder. The `copy` method will also write a
`reachability-metadata.properties` file that indicates if the
'override' property was set on the metadata.

* Add `add-metadata-hints` goal to Maven Plugin

Update the maven plugin with a new `add-metadata-hints` goal that can
be used to bundle hints obtained from the metadata repository into
the jar.

* Add `ReachabilityMetadataCopyTask` to Gradle Plugin

Update the gradle plugin with a new `ReachabilityMetadataCopyTask`
class that can be used to copy hints obtained from the metadata
repository.

* Refactor common code in NativeImagePlugin

Refactor the `configureJvmReachabilityConfigurationDirectories`
and `configureJvmReachabilityExcludeConfigArgs` to use a common
method since they share the same logic.

common/graalvm-reachability-metadata/src/main/java/org/graalvm/reachability/DirectoryConfiguration.java

@@ -40,15 +40,34 @@
  */
 package org.graalvm.reachability;
 
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Collection;
 
 public class DirectoryConfiguration {
 
+    private static final String PROPERTIES = "reachability-metadata.properties";
+
+    private final String groupId;
+
+    private final String artifactId;
+
+    private final String version;
+
     private final Path directory;
 
     private final boolean override;
 
-    public DirectoryConfiguration(Path directory, boolean override) {
+    public DirectoryConfiguration(String groupId, String artifactId, String version, Path directory, boolean override) {
+        this.groupId = groupId;
+        this.artifactId = artifactId;
+        this.version = version;
         this.directory = directory;
         this.override = override;
     }
@@ -60,4 +79,49 @@ public Path getDirectory() {
     public boolean isOverride() {
         return override;
     }
+
+    public static void copy(Collection<DirectoryConfiguration> configurations, Path destinationDirectory) throws IOException {
+        Path nativeImageDestination = destinationDirectory.resolve("META-INF").resolve("native-image");
+        for (DirectoryConfiguration configuration : configurations) {
+            Path target = nativeImageDestination
+                    .resolve(configuration.groupId)
+                    .resolve(configuration.artifactId)
+                    .resolve((configuration.version != null) ? configuration.version :
+                            configuration.getDirectory().getFileName().toString());
+            copyFileTree(configuration.directory, target);
+            writeConfigurationProperties(configuration, target);
+        }
+    }
+
+    private static void copyFileTree(Path source, Path target) throws IOException {
+        Files.walkFileTree(source, new SimpleFileVisitor<Path>() {
+
+            @Override
+            public FileVisitResult preVisitDirectory(Path directory, BasicFileAttributes attrs) throws IOException {
+                Files.createDirectories(target.resolve(source.relativize(directory)));
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                if (!"index.json".equalsIgnoreCase(file.getFileName().toString())) {
+                    Files.copy(file, target.resolve(source.relativize(file)), StandardCopyOption.REPLACE_EXISTING);
+                }
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+
+    private static void writeConfigurationProperties(DirectoryConfiguration configuration, Path target)
+            throws IOException {
+        StringBuilder content = new StringBuilder();
+        if (configuration.isOverride()) {
+            content.append("override=true\n");
+        }
+        if (content.length() > 0) {
+            Files.write(target.resolve(PROPERTIES), content.toString().getBytes(StandardCharsets.ISO_8859_1));
+        }
+    }
 }
+
+

common/graalvm-reachability-metadata/src/main/java/org/graalvm/reachability/internal/FileSystemRepository.java

@@ -112,7 +112,7 @@ public Set<DirectoryConfiguration> findConfigurationsFor(Consumer<? super Query>
                                 Optional<DirectoryConfiguration> configuration = index.findConfiguration(groupId, artifactId, version);
                                 if (!configuration.isPresent() && artifactQuery.isUseLatestVersion()) {
                                     logger.log(groupId, artifactId, version, "Configuration directory not found. Trying latest version.");
-                                    configuration = index.findLatestConfigurationFor(groupId, artifactId);
+                                    configuration = index.findLatestConfigurationFor(groupId, artifactId, version);
                                     if (!configuration.isPresent()) {
                                         logger.log(groupId, artifactId, version, "Latest version not found!");
                                     }

common/graalvm-reachability-metadata/src/main/java/org/graalvm/reachability/internal/index/artifacts/SingleModuleJsonVersionToConfigDirectoryIndex.java

@@ -91,7 +91,13 @@ private Map<String, List<Artifact>> parseIndexFile(Path rootPath) {
      */
     @Override
     public Optional<DirectoryConfiguration> findConfiguration(String groupId, String artifactId, String version) {
-        return findConfigurationFor(groupId, artifactId, artifact -> artifact.getVersions().contains(version));
+        return findConfigurationFor(groupId, artifactId, version, artifact -> artifact.getVersions().contains(version));
+    }
+
+    @Override
+    @Deprecated
+    public Optional<DirectoryConfiguration> findLatestConfigurationFor(String groupId, String artifactId) {
+        return findConfigurationFor(groupId, artifactId, null, Artifact::isLatest);
     }
 
     /**
@@ -102,11 +108,12 @@ public Optional<DirectoryConfiguration> findConfiguration(String groupId, String
      * @return a configuration directory, or empty if no configuration directory is available
      */
     @Override
-    public Optional<DirectoryConfiguration> findLatestConfigurationFor(String groupId, String artifactId) {
-        return findConfigurationFor(groupId, artifactId, Artifact::isLatest);
+    public Optional<DirectoryConfiguration> findLatestConfigurationFor(String groupId, String artifactId, String version) {
+        return findConfigurationFor(groupId, artifactId, version, Artifact::isLatest);
     }
 
-    private Optional<DirectoryConfiguration> findConfigurationFor(String groupId, String artifactId, Predicate<? super Artifact> predicate) {
+    private Optional<DirectoryConfiguration> findConfigurationFor(String groupId, String artifactId, String version,
+            Predicate<? super Artifact> predicate) {
         String module = groupId + ":" + artifactId;
         List<Artifact> artifacts = index.get(module);
         if (artifacts == null) {
@@ -116,7 +123,8 @@ private Optional<DirectoryConfiguration> findConfigurationFor(String groupId, St
                 .filter(artifact -> artifact.getModule().equals(module))
                 .filter(predicate)
                 .findFirst()
-                .map(artifact -> new DirectoryConfiguration(moduleRoot.resolve(artifact.getDirectory()), artifact.isOverride()));
+                .map(artifact -> new DirectoryConfiguration(groupId, artifactId, version,
+                        moduleRoot.resolve(artifact.getDirectory()), artifact.isOverride()));
     }
 
 }

common/graalvm-reachability-metadata/src/main/java/org/graalvm/reachability/internal/index/artifacts/VersionToConfigDirectoryIndex.java

@@ -60,6 +60,17 @@ public interface VersionToConfigDirectoryIndex {
      * @param groupId the group ID of the artifact
      * @param artifactId the artifact ID of the artifact
      * @return a configuration, or empty if no configuration directory is available
+     * @deprecated in favor of {@link #findLatestConfigurationFor(String, String, String)}
      */
+    @Deprecated
     Optional<DirectoryConfiguration> findLatestConfigurationFor(String groupId, String artifactId);
+
+    /**
+     * Returns the latest configuration for the requested artifact.
+     * @param groupId the group ID of the artifact
+     * @param artifactId the artifact ID of the artifact
+     * @param version the version of the artifact
+     * @return a configuration, or empty if no configuration directory is available
+     */
+    Optional<DirectoryConfiguration> findLatestConfigurationFor(String groupId, String artifactId, String version);
 }

common/graalvm-reachability-metadata/src/test/java/org/graalvm/reachability/DirectoryConfigurationTest.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2020, 2022 Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.graalvm.reachability;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Properties;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class DirectoryConfigurationTest {
+
+    @TempDir
+    Path temp;
+
+    @Test
+    void copyCopiesFiles() throws IOException {
+        Path directory = temp.resolve("source/com.example.group/artifact/123");
+        Path target = temp.resolve("target");
+        createJsonFiles(directory);
+        DirectoryConfiguration configuration = new DirectoryConfiguration("com.example.group", "artifact", "123", directory, false);
+        DirectoryConfiguration.copy(Arrays.asList(configuration), target);
+        assertTrue(Files.exists(target.resolve("META-INF/native-image/com.example.group/artifact/123/reflect-config.json")));
+        assertTrue(Files.exists(target.resolve("META-INF/native-image/com.example.group/artifact/123/other.json")));
+        assertFalse(Files.exists(target.resolve("META-INF/native-image/com.example.group/artifact/123/index.json")));
+        assertFalse(Files.exists(target.resolve("META-INF/native-image/com.example.group/artifact/123/reachability-metadata.properties")));
+    }
+
+    @Test
+    void copyWhenHasOverrideGeneratesMetadataRepositoryProperties() throws IOException {
+        Path directory = temp.resolve("source/com.example.group/artifact/123");
+        Path target = temp.resolve("target");
+        createJsonFiles(directory);
+        DirectoryConfiguration configuration = new DirectoryConfiguration("com.example.group", "artifact", "123", directory, true);
+        DirectoryConfiguration.copy(Arrays.asList(configuration), target);
+        Path propertiesFile = target.resolve("META-INF/native-image/com.example.group/artifact/123/reachability-metadata.properties");
+        Properties properties = new Properties();
+        properties.load(new FileInputStream(propertiesFile.toFile()));
+        assertEquals("true", properties.getProperty("override"));
+    }
+
+    private void createJsonFiles(Path directory) throws IOException {
+        Files.createDirectories(directory);
+        byte[] json = "{}".getBytes();
+        Files.copy(new ByteArrayInputStream(json), directory.resolve("index.json"));
+        Files.copy(new ByteArrayInputStream(json), directory.resolve("reflect-config.json"));
+        Files.copy(new ByteArrayInputStream(json), directory.resolve("other.json"));
+    }
+
+}

common/graalvm-reachability-metadata/src/test/java/org/graalvm/reachability/internal/index/artifacts/SingleModuleJsonVersionToConfigDirectoryIndexTest.java

@@ -88,7 +88,7 @@ void checkIndex() throws URISyntaxException {
         config = index.findConfiguration("com.foo", "nope", "1.0");
         assertFalse(config.isPresent());
 
-        Optional<DirectoryConfiguration> latest = index.findLatestConfigurationFor("com.foo", "bar");
+        Optional<DirectoryConfiguration> latest = index.findLatestConfigurationFor("com.foo", "bar", "123");
         assertTrue(latest.isPresent());
         assertEquals(repoPath.resolve("2.0"), latest.get().getDirectory());
         assertTrue(latest.get().isOverride());

docs/src/docs/asciidoc/gradle-plugin.adoc

@@ -411,6 +411,32 @@ include::../snippets/gradle/groovy/build.gradle[tags=specify-metadata-version-fo
 include::../snippets/gradle/kotlin/build.gradle.kts[tags=specify-metadata-version-for-library]
 ----
 
+=== Including metadata repository files
+
+By default, reachability metadata will be used only when your native image is generated.
+In some situations, you may want a copy of the reachability metadata to use directly.
+
+For example, copying the reachability metadata into your jar can be useful when some other process is responsible for converting your jar into a native image.
+You might be generating a shaded jar and using a https://paketo.io/[Paketo buildpack] to convert it to a native image.
+
+To download a copy of the metadata into the `build/native-reachability-metadata` directory you can the `collectReachabilityMetadata` task.
+Files will be downloaded into `META-INF/native-image/<groupId>/<versionId>` subdirectories.
+
+To include metadata repository inside your jar you can link to the task using the `jar` DSL `from` directive:
+
+.Including metadata repository files
+[source, groovy, role="multi-language-sample"]
+----
+include::../snippets/gradle/groovy/build.gradle[tags=include-metadata]
+----
+
+[source, kotlin, role="multi-language-sample"]
+----
+include::../snippets/gradle/kotlin/build.gradle.kts[tags=include-metadata]
+----
+
+For more advanced configurations you can declare a `org.graalvm.buildtools.gradle.tasks.CollectReachabilityMetadata` task and set the appropriate properties.
+
 [[plugin-configurations]]
 == Configurations defined by the plugin
 

docs/src/docs/asciidoc/maven-plugin.adoc

@@ -660,6 +660,23 @@ This may be interesting if there's no specific metadata available for the partic
 include::../../../../samples/native-config-integration/pom.xml[tag=metadata-force-version]
 ----
 
+=== Adding metadata respoistory files
+
+By default, repository metadata will be used only when your native image is generated.
+In some situations, you may want to include the metadata directly inside your jar.
+
+Adding metadata to your jar can be useful when some other process is responsible for converting your jar into a native image.
+For example, you might be generating a shaded jar and using a https://paketo.io/[Paketo buildpack] to convert it to a native image.
+
+To include metadata repository inside your jar you can use the `add-reachability-metadata` goal.
+Typically the goal will be included in an execution step where by default it will be bound to the `generate-resources` phase:
+
+.Configuring the `add-reachability-metadata` goal to execute with the `generate-resources` phase
+[source,xml,indent=0]
+----
+include::../../../../samples/native-config-integration/pom.xml[tag=add-reachability-metadata-execution]
+----
+
 [[javadocs]]
 == Javadocs
 

docs/src/docs/snippets/gradle/groovy/build.gradle

@@ -207,3 +207,9 @@ graalvmNative {
     }
 }
 // end::specify-metadata-version-for-library[]
+
+// tag::include-metadata[]
+tasks.named("jar") {
+    from collectReachabilityMetadata
+}
+// end::include-metadata[]

docs/src/docs/snippets/gradle/kotlin/build.gradle.kts

@@ -221,3 +221,9 @@ graalvmNative {
     }
 }
 // end::specify-metadata-version-for-library[]
+
+// tag::include-metadata[]
+tasks.named("jar", Jar) {
+	from(collectReachabilityMetadata)
+}
+// end::include-metadata[]