From d0581ed3b3408bab40fe33edb16d7554ce79a1c4 Mon Sep 17 00:00:00 2001 From: Joel Rudsberg Date: Tue, 24 Sep 2024 12:28:04 +0200 Subject: [PATCH] Improve SBOM documentation and code quality --- .../sbom/ArtifactToPackageNameResolver.java | 26 +++- .../buildtools/maven/sbom/SBOMGenerator.java | 100 +++++++++---- .../maven/sbom/ShadedPackageNameResolver.java | 139 +++++++++++------- 3 files changed, 175 insertions(+), 90 deletions(-) diff --git a/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/sbom/ArtifactToPackageNameResolver.java b/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/sbom/ArtifactToPackageNameResolver.java index afb87bbb..ab8425b5 100644 --- a/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/sbom/ArtifactToPackageNameResolver.java +++ b/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/sbom/ArtifactToPackageNameResolver.java @@ -71,20 +71,40 @@ final class ArtifactToPackageNameResolver { this.shadedPackageNameResolver = new ShadedPackageNameResolver(mavenProject, mainClass); } - Set getArtifactPackageMappings() throws Exception { + /** + * Maps the artifacts of the maven project to {@link ArtifactAdapter}s. {@link ArtifactAdapter#packageNames} will + * be non-empty if package names could accurately be derived for an artifact. If not, it will be non-empty and + * {@link ArtifactAdapter#prunable} will be set to false. {@link ArtifactAdapter#prunable} will also be set to + * false if an artifact is not the main artifact and its part of a shaded jar. + * + * @return the artifacts of this project as {@link ArtifactAdapter}s. + * @throws Exception if an error was encountered when deriving the artifacts. + */ + Set getArtifactAdapters() throws Exception { Set artifactsWithPackageNameMappings = new HashSet<>(); List artifacts = new ArrayList<>(mavenProject.getArtifacts()); /* Purposefully add the project artifact last. This is important for the resolution of shaded jars. */ artifacts.add(mavenProject.getArtifact()); for (Artifact artifact : artifacts) { Optional optionalArtifact = resolvePackageNamesFromArtifact(artifact); - optionalArtifact.ifPresent(artifactsWithPackageNameMappings::add); + if (optionalArtifact.isPresent()) { + artifactsWithPackageNameMappings.add(optionalArtifact.get()); + } else { + /* If resolve failed, then there are no package name mappings, so we mark it as not prunable. */ + var artifactAdapter = ArtifactAdapter.fromMavenArtifact(artifact); + artifactAdapter.prunable = false; + artifactsWithPackageNameMappings.add(artifactAdapter); + } } + /* + * Currently we cannot ensure that package name are derived accurately for shaded dependencies. + * Thus, we mark such artifacts as non-prunable. + */ Set dependencies = artifactsWithPackageNameMappings.stream() .filter(v -> !v.equals(mavenProject.getArtifact())) .collect(Collectors.toSet()); - ShadedPackageNameResolver.markShadedDependencies(dependencies); + ShadedPackageNameResolver.markShadedArtifactsAsNonPrunable(dependencies); return artifactsWithPackageNameMappings; } diff --git a/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/sbom/SBOMGenerator.java b/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/sbom/SBOMGenerator.java index bafeaff4..c9981571 100644 --- a/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/sbom/SBOMGenerator.java +++ b/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/sbom/SBOMGenerator.java @@ -64,27 +64,21 @@ import static org.twdata.maven.mojoexecutor.MojoExecutor.*; /** - * Generates an enhanced Software Bill of Materials (SBOM) for Native Image consumption and refinement. + * Generates a Software Bill of Materials (SBOM) that is augmented and refined by Native Image. *

- * Process overview: - * 1. Utilizes the cyclonedx-maven-plugin to create a baseline SBOM. - * 2. Augments the baseline SBOM components with additional metadata (see {@link AddedComponentFields}): - * * "packageNames": A list of all package names associated with each component. - * * "jarPath": Path to the component jar. - * * "prunable": Boolean indicating if the component can be pruned. We currently set this to false for - * any dependencies to the main component that are shaded. - * 3. Stores the enhanced SBOM at a known location. - * 4. Native Image then processes this SBOM during its static analysis: - * * Unreachable components are removed. - * * Unnecessary dependency relationships are pruned. + * Approach: + * 1. The cyclonedx-maven-plugin creates a baseline SBOM. + * 2. The components of the baseline SBOM are updated with additional metadata, most importantly being the set of package + * names associated with the component (see {@link AddedComponentFields} for all additional metadata). + * 3. The SBOM is stored at a known location. + * 4. Native Image processes the SBOM and removes unreachable components and unnecessary dependencies. *

- * Creating the package-name-to-component mapping in the context of Native Image, without any build-system - * knowledge is difficult, which was the primary motivation for realizing this approach. + * Creating the package-name-to-component mapping in the context of Native Image, without the knowledge known at the + * plugin build-time is difficult, which was the primary motivation for realizing this approach. *

* Benefits: * * Great Baseline: Produces an industry-standard SBOM at minimum. - * * Enhanced Accuracy: Native Image static analysis refines the SBOM, - * potentially significantly improving its accuracy. + * * Enhanced Accuracy: Native Image augments and refines the SBOM, potentially significantly improving its accuracy. */ final public class SBOMGenerator { private final MavenProject mavenProject; @@ -94,12 +88,24 @@ final public class SBOMGenerator { private final String mainClass; private final Logger logger; + private static final String cycloneDXPluginName = "cyclonedx-maven-plugin"; private static final String SBOM_NAME = "WIP_SBOM"; private static final String FILE_FORMAT = "json"; private static final class AddedComponentFields { + /** + * The package names associated with this component. + */ static final String packageNames = "packageNames"; + /** + * The path to the jar containing the class files. For a component embedded in a shaded jar, the path must + * be pointing to the shaded jar. + */ static final String jarPath = "jarPath"; + /** + * If set to false, then this component and all its transitive dependencies SHOULD NOT be pruned by Native Image. + * This is set to false when the package names could not be derived accurately. + */ static final String prunable = "prunable"; } @@ -124,15 +130,16 @@ public SBOMGenerator( * @throws MojoExecutionException if SBOM creation fails. */ public void generate() throws MojoExecutionException { + String outputDirectory = mavenProject.getBuild().getDirectory(); + Path sbomPath = Paths.get(outputDirectory, SBOM_NAME + "." + FILE_FORMAT); try { - String outputDirectory = mavenProject.getBuild().getDirectory(); /* Suppress the output from the cyclonedx-maven-plugin. */ int loggingLevel = logger.getThreshold(); logger.setThreshold(Logger.LEVEL_DISABLED); executeMojo( plugin( groupId("org.cyclonedx"), - artifactId("cyclonedx-maven-plugin"), + artifactId(cycloneDXPluginName), version("2.8.1") ), goal("makeAggregateBom"), @@ -146,46 +153,77 @@ public void generate() throws MojoExecutionException { ); logger.setThreshold(loggingLevel); - Path sbomPath = Paths.get(outputDirectory, SBOM_NAME + "." + FILE_FORMAT); + if (!Files.exists(sbomPath)) { return; } + // TODO: debugging, remove before merge + Path unmodifiedPath = Paths.get(outputDirectory, "SBOM_UNMODIFIED.json"); + Files.deleteIfExists(unmodifiedPath); + Files.copy(sbomPath, unmodifiedPath); + var resolver = new ArtifactToPackageNameResolver(mavenProject, repositorySystem, mavenSession.getRepositorySession(), mainClass); - Set artifactsWithPackageNames = resolver.getArtifactPackageMappings(); - augmentSBOM(sbomPath, artifactsWithPackageNames); + Set artifacts = resolver.getArtifactAdapters(); + augmentSBOM(sbomPath, artifacts); + + // TODO: debugging, remove before merge + Path testPath = Paths.get(outputDirectory, "SBOM_AUGMENTED.json"); + Files.deleteIfExists(testPath); + Files.copy(sbomPath, testPath); + } catch (Exception exception) { + deleteFileIfExists(sbomPath); String errorMsg = String.format("Failed to create SBOM. Please try again and report this issue if it persists. " + "To bypass this failure, disable SBOM generation by setting %s to false.", NativeCompileNoForkMojo.enableSBOMParamName); throw new MojoExecutionException(errorMsg, exception); } } - private void augmentSBOM(Path sbomPath, Set artifactToPackageNames) throws IOException { + private static void deleteFileIfExists(Path sbomPath) { + try { + Files.deleteIfExists(sbomPath); + } catch (IOException e) { + /* Failed to delete file. */ + } + } + + /** + * Augments the base SBOM with information from the derived {@param artifacts}. + * + * @param baseSBOMPath path to the base SBOM generated by the cyclonedx plugin. + * @param artifacts artifacts that possibly have been extended with package name data. + */ + private void augmentSBOM(Path baseSBOMPath, Set artifacts) throws IOException { ObjectMapper objectMapper = new ObjectMapper(); - ObjectNode sbomJson = (ObjectNode) objectMapper.readTree(Files.newInputStream(sbomPath)); + ObjectNode sbomJson = (ObjectNode) objectMapper.readTree(Files.newInputStream(baseSBOMPath)); ArrayNode componentsArray = (ArrayNode) sbomJson.get("components"); if (componentsArray == null) { - return; + throw new RuntimeException(String.format("SBOM generated by %s contained no components.", cycloneDXPluginName)); } - /* - * Iterates over the components and finds the associated artifact by equality checks of the GAV coordinates. - * If a match is found, the component is augmented. - */ - componentsArray.forEach(componentNode -> augmentComponentNode(componentNode, artifactToPackageNames, objectMapper)); + /* Augment the "components" */ + componentsArray.forEach(componentNode -> augmentComponentNode(componentNode, artifacts, objectMapper)); /* Augment the main component in "metadata/component" */ JsonNode metadataNode = sbomJson.get("metadata"); if (metadataNode != null && metadataNode.has("component")) { - augmentComponentNode(metadataNode.get("component"), artifactToPackageNames, objectMapper); + augmentComponentNode(metadataNode.get("component"), artifacts, objectMapper); } /* Save the augmented SBOM back to the file */ - objectMapper.writerWithDefaultPrettyPrinter().writeValue(Files.newOutputStream(sbomPath), sbomJson); + objectMapper.writerWithDefaultPrettyPrinter().writeValue(Files.newOutputStream(baseSBOMPath), sbomJson); } + /** + * Updates the {@param componentNode} with {@link AddedComponentFields} from the artifact in {@param artifactsWithPackageNames} + * with matching GAV coordinates. + * + * @param componentNode the node in the base SBOM that should be augmented. + * @param artifactsWithPackageNames the artifact with information for {@link AddedComponentFields}. + * @param objectMapper the objectMapper that is used to write the updates. + */ private void augmentComponentNode(JsonNode componentNode, Set artifactsWithPackageNames, ObjectMapper objectMapper) { String groupField = "group"; String nameField = "name"; diff --git a/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/sbom/ShadedPackageNameResolver.java b/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/sbom/ShadedPackageNameResolver.java index 010afc08..43c4c51b 100644 --- a/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/sbom/ShadedPackageNameResolver.java +++ b/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/sbom/ShadedPackageNameResolver.java @@ -41,7 +41,6 @@ package org.graalvm.buildtools.maven.sbom; import org.apache.maven.model.Plugin; -import org.apache.maven.model.PluginExecution; import org.apache.maven.project.MavenProject; import org.apache.maven.shared.utils.xml.Xpp3Dom; @@ -50,7 +49,6 @@ import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.HashSet; -import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; @@ -59,8 +57,11 @@ class ShadedPackageNameResolver { private final MavenProject mavenProject; - private final Optional optionalShadePlugin; - /* + /** + * The shade plugin for this {@link ShadedPackageNameResolver#mavenProject} if used, otherwise null. + */ + private final Plugin shadePlugin; + /** * Set of possible directory paths containing class files in a jar file system. Examples of the keys are: * "org/json" and "org/apache/commons/collections/map". */ @@ -71,7 +72,7 @@ class ShadedPackageNameResolver { ShadedPackageNameResolver(MavenProject mavenProject, String mainClass) { this.mavenProject = mavenProject; - this.optionalShadePlugin = getShadePluginIfUsed(mavenProject); + this.shadePlugin = getShadePluginIfUsed(mavenProject); this.pathToClassFilesDirectories = new HashSet<>(); this.visitedPathToClassFileDirectories = new HashSet<>(); this.mainClass = mainClass; @@ -82,13 +83,13 @@ class ShadedPackageNameResolver { * to the fat or shaded jar, but instead to the local repository. This method tries to return a path to * the directory containing the class files inside the fat or shaded jar. If the artifact is not part of * a fat or shaded jar, {@param jarPath} is returned. - * + *

* NOTE: * - To improve chances of successful resolution, it is important to call this method with the main * artifact last. * - Should not be called with the same artifact more than once. - * - This only works if the main artifact is shaded, i.e. shaded dependencies are not handled. Currently, - * we disable any pruning by Native Image of shaded dependencies since we cannot guarantee its correctness. + * - Shaded dependencies to the main artifact are not handled. Currently, we disable any pruning by + * Native Image of shaded dependencies since we cannot guarantee its correctness. * * @param jarPath the jar path as reported by the original Artifact. * @param artifact the artifact with its class files inside the {@param jarPath}. @@ -100,13 +101,12 @@ Optional resolvePackageNamesFromPossiblyShadedJar(Path jarPath, } /* If the shade plugin is not used, then we are not dealing with a fat or shaded jar. */ - if (optionalShadePlugin.isEmpty()) { + if (shadePlugin == null) { return handleNonShadedCase(artifact, jarPath); } /* Recover the path to the shaded jar by querying the shade plugin object. */ - Plugin shadePlugin = optionalShadePlugin.get(); - Optional optionalShadedJarPath = getShadedJarPath(shadePlugin); + Optional optionalShadedJarPath = getShadedJarPath(); if (optionalShadedJarPath.isEmpty()) { return handleNonShadedCase(artifact, jarPath); } @@ -118,12 +118,12 @@ Optional resolvePackageNamesFromPossiblyShadedJar(Path jarPath, return handleNonShadedCase(artifact, jarPath); } - /* Attempt to derive which shaded directory contains the class files of this artifact. */ - Optional> optionalDirectories = resolveDirectoriesWithClasses(jarFileSystem, jarPath, artifact, shadePlugin); - if (optionalDirectories.isPresent()) { - Set containingDirectories = optionalDirectories.get(); + /* Derive the directories of this artifact containing class files and retrieve the package names from those files. */ + Optional> optionalClassFileDirectories = resolveArtifactClassFileDirectories(jarFileSystem, jarPath, artifact); + if (optionalClassFileDirectories.isPresent()) { + Set classFileDirectories = optionalClassFileDirectories.get(); Set packageNames = new HashSet<>(); - for (var directory : containingDirectories) { + for (var directory : classFileDirectories) { Set newPackageNames = FileWalkerUtility.walkFileSystemAndCollectPackageNames(jarFileSystem, directory) .orElse(Set.of()); packageNames.addAll(newPackageNames); @@ -131,19 +131,23 @@ Optional resolvePackageNamesFromPossiblyShadedJar(Path jarPath, artifact.setPackageNames(packageNames); artifact.setJarPath(shadedJarPath.toUri()); return Optional.of(artifact); - } - return handleNonShadedCase(artifact, jarPath); + } + return Optional.empty(); } - static void markShadedDependencies(Set dependencies) throws IOException { - for (ArtifactAdapter dependency : dependencies) { - if (isShaded(dependency)) { - dependency.prunable = false; + static void markShadedArtifactsAsNonPrunable(Set artifacts) throws IOException { + for (ArtifactAdapter artifact : artifacts) { + if (isShaded(artifact)) { + artifact.prunable = false; } } } private static boolean isShaded(ArtifactAdapter artifact) throws IOException { + if (artifact.jarPath == null) { + return false; + } + FileSystem jarFileSystem = getOrCreateFileSystem(Paths.get(artifact.jarPath)); Optional optionalMetaInfPath = getMetaInfArtifactPath(jarFileSystem, artifact); if (optionalMetaInfPath.isEmpty()) { @@ -169,7 +173,7 @@ private Optional handleNonShadedCase(ArtifactAdapter artifactAd return Optional.of(artifactAdapter); } - private Optional getShadedJarPath(Plugin shadePlugin) { + private Optional getShadedJarPath() { Path targetDirectory = Paths.get(mavenProject.getBuild().getDirectory()); Optional outputFile = getParameterFromPlugin(shadePlugin, "outputFile"); @@ -229,10 +233,51 @@ private static Optional getMetaInfArtifactPath(FileSystem jarFileSystem, A return Optional.of(path); } - private Optional> resolveDirectoriesWithClasses(FileSystem jarFileSystem, Path jarPath, ArtifactAdapter artifact, Plugin shadePlugin) throws IOException { - // TODO: this only handles cases where there's no relocation or where ALL files are relocated. Thus, partial or multiple relocations are not supported. + /** + * Finds the paths to the directories containing the class files for the {@param artifact} inside the jar. + * For example, if {@param artifact} represents commons-validator and the content of a fat jar looks like this: + * + * 

+     * org/
+     * ├── apache/
+     * │   ├── commons/
+     * │   │   ├── validator/
+     * │   │   │   ├── UrlValidator.class
+     * │   │   │   ├── routines/
+     * │   │   │   │   └── ValidatorUtils.class
+     * │   │   │   ├── util/
+     * │   │   │   │   └── Flags.class
+     * │   │   ├── digester/
+     * │   │   │   ├── plugins/
+     * │   │   │   │   └── strategies/
+     * │   │   │   │       └── DigesterPlugin.class
+     * ├── json/
+     * │   └── org/
+     * │       └── json/
+     * │           └── JSONObject.class
+     * META-INF/...
+     *
+ * + * Then the method would return only the path to the directories of commons-validator: + * + * 
+     * org/apache/commons/validator/
+     * org/apache/commons/validator/routines/
+     * org/apache/commons/validator/util/
+     *
+ * + * NOTE: partial relocations--when a subset of the class files are relocated--is not supported and + * {@link Optional#empty()} is returned in these cases. + * + * @param jarFileSystem The filesystem representing the JAR. + * @param jarPath The path within the JAR file to search. + * @param artifact The artifact whose class directories should be found. + * @return A list of paths containing the class files for the artifact. + * @throws IOException if an error occurs while reading the JAR. + */ + private Optional> resolveArtifactClassFileDirectories(FileSystem jarFileSystem, Path jarPath, ArtifactAdapter artifact) throws IOException { if (pathToClassFilesDirectories.isEmpty()) { - Set potentialDirectories = collectPotentialDirectories(jarFileSystem.getPath("/")); + Set potentialDirectories = resolveDirectoriesContainingClassFiles(jarFileSystem.getPath("/")); if (potentialDirectories.isEmpty()) { return Optional.empty(); } @@ -254,16 +299,16 @@ private Optional> resolveDirectoriesWithClasses(FileSystem jarFileSyst } /* - * If relocations are not used then matching directly with the GAV coordinates should work. + * Try to match directly with the GAV coordinates. */ - if (!areRelocationsUsed(shadePlugin)) { - Optional resolvedPath = tryResolveUsingGAVCoordinates(jarFileSystem.getPath("/"), artifact); - if (resolvedPath.isPresent()) return Optional.of(Set.of(resolvedPath.get())); + Optional resolvedPath = tryResolveUsingGAVCoordinates(jarFileSystem.getPath("/"), artifact); + if (resolvedPath.isPresent()) { + return Optional.of(Set.of(resolvedPath.get())); } boolean isMainArtifact = artifact.equals(mavenProject.getArtifact()); if (isMainArtifact) { - Optional resolvedPath = findTopClassDirectory(jarFileSystem.getPath("/"), mainClass); + resolvedPath = findTopClassDirectory(jarFileSystem.getPath("/"), mainClass); if (resolvedPath.isPresent()) { return Optional.of(Set.of(resolvedPath.get())); } @@ -375,14 +420,14 @@ private Optional> resolveDirectoriesFromClassNameMatching(ArtifactAdap return Optional.of(matchingDirectories); } - private Set collectPotentialDirectories(Path rootPath) throws IOException { - Set potentialDirectories = new HashSet<>(); + private Set resolveDirectoriesContainingClassFiles(Path rootPath) throws IOException { + Set directories = new HashSet<>(); Files.walkFileTree(rootPath, new SimpleFileVisitor<>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { if (file.toString().endsWith(".class")) { Path classDirectory = file.getParent(); - potentialDirectories.add(classDirectory); + directories.add(classDirectory); return FileVisitResult.CONTINUE; } return FileVisitResult.CONTINUE; @@ -397,7 +442,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { return FileVisitResult.CONTINUE; } }); - return potentialDirectories; + return directories; } private Path pathFromGAVCoordinates(Path basePath, ArtifactAdapter artifact, boolean useArtifactId) throws IOException { @@ -438,29 +483,11 @@ private static Optional getParameterFromPlugin(Plugin plugin, String par return Optional.empty(); } - private static boolean areRelocationsUsed(Plugin shadePlugin) { - List executions = shadePlugin.getExecutions(); - if (executions == null || executions.isEmpty()) { - return false; - } - - for (PluginExecution execution : executions) { - org.codehaus.plexus.util.xml.Xpp3Dom configuration = (org.codehaus.plexus.util.xml.Xpp3Dom) execution.getConfiguration(); - if (configuration != null) { - org.codehaus.plexus.util.xml.Xpp3Dom relocationsNode = configuration.getChild("relocations"); - if (relocationsNode != null && relocationsNode.getChildCount() > 0) { - return true; - } - } - } - - return false; - } - - private static Optional getShadePluginIfUsed(MavenProject mavenProject) { + private static Plugin getShadePluginIfUsed(MavenProject mavenProject) { return mavenProject.getBuildPlugins().stream() .filter(v -> mavenShadePluginName.equals(v.getArtifactId())) - .findFirst(); + .findFirst() + .orElse(null); } private static FileSystem getOrCreateFileSystem(Path jarPath) throws IOException {