From f512b9b5dd03489f77baa528405fbb62a92a4cc7 Mon Sep 17 00:00:00 2001 From: George Gastaldi Date: Fri, 7 May 2021 12:42:21 -0300 Subject: [PATCH] Introduce UberJarMergedResourceBuildItem and UberJarIgnoredResourceBuildItem Fixes #5677 --- .../UberJarIgnoredResourceBuildItem.java | 20 +++++++ .../UberJarMergedResourceBuildItem.java | 20 +++++++ .../pkg/steps/JarResultBuildStep.java | 60 ++++++++++++++++--- .../deployment/UberJarConfigBuildStep.java | 22 +++++++ .../UberJarIgnoredResourceBuildItemTest.java | 52 ++++++++++++++++ .../UberJarMergedResourceBuildItemTest.java | 53 ++++++++++++++++ 6 files changed, 219 insertions(+), 8 deletions(-) create mode 100644 core/deployment/src/main/java/io/quarkus/deployment/pkg/builditem/UberJarIgnoredResourceBuildItem.java create mode 100644 core/deployment/src/main/java/io/quarkus/deployment/pkg/builditem/UberJarMergedResourceBuildItem.java create mode 100644 core/test-extension/deployment/src/main/java/io/quarkus/extest/deployment/UberJarConfigBuildStep.java create mode 100644 core/test-extension/deployment/src/test/java/io/quarkus/deployment/pkg/builditem/UberJarIgnoredResourceBuildItemTest.java create mode 100644 core/test-extension/deployment/src/test/java/io/quarkus/deployment/pkg/builditem/UberJarMergedResourceBuildItemTest.java diff --git a/core/deployment/src/main/java/io/quarkus/deployment/pkg/builditem/UberJarIgnoredResourceBuildItem.java b/core/deployment/src/main/java/io/quarkus/deployment/pkg/builditem/UberJarIgnoredResourceBuildItem.java new file mode 100644 index 00000000000000..00551260490b19 --- /dev/null +++ b/core/deployment/src/main/java/io/quarkus/deployment/pkg/builditem/UberJarIgnoredResourceBuildItem.java @@ -0,0 +1,20 @@ +package io.quarkus.deployment.pkg.builditem; + +import io.quarkus.builder.item.MultiBuildItem; +import io.smallrye.common.constraint.Assert; + +/** + * Ignore resources when building an Uber Jar + */ +public final class UberJarIgnoredResourceBuildItem extends MultiBuildItem { + + private final String path; + + public UberJarIgnoredResourceBuildItem(String path) { + this.path = Assert.checkNotEmptyParam("UberJarIgnoredResourceBuildItem.path", path); + } + + public String getPath() { + return path; + } +} diff --git a/core/deployment/src/main/java/io/quarkus/deployment/pkg/builditem/UberJarMergedResourceBuildItem.java b/core/deployment/src/main/java/io/quarkus/deployment/pkg/builditem/UberJarMergedResourceBuildItem.java new file mode 100644 index 00000000000000..d38546c55316db --- /dev/null +++ b/core/deployment/src/main/java/io/quarkus/deployment/pkg/builditem/UberJarMergedResourceBuildItem.java @@ -0,0 +1,20 @@ +package io.quarkus.deployment.pkg.builditem; + +import io.quarkus.builder.item.MultiBuildItem; +import io.smallrye.common.constraint.Assert; + +/** + * Merge duplicate resources from multiple JARs when building an Uber Jar + */ +public final class UberJarMergedResourceBuildItem extends MultiBuildItem { + + private final String path; + + public UberJarMergedResourceBuildItem(String path) { + this.path = Assert.checkNotEmptyParam("UberJarMergedResourceBuildItem.path", path); + } + + public String getPath() { + return path; + } +} diff --git a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java index cba8cad2f3ce72..92d924d1a5154b 100644 --- a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java +++ b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java @@ -77,6 +77,8 @@ import io.quarkus.deployment.pkg.builditem.LegacyJarRequiredBuildItem; import io.quarkus.deployment.pkg.builditem.NativeImageSourceJarBuildItem; import io.quarkus.deployment.pkg.builditem.OutputTargetBuildItem; +import io.quarkus.deployment.pkg.builditem.UberJarIgnoredResourceBuildItem; +import io.quarkus.deployment.pkg.builditem.UberJarMergedResourceBuildItem; import io.quarkus.deployment.pkg.builditem.UberJarRequiredBuildItem; import io.quarkus.deployment.util.FileUtil; @@ -135,21 +137,37 @@ public boolean test(String path) { }; private static final Logger log = Logger.getLogger(JarResultBuildStep.class); + private static final BiPredicate IS_JSON_FILE_PREDICATE = new IsJsonFilePredicate(); + public static final String DEPLOYMENT_CLASS_PATH_DAT = "deployment-class-path.dat"; + public static final String BUILD_SYSTEM_PROPERTIES = "build-system.properties"; + public static final String DEPLOYMENT_LIB = "deployment"; + public static final String APPMODEL_DAT = "appmodel.dat"; + public static final String QUARKUS_RUN_JAR = "quarkus-run.jar"; + public static final String QUARKUS_APP_DEPS = "quarkus-app-dependencies.txt"; + public static final String BOOT_LIB = "boot"; + public static final String LIB = "lib"; + public static final String MAIN = "main"; + public static final String GENERATED_BYTECODE_JAR = "generated-bytecode.jar"; + public static final String TRANSFORMED_BYTECODE_JAR = "transformed-bytecode.jar"; + public static final String APP = "app"; + public static final String QUARKUS = "quarkus"; + public static final String DEFAULT_FAST_JAR_DIRECTORY_NAME = "quarkus-app"; + public static final String MP_CONFIG_FILE = "META-INF/microprofile-config.properties"; @BuildStep @@ -188,6 +206,8 @@ public JarBuildItem buildRunnerJar(CurateOutcomeBuildItem curateOutcomeBuildItem List generatedClasses, List generatedResources, List uberJarRequired, + List uberJarMergedResourceBuildItems, + List uberJarIgnoredResourceBuildItems, List legacyJarRequired, QuarkusBuildCloseablesBuildItem closeablesBuildItem, List additionalApplicationArchiveBuildItems, @@ -205,7 +225,8 @@ public JarBuildItem buildRunnerJar(CurateOutcomeBuildItem curateOutcomeBuildItem if (legacyJarRequired.isEmpty() && (!uberJarRequired.isEmpty() || packageConfig.type.equalsIgnoreCase(PackageConfig.UBER_JAR))) { return buildUberJar(curateOutcomeBuildItem, outputTargetBuildItem, transformedClasses, applicationArchivesBuildItem, - packageConfig, applicationInfo, generatedClasses, generatedResources, mainClassBuildItem); + packageConfig, applicationInfo, generatedClasses, generatedResources, uberJarMergedResourceBuildItems, + uberJarIgnoredResourceBuildItems, mainClassBuildItem); } else if (!legacyJarRequired.isEmpty() || packageConfig.isLegacyJar() || packageConfig.type.equalsIgnoreCase(PackageConfig.LEGACY)) { return buildLegacyThinJar(curateOutcomeBuildItem, outputTargetBuildItem, transformedClasses, @@ -252,6 +273,8 @@ private JarBuildItem buildUberJar(CurateOutcomeBuildItem curateOutcomeBuildItem, ApplicationInfoBuildItem applicationInfo, List generatedClasses, List generatedResources, + List mergeResources, + List ignoredResources, MainClassBuildItem mainClassBuildItem) throws Exception { //we use the -runner jar name, unless we are building both types @@ -267,6 +290,8 @@ private JarBuildItem buildUberJar(CurateOutcomeBuildItem curateOutcomeBuildItem, applicationInfo, generatedClasses, generatedResources, + mergeResources, + ignoredResources, mainClassBuildItem, runnerJar); @@ -292,6 +317,8 @@ private void buildUberJar0(CurateOutcomeBuildItem curateOutcomeBuildItem, ApplicationInfoBuildItem applicationInfo, List generatedClasses, List generatedResources, + List mergedResources, + List ignoredResources, MainClassBuildItem mainClassBuildItem, Path runnerJar) throws Exception { try (FileSystem runnerZipFs = ZipUtils.newZip(runnerJar)) { @@ -301,8 +328,14 @@ private void buildUberJar0(CurateOutcomeBuildItem curateOutcomeBuildItem, final Map seen = new HashMap<>(); final Map> duplicateCatcher = new HashMap<>(); final Map> concatenatedEntries = new HashMap<>(); + final Set mergeResourcePaths = mergedResources.stream() + .map(UberJarMergedResourceBuildItem::getPath) + .collect(Collectors.toSet()); Set finalIgnoredEntries = new HashSet<>(IGNORED_ENTRIES); packageConfig.userConfiguredIgnoredEntries.ifPresent(finalIgnoredEntries::addAll); + ignoredResources.stream() + .map(UberJarIgnoredResourceBuildItem::getPath) + .forEach(finalIgnoredEntries::add); final List appDeps = curateOutcomeBuildItem.getEffectiveModel().getUserDependencies(); @@ -328,12 +361,13 @@ private void buildUberJar0(CurateOutcomeBuildItem curateOutcomeBuildItem, try (FileSystem artifactFs = ZipUtils.newFileSystem(resolvedDep)) { for (final Path root : artifactFs.getRootDirectories()) { walkFileDependencyForDependency(root, runnerZipFs, seen, duplicateCatcher, concatenatedEntries, - finalIgnoredEntries, appDep, transformedFromThisArchive); + finalIgnoredEntries, appDep, transformedFromThisArchive, mergeResourcePaths); } } } else { walkFileDependencyForDependency(resolvedDep, runnerZipFs, seen, duplicateCatcher, - concatenatedEntries, finalIgnoredEntries, appDep, transformedFromThisArchive); + concatenatedEntries, finalIgnoredEntries, appDep, transformedFromThisArchive, + mergeResourcePaths); } } } @@ -382,7 +416,8 @@ private static boolean includeAppDep(AppDependency appDep, Optional seen, Map> duplicateCatcher, Map> concatenatedEntries, - Set finalIgnoredEntries, AppDependency appDep, Set transformedFromThisArchive) throws IOException { + Set finalIgnoredEntries, AppDependency appDep, Set transformedFromThisArchive, + Set mergeResourcePaths) throws IOException { final Path metaInfDir = root.resolve("META-INF"); Files.walkFileTree(root, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new SimpleFileVisitor() { @@ -414,7 +449,8 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) boolean transformed = transformedFromThisArchive != null && transformedFromThisArchive.contains(relativePath); if (!transformed) { - if (CONCATENATED_ENTRIES_PREDICATE.test(relativePath)) { + if (CONCATENATED_ENTRIES_PREDICATE.test(relativePath) + || mergeResourcePaths.contains(relativePath)) { concatenatedEntries.computeIfAbsent(relativePath, (u) -> new ArrayList<>()) .add(Files.readAllBytes(file)); return FileVisitResult.CONTINUE; @@ -845,7 +881,6 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) /** * Native images are built from a specially created jar file. This allows for changes in how the jar file is generated. - * */ @BuildStep public NativeImageSourceJarBuildItem buildNativeImageJar(CurateOutcomeBuildItem curateOutcomeBuildItem, @@ -858,7 +893,9 @@ public NativeImageSourceJarBuildItem buildNativeImageJar(CurateOutcomeBuildItem List nativeImageResources, List generatedResources, MainClassBuildItem mainClassBuildItem, - List uberJarRequired) throws Exception { + List uberJarRequired, + List mergeResources, + List ignoreResources) throws Exception { Path targetDirectory = outputTargetBuildItem.getOutputDirectory() .resolve(outputTargetBuildItem.getBaseName() + "-native-image-source-jar"); IoUtils.createOrEmptyDir(targetDirectory); @@ -877,7 +914,9 @@ public NativeImageSourceJarBuildItem buildNativeImageJar(CurateOutcomeBuildItem final NativeImageSourceJarBuildItem nativeImageSourceJarBuildItem = buildNativeImageUberJar(curateOutcomeBuildItem, outputTargetBuildItem, transformedClasses, applicationArchivesBuildItem, - packageConfig, applicationInfo, allClasses, generatedResources, mainClassBuildItem, targetDirectory); + packageConfig, applicationInfo, allClasses, generatedResources, mergeResources, + ignoreResources, mainClassBuildItem, + targetDirectory); // additionally copy any json config files to a location accessible by native-image tool during // native-image generation copyJsonConfigFiles(applicationArchivesBuildItem, targetDirectory); @@ -926,6 +965,8 @@ private NativeImageSourceJarBuildItem buildNativeImageUberJar(CurateOutcomeBuild ApplicationInfoBuildItem applicationInfo, List generatedClasses, List generatedResources, + List mergeResources, + List ignoreResources, MainClassBuildItem mainClassBuildItem, Path targetDirectory) throws Exception { //we use the -runner jar name, unless we are building both types @@ -940,6 +981,8 @@ private NativeImageSourceJarBuildItem buildNativeImageUberJar(CurateOutcomeBuild applicationInfo, generatedClasses, generatedResources, + mergeResources, + ignoreResources, mainClassBuildItem, runnerJar); @@ -1126,6 +1169,7 @@ private void copyCommonContent(FileSystem runnerZipFs, Map> for (Map.Entry> entry : concatenatedEntries.entrySet()) { try (final OutputStream os = wrapForJDK8232879( Files.newOutputStream(runnerZipFs.getPath(entry.getKey())))) { + // TODO: Handle merging of XMLs for (byte[] i : entry.getValue()) { os.write(i); os.write('\n'); diff --git a/core/test-extension/deployment/src/main/java/io/quarkus/extest/deployment/UberJarConfigBuildStep.java b/core/test-extension/deployment/src/main/java/io/quarkus/extest/deployment/UberJarConfigBuildStep.java new file mode 100644 index 00000000000000..5299b7832ebb8c --- /dev/null +++ b/core/test-extension/deployment/src/main/java/io/quarkus/extest/deployment/UberJarConfigBuildStep.java @@ -0,0 +1,22 @@ +package io.quarkus.extest.deployment; + +import io.quarkus.deployment.annotations.BuildStep; +import io.quarkus.deployment.pkg.builditem.UberJarIgnoredResourceBuildItem; +import io.quarkus.deployment.pkg.builditem.UberJarMergedResourceBuildItem; + +/** + * Used in UberJarMergedResourceBuildItemTest + */ +public class UberJarConfigBuildStep { + + @BuildStep + UberJarMergedResourceBuildItem uberJarMergedResourceBuildItem() { + return new UberJarMergedResourceBuildItem("META-INF/cxf/bus-extensions.txt"); + } + + @BuildStep + UberJarIgnoredResourceBuildItem uberJarIgnoredResourceBuildItem() { + return new UberJarIgnoredResourceBuildItem("META-INF/cxf/cxf.fixml"); + } + +} diff --git a/core/test-extension/deployment/src/test/java/io/quarkus/deployment/pkg/builditem/UberJarIgnoredResourceBuildItemTest.java b/core/test-extension/deployment/src/test/java/io/quarkus/deployment/pkg/builditem/UberJarIgnoredResourceBuildItemTest.java new file mode 100644 index 00000000000000..e6660b46520607 --- /dev/null +++ b/core/test-extension/deployment/src/test/java/io/quarkus/deployment/pkg/builditem/UberJarIgnoredResourceBuildItemTest.java @@ -0,0 +1,52 @@ +package io.quarkus.deployment.pkg.builditem; + +import java.io.IOException; +import java.net.URL; +import java.util.Collections; +import java.util.List; + +import io.quarkus.bootstrap.model.AppArtifact; +import io.quarkus.runtime.Quarkus; +import io.quarkus.runtime.annotations.QuarkusMain; +import io.quarkus.test.QuarkusProdModeTest; +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.spec.JavaArchive; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.assertj.core.api.Assertions.assertThat; + +class UberJarIgnoredResourceBuildItemTest { + + @RegisterExtension + static final QuarkusProdModeTest runner = new QuarkusProdModeTest() + .setArchiveProducer(() -> ShrinkWrap.create(JavaArchive.class) + .addAsManifestResource("application.properties", "microprofile-config.properties") + .addClass(UberJarMain.class)) + .setApplicationName("uber-jar-merge") + .setApplicationVersion("0.1-SNAPSHOT") + .setRun(true) + .setExpectExit(true) + .overrideConfigKey("quarkus.package.type", "uber-jar") + .setForcedDependencies( + Collections.singletonList( + // META-INF/cxf/cxf.fixml should be present in the cxf-rt-transports-http and cxf-core JARs + new AppArtifact("org.apache.cxf", "cxf-rt-transports-http", "3.4.3"))); + + @Test + public void testResourceWasIgnored() throws IOException { + assertThat(runner.getStartupConsoleOutput()).contains("RESOURCES: 0"); + assertThat(runner.getExitCode()).isZero(); + } + + @QuarkusMain + public static class UberJarMain { + + public static void main(String[] args) throws IOException { + List resources = Collections + .list(UberJarMain.class.getClassLoader().getResources("META-INF/cxf/cxf.fixml")); + System.out.println("RESOURCES: " + resources.size()); + } + + } +} diff --git a/core/test-extension/deployment/src/test/java/io/quarkus/deployment/pkg/builditem/UberJarMergedResourceBuildItemTest.java b/core/test-extension/deployment/src/test/java/io/quarkus/deployment/pkg/builditem/UberJarMergedResourceBuildItemTest.java new file mode 100644 index 00000000000000..854ac42d5b6892 --- /dev/null +++ b/core/test-extension/deployment/src/test/java/io/quarkus/deployment/pkg/builditem/UberJarMergedResourceBuildItemTest.java @@ -0,0 +1,53 @@ +package io.quarkus.deployment.pkg.builditem; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.net.URL; +import java.util.Collections; +import java.util.List; + +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.spec.JavaArchive; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import io.quarkus.bootstrap.model.AppArtifact; +import io.quarkus.runtime.Quarkus; +import io.quarkus.runtime.annotations.QuarkusMain; +import io.quarkus.test.QuarkusProdModeTest; + +class UberJarMergedResourceBuildItemTest { + + @RegisterExtension + static final QuarkusProdModeTest runner = new QuarkusProdModeTest() + .setArchiveProducer(() -> ShrinkWrap.create(JavaArchive.class) + .addAsManifestResource("application.properties", "microprofile-config.properties") + .addClass(UberJarMain.class)) + .setApplicationName("uber-jar-merge") + .setApplicationVersion("0.1-SNAPSHOT") + .setRun(true) + .setExpectExit(true) + .overrideConfigKey("quarkus.package.type", "uber-jar") + .setForcedDependencies( + Collections.singletonList( + // META-INF/cxf/bus-extensions.txt should be present in the cxf-rt-transports-http and cxf-core JARs + new AppArtifact("org.apache.cxf", "cxf-rt-transports-http", "3.4.3"))); + + @Test + public void testResourceWasMerged() throws IOException { + assertThat(runner.getStartupConsoleOutput()).contains("RESOURCES: 1"); + assertThat(runner.getExitCode()).isZero(); + } + + @QuarkusMain + public static class UberJarMain { + + public static void main(String[] args) throws IOException { + List resources = Collections + .list(UberJarMain.class.getClassLoader().getResources("META-INF/cxf/bus-extensions.txt")); + System.out.println("RESOURCES: " + resources.size()); + } + + } +}