Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable caching for quarkus:build and declare dependencies as inputs #37622

Merged
merged 10 commits into from
Feb 16, 2024
5 changes: 5 additions & 0 deletions .github/workflows/ci-actions-incremental.yml
Original file line number Diff line number Diff line change
@@ -1072,6 +1072,11 @@ jobs:
build-scan-capture-strategy: ON_DEMAND
job-name: "Native Tests - ${{matrix.category}}"
wrapper-init: true
- name: Cache Quarkus metadata
uses: actions/cache@v3
with:
path: '**/.quarkus/quarkus-prod-config-dump'
key: ${{ runner.os }}-quarkus-metadata
- name: Build
env:
TEST_MODULES: ${{matrix.test-modules}}
5 changes: 5 additions & 0 deletions .mvn/extensions.xml
Original file line number Diff line number Diff line change
@@ -9,4 +9,9 @@
<artifactId>common-custom-user-data-maven-extension</artifactId>
<version>1.12.5</version>
</extension>
<extension>
<groupId>com.gradle</groupId>
<artifactId>quarkus-build-caching-extension</artifactId>
<version>0.10</version>
</extension>
</extensions>
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
package io.quarkus.maven;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Method;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.zip.Adler32;
import java.util.zip.Checksum;

import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
@@ -18,6 +25,7 @@
import io.quarkus.bootstrap.app.CuratedApplication;
import io.quarkus.bootstrap.classloading.QuarkusClassLoader;
import io.quarkus.bootstrap.model.ApplicationModel;
import io.quarkus.maven.dependency.DependencyFlags;
import io.quarkus.runtime.LaunchMode;

/**
@@ -58,6 +66,18 @@ public class TrackConfigChangesMojo extends QuarkusBootstrapMojo {
@Parameter(defaultValue = "false", property = "quarkus.track-config-changes.dump-current-when-recorded-unavailable")
boolean dumpCurrentWhenRecordedUnavailable;

/**
* Whether to dump Quarkus application dependencies along with their checksums
*/
@Parameter(defaultValue = "true", property = "quarkus.track-config-changes.dump-dependencies")
boolean dumpDependencies;

/**
* Dependency dump file
*/
@Parameter(property = "quarkus.track-config-changes.dependencies-file")
File dependenciesFile;

@Override
protected boolean beforeExecute() throws MojoExecutionException, MojoFailureException {
if (skip) {
@@ -82,16 +102,6 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException {
getLog().debug("Bootstrapping Quarkus application in mode " + launchMode);
}

Path targetFile;
if (outputFile == null) {
targetFile = outputDirectory.toPath()
.resolve("quarkus-" + launchMode.getDefaultProfile() + "-config-check");
} else if (outputFile.isAbsolute()) {
targetFile = outputFile.toPath();
} else {
targetFile = outputDirectory.toPath().resolve(outputFile.toPath());
}

Path compareFile;
if (this.recordedBuildConfigFile == null) {
compareFile = recordedBuildConfigDirectory.toPath()
@@ -102,34 +112,64 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException {
compareFile = recordedBuildConfigDirectory.toPath().resolve(this.recordedBuildConfigFile.toPath());
}

final Properties compareProps = new Properties();
if (Files.exists(compareFile)) {
try (BufferedReader reader = Files.newBufferedReader(compareFile)) {
compareProps.load(reader);
} catch (IOException e) {
throw new RuntimeException("Failed to read " + compareFile, e);
}
} else if (!dumpCurrentWhenRecordedUnavailable) {
getLog().info(compareFile + " not found");
final boolean prevConfigExists = Files.exists(compareFile);
if (!prevConfigExists && !dumpCurrentWhenRecordedUnavailable && !dumpDependencies) {
getLog().info("Config dump from the previous build does not exist at " + compareFile);
return;
}

CuratedApplication curatedApplication = null;
QuarkusClassLoader deploymentClassLoader = null;
final ClassLoader originalCl = Thread.currentThread().getContextClassLoader();
Properties actualProps;
final boolean clearPackageTypeSystemProperty = setPackageTypeSystemPropertyIfNativeProfileEnabled();
try {
curatedApplication = bootstrapApplication(launchMode);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FWIW, I benchmarked things a bit and resolving the dependencies is quite slow. Even for a relatively simple IT as the Hibernate Validator one, it takes ~ 0.8 s.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is, this is something i've been looking into for the last few weeks. But there is not much we can do especially for relatively simple projects, since most of the time is spent in the Maven artifact resolver itself.

However, the app model resolution will typically be necessary either way. It's about at which point it's performed. Here, we are moving this step to an earlier phase and it's cached for later phases if they get executed.

deploymentClassLoader = curatedApplication.createDeploymentClassLoader();
Thread.currentThread().setContextClassLoader(deploymentClassLoader);

final Class<?> codeGenerator = deploymentClassLoader.loadClass("io.quarkus.deployment.CodeGenerator");
final Method dumpConfig = codeGenerator.getMethod("dumpCurrentConfigValues", ApplicationModel.class, String.class,
Properties.class, QuarkusClassLoader.class, Properties.class, Path.class);
dumpConfig.invoke(null, curatedApplication.getApplicationModel(),
launchMode.name(), getBuildSystemProperties(true),
deploymentClassLoader, compareProps, targetFile);
if (prevConfigExists || dumpCurrentWhenRecordedUnavailable) {
final Path targetFile = getOutputFile(outputFile, launchMode.getDefaultProfile(), "-config-check");
Properties compareProps = new Properties();
if (prevConfigExists) {
try (BufferedReader reader = Files.newBufferedReader(compareFile)) {
compareProps.load(reader);
} catch (IOException e) {
throw new RuntimeException("Failed to read " + compareFile, e);
}
}

deploymentClassLoader = curatedApplication.createDeploymentClassLoader();
Thread.currentThread().setContextClassLoader(deploymentClassLoader);

final Class<?> codeGenerator = deploymentClassLoader.loadClass("io.quarkus.deployment.CodeGenerator");
final Method dumpConfig = codeGenerator.getMethod("dumpCurrentConfigValues", ApplicationModel.class,
String.class,
Properties.class, QuarkusClassLoader.class, Properties.class, Path.class);
dumpConfig.invoke(null, curatedApplication.getApplicationModel(),
launchMode.name(), getBuildSystemProperties(true),
deploymentClassLoader, compareProps, targetFile);
}

if (dumpDependencies) {
final List<String> deps = new ArrayList<>();
for (var d : curatedApplication.getApplicationModel().getDependencies(DependencyFlags.DEPLOYMENT_CP)) {
gsmet marked this conversation as resolved.
Show resolved Hide resolved
StringBuilder entry = new StringBuilder(d.toGACTVString());
if (d.isSnapshot()) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I decided to avoid computing the checksum if the dependency is not a snapshot one. They should be immutable in this case.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Final versions could still be built locally. For example before releasing but not necessarily then.

var adler32 = new Adler32();
updateChecksum(adler32, d.getResolvedPaths());
entry.append(" ").append(adler32.getValue());
}

deps.add(entry.toString());
}
Collections.sort(deps);
final Path targetFile = getOutputFile(dependenciesFile, launchMode.getDefaultProfile(),
"-dependency-checksums.txt");
Files.createDirectories(targetFile.getParent());
try (BufferedWriter writer = Files.newBufferedWriter(targetFile)) {
for (var s : deps) {
writer.write(s);
writer.newLine();
}
}
}
} catch (Exception any) {
throw new MojoExecutionException("Failed to bootstrap Quarkus application", any);
} finally {
@@ -142,4 +182,44 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException {
}
}
}

private Path getOutputFile(File outputFile, String profile, String fileNameSuffix) {
if (outputFile == null) {
return outputDirectory.toPath().resolve("quarkus-" + profile + fileNameSuffix);
}
if (outputFile.isAbsolute()) {
return outputFile.toPath();
}
return outputDirectory.toPath().resolve(outputFile.toPath());
}

private static void updateChecksum(Checksum checksum, Iterable<Path> pc) throws IOException {
for (var path : sort(pc)) {
if (Files.isDirectory(path)) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
updateChecksum(checksum, stream);
}
} else {
checksum.update(Files.readAllBytes(path));
}
}
}

private static Iterable<Path> sort(Iterable<Path> original) {
var i = original.iterator();
if (!i.hasNext()) {
return List.of();
}
var o = i.next();
if (!i.hasNext()) {
return List.of(o);
}
final List<Path> sorted = new ArrayList<>();
sorted.add(o);
while (i.hasNext()) {
sorted.add(i.next());
}
Collections.sort(sorted);
return sorted;
}
}
5 changes: 5 additions & 0 deletions docs/src/main/asciidoc/config-reference.adoc
Original file line number Diff line number Diff line change
@@ -738,6 +738,11 @@
The `track-config-changes` goal looks for `${project.basedir}/.quarkus/quarkus-prod-config-dump` (file name and directory are configurable) and, if the file already exists, checks whether the values stored in the config dump have changed.
It will log the changed options and save the current values of each of the options present in `${project.basedir}/.quarkus/quarkus-prod-config-dump` in `${project.basedir}/target/quarkus-prod-config.check` (the target file name and location can be configured). If the build time configuration has not changed since the last build both `${project.basedir}/.quarkus/quarkus-prod-config-dump` and `${project.basedir}/.quarkus/quarkus-prod-config-dump` will be identical.

==== Dump Quarkus application dependencies

In addition to dumping configuration values, `track-config-changes` goal also dumps all the Quarkus application dependencies, including Quarkus build time dependencies, along with their checksums (Adler32). This file could be used to check whether Quarkus build classpath has changed since the previous run.
By default, the dependency checksums will be stored under `target/quarkus-prod-dependency-checksums.txt` file. A different location could be configured using plugin parameters.

Check warning on line 744 in docs/src/main/asciidoc/config-reference.adoc

GitHub Actions / Linting with Vale

[vale] reported by reviewdog 🐶 [Quarkus.TermsSuggestions] Depending on the context, consider using 'by using' or 'that uses' rather than 'using'. Raw Output: {"message": "[Quarkus.TermsSuggestions] Depending on the context, consider using 'by using' or 'that uses' rather than 'using'.", "location": {"path": "docs/src/main/asciidoc/config-reference.adoc", "range": {"start": {"line": 744, "column": 152}}}, "severity": "INFO"}

==== Dump current build configuration when the recorded configuration isn't found

By default, `track-config-changes` looks for the configuration recorded during previous build and does nothing if it's not found. Enabling `dumpCurrentWhenRecordedUnavailable` parameter will make it dump the current build configuration
Original file line number Diff line number Diff line change
@@ -42,6 +42,10 @@ default boolean isJar() {
return TYPE_JAR.equals(getType());
}

default boolean isSnapshot() {
return getVersion() != null && getVersion().endsWith("-SNAPSHOT");
}

default String toGACTVString() {
return getGroupId() + ":" + getArtifactId() + ":" + getClassifier() + ":" + getType() + ":" + getVersion();
}
57 changes: 57 additions & 0 deletions independent-projects/parent/pom.xml
Original file line number Diff line number Diff line change
@@ -424,6 +424,63 @@
</runtimeClassPath>
</normalization>
<plugins>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<inputs>
<fileSets>
<fileSet>
<name>dependency-checksums</name>
<paths>
<path>${project.build.directory}</path>
</paths>
<includes>
<include>quarkus-*-dependency-checksums.txt</include>
</includes>
<normalization>RELATIVE_PATH</normalization>
</fileSet>
</fileSets>
</inputs>
</plugin>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<inputs>
<fileSets>
<fileSet>
<name>dependency-checksums</name>
<paths>
<path>${project.build.directory}</path>
</paths>
<includes>
<include>quarkus-*-dependency-checksums.txt</include>
</includes>
<normalization>RELATIVE_PATH</normalization>
</fileSet>
</fileSets>
</inputs>
</plugin>
<plugin>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-maven-plugin</artifactId>
<executions>
<execution>
<id>default</id>
<inputs>
<fileSets>
<fileSet>
<name>dependency-checksums</name>
<paths>
<path>${project.build.directory}</path>
</paths>
<includes>
<include>quarkus-*-dependency-checksums.txt</include>
</includes>
<normalization>RELATIVE_PATH</normalization>
</fileSet>
</fileSets>
</inputs>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-maven-plugin</artifactId>
13 changes: 13 additions & 0 deletions integration-tests/main/pom.xml
Original file line number Diff line number Diff line change
@@ -23,6 +23,9 @@

<!-- do not update this dependency, it is only used for testing -->
<webjar.jquery-ui.version>1.13.0</webjar.jquery-ui.version>

<quarkus.config-tracking.enabled>true</quarkus.config-tracking.enabled>
<quarkus.native.container-build>true</quarkus.native.container-build>
zakkak marked this conversation as resolved.
Show resolved Hide resolved
</properties>

<dependencies>
@@ -507,6 +510,16 @@
<groupId>io.quarkus</groupId>
<artifactId>quarkus-maven-plugin</artifactId>
<executions>
<execution>
<id>track-config-changes</id>
<phase>process-resources</phase>
<goals>
<goal>track-config-changes</goal>
</goals>
<configuration>
<dumpCurrentWhenRecordedUnavailable>true</dumpCurrentWhenRecordedUnavailable>
</configuration>
</execution>
<execution>
<goals>
<goal>build</goal>
13 changes: 12 additions & 1 deletion integration-tests/pom.xml
Original file line number Diff line number Diff line change
@@ -70,6 +70,17 @@
<groupId>io.quarkus</groupId>
<artifactId>quarkus-maven-plugin</artifactId>
<version>${project.version}</version>
<executions>
<execution>
<!-- for now this execution will only dump Quarkus application
dependency checksums that could indicate build time classpath changes -->
<id>track-config-changes</id>
<phase>process-resources</phase>
<goals>
<goal>track-config-changes</goal>
</goals>
</execution>
</executions>
<configuration>
<noDeps>true</noDeps>
<skip>${quarkus.build.skip}</skip>
@@ -304,7 +315,7 @@
<module>amazon-lambda-rest-resteasy-reactive</module>
<module>amazon-lambda-rest-reactive-routes</module>
<module>amazon-lambda-rest-funqy</module>
<module>amazon-lambda-rest-servlet</module>
<module>amazon-lambda-rest-servlet</module>
<module>amazon-lambda-http-resteasy</module>
<module>amazon-lambda-http-resteasy-reactive</module>
<module>container-image</module>