Merge pull request #33749 from yrodiere/pull-strategy

Configurable strategy to control GraalVM build image pulling

core/deployment/src/main/java/io/quarkus/deployment/pkg/NativeConfig.java

@@ -14,6 +14,7 @@
 import io.smallrye.config.ConfigMapping;
 import io.smallrye.config.WithConverter;
 import io.smallrye.config.WithDefault;
+import io.smallrye.config.WithParentName;
 
 @ConfigRoot(phase = ConfigPhase.BUILD_TIME)
 @ConfigMapping(prefix = "quarkus.native")
@@ -208,21 +209,44 @@ default boolean isExplicitContainerBuild() {
     }
 
     /**
-     * The docker image to use to do the image build. It can be one of `graalvm`, `mandrel`, or the full image path, e.g.
-     * {@code quay.io/quarkus/ubi-quarkus-mandrel-builder-image:22.3-java17}.
+     * Configuration related to the builder image, when performing native builds in a container.
      */
-    @WithDefault("${platform.quarkus.native.builder-image}")
-    @ConfigDocDefault("mandrel")
-    String builderImage();
+    BuilderImageConfig builderImage();
 
-    default String getEffectiveBuilderImage() {
-        final String builderImageName = this.builderImage().toUpperCase();
-        if (builderImageName.equals(BuilderImageProvider.GRAALVM.name())) {
-            return DEFAULT_GRAALVM_BUILDER_IMAGE;
-        } else if (builderImageName.equals(BuilderImageProvider.MANDREL.name())) {
-            return DEFAULT_MANDREL_BUILDER_IMAGE;
-        } else {
-            return this.builderImage();
+    interface BuilderImageConfig {
+        /**
+         * The docker image to use to do the image build. It can be one of `graalvm`, `mandrel`, or the full image path, e.g.
+         * {@code quay.io/quarkus/ubi-quarkus-mandrel-builder-image:22.3-java17}.
+         */
+        @WithParentName
+        @WithDefault("${platform.quarkus.native.builder-image}")
+        @ConfigDocDefault("mandrel")
+        String image();
+
+        /**
+         * The strategy for pulling the builder image during the build.
+         * <p>
+         * Defaults to 'always', which will always pull the most up-to-date image;
+         * useful to keep up with fixes when a (floating) tag is updated.
+         * <p>
+         * Use 'missing' to only pull if there is no image locally;
+         * useful on development environments where building with out-of-date images is acceptable
+         * and bandwidth may be limited.
+         * <p>
+         * Use 'never' to fail the build if there is no image locally.
+         */
+        @WithDefault("always")
+        ImagePullStrategy pull();
+
+        default String getEffectiveImage() {
+            final String builderImageName = this.image().toUpperCase();
+            if (builderImageName.equals(BuilderImageProvider.GRAALVM.name())) {
+                return DEFAULT_GRAALVM_BUILDER_IMAGE;
+            } else if (builderImageName.equals(BuilderImageProvider.MANDREL.name())) {
+                return DEFAULT_MANDREL_BUILDER_IMAGE;
+            } else {
+                return this.image();
+            }
         }
     }
 
@@ -466,4 +490,25 @@ enum MonitoringOption {
         JMXCLIENT,
         ALL
     }
+
+    enum ImagePullStrategy {
+        /**
+         * Always pull the most recent image.
+         */
+        ALWAYS("always"),
+        /**
+         * Only pull the image if it's missing locally.
+         */
+        MISSING("missing"),
+        /**
+         * Never pull any image; fail if the image is missing locally.
+         */
+        NEVER("never");
+
+        public final String commandLineParamValue;
+
+        ImagePullStrategy(String commandLineParamValue) {
+            this.commandLineParamValue = commandLineParamValue;
+        }
+    }
 }

core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildContainerRunner.java

@@ -30,7 +30,8 @@ protected NativeImageBuildContainerRunner(NativeConfig nativeConfig) {
         this.nativeConfig = nativeConfig;
         containerRuntime = nativeConfig.containerRuntime().orElseGet(ContainerRuntimeUtil::detectContainerRuntime);
 
-        this.baseContainerRuntimeArgs = new String[] { "--env", "LANG=C", "--rm" };
+        this.baseContainerRuntimeArgs = new String[] { "--env", "LANG=C", "--rm",
+                "--pull", nativeConfig.builderImage().pull().commandLineParamValue };
 
         containerName = "build-native-" + RandomStringUtils.random(5, true, false);
     }
@@ -47,16 +48,51 @@ public void setup(boolean processInheritIODisabled) {
             // we pull the docker image in order to give users an indication of which step the process is at
             // it's not strictly necessary we do this, however if we don't the subsequent version command
             // will appear to block and no output will be shown
-            String effectiveBuilderImage = nativeConfig.getEffectiveBuilderImage();
-            log.info("Checking image status " + effectiveBuilderImage);
+            String effectiveBuilderImage = nativeConfig.builderImage().getEffectiveImage();
+            var builderImagePull = nativeConfig.builderImage().pull();
+            log.infof("Checking status of builder image '%s'", effectiveBuilderImage);
+            if (builderImagePull != NativeConfig.ImagePullStrategy.ALWAYS) {
+                Process imageInspectProcess = null;
+                try {
+                    final ProcessBuilder pb = new ProcessBuilder(
+                            Arrays.asList(containerRuntime.getExecutableName(), "image", "inspect",
+                                    "-f", "{{ .Id }}",
+                                    effectiveBuilderImage))
+                            // We only need the command's return status
+                            .redirectOutput(ProcessBuilder.Redirect.DISCARD)
+                            .redirectError(ProcessBuilder.Redirect.DISCARD);
+                    imageInspectProcess = pb.start();
+                    if (imageInspectProcess.waitFor() != 0) {
+                        if (builderImagePull == NativeConfig.ImagePullStrategy.NEVER) {
+                            throw new RuntimeException(
+                                    "Could not find builder image '" + effectiveBuilderImage
+                                            + "' locally and 'quarkus.native.builder-image.pull' is set to 'never'.");
+                        } else {
+                            log.infof("Could not find builder image '%s' locally, pulling the builder image",
+                                    effectiveBuilderImage);
+                        }
+                    } else {
+                        log.infof("Found builder image '%s' locally, skipping image pulling", effectiveBuilderImage);
+                        return;
+                    }
+                } catch (IOException | InterruptedException e) {
+                    throw new RuntimeException("Failed to check status of builder image '" + effectiveBuilderImage + "'", e);
+                } finally {
+                    if (imageInspectProcess != null) {
+                        imageInspectProcess.destroy();
+                    }
+                }
+            }
             Process pullProcess = null;
             try {
                 final ProcessBuilder pb = new ProcessBuilder(
                         Arrays.asList(containerRuntime.getExecutableName(), "pull", effectiveBuilderImage));
                 pullProcess = ProcessUtil.launchProcess(pb, processInheritIODisabled);
-                pullProcess.waitFor();
+                if (pullProcess.waitFor() != 0) {
+                    throw new RuntimeException("Failed to pull builder image '" + effectiveBuilderImage + "'");
+                }
             } catch (IOException | InterruptedException e) {
-                throw new RuntimeException("Failed to pull builder image " + effectiveBuilderImage, e);
+                throw new RuntimeException("Failed to pull builder image '" + effectiveBuilderImage + "'", e);
             } finally {
                 if (pullProcess != null) {
                     pullProcess.destroy();
@@ -123,7 +159,8 @@ protected List<String> getContainerRuntimeBuildArgs(Path outputDir) {
     protected String[] buildCommand(String dockerCmd, List<String> containerRuntimeArgs, List<String> command) {
         return Stream
                 .of(Stream.of(containerRuntime.getExecutableName()), Stream.of(dockerCmd), Stream.of(baseContainerRuntimeArgs),
-                        containerRuntimeArgs.stream(), Stream.of(nativeConfig.getEffectiveBuilderImage()), command.stream())
+                        containerRuntimeArgs.stream(), Stream.of(nativeConfig.builderImage().getEffectiveImage()),
+                        command.stream())
                 .flatMap(Function.identity()).toArray(String[]::new);
     }
 

core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildStep.java

@@ -149,7 +149,7 @@ ArtifactResultBuildItem nativeSourcesResult(NativeConfig nativeConfig,
             Files.writeString(outputDir.resolve("native-image.args"), String.join(" ", command));
             Files.writeString(outputDir.resolve("graalvm.version"), GraalVM.Version.CURRENT.version.toString());
             if (nativeImageRunner.isContainerBuild()) {
-                Files.writeString(outputDir.resolve("native-builder.image"), nativeConfig.getEffectiveBuilderImage());
+                Files.writeString(outputDir.resolve("native-builder.image"), nativeConfig.builderImage().getEffectiveImage());
             }
         } catch (IOException | RuntimeException e) {
             throw new RuntimeException("Failed to build native image sources", e);

core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/UpxCompressionBuildStep.java

@@ -44,7 +44,7 @@ public void compress(NativeConfig nativeConfig, NativeImageRunnerBuildItem nativ
             return;
         }
 
-        String effectiveBuilderImage = nativeConfig.getEffectiveBuilderImage();
+        String effectiveBuilderImage = nativeConfig.builderImage().getEffectiveImage();
         Optional<File> upxPathFromSystem = getUpxFromSystem();
         if (upxPathFromSystem.isPresent()) {
             log.debug("Running UPX from system path");

core/deployment/src/test/java/io/quarkus/deployment/pkg/NativeConfigTest.java

@@ -8,23 +8,23 @@ class NativeConfigTest {
 
     @Test
     public void testBuilderImageProperlyDetected() {
-        assertThat(createConfig("graalvm").getEffectiveBuilderImage()).contains("ubi-quarkus-graalvmce-builder-image")
+        assertThat(createConfig("graalvm").builderImage().getEffectiveImage()).contains("ubi-quarkus-graalvmce-builder-image")
                 .contains("java17");
-        assertThat(createConfig("GraalVM").getEffectiveBuilderImage()).contains("ubi-quarkus-graalvmce-builder-image")
+        assertThat(createConfig("GraalVM").builderImage().getEffectiveImage()).contains("ubi-quarkus-graalvmce-builder-image")
                 .contains("java17");
-        assertThat(createConfig("GraalVM").getEffectiveBuilderImage()).contains("ubi-quarkus-graalvmce-builder-image")
+        assertThat(createConfig("GraalVM").builderImage().getEffectiveImage()).contains("ubi-quarkus-graalvmce-builder-image")
                 .contains("java17");
-        assertThat(createConfig("GRAALVM").getEffectiveBuilderImage()).contains("ubi-quarkus-graalvmce-builder-image")
+        assertThat(createConfig("GRAALVM").builderImage().getEffectiveImage()).contains("ubi-quarkus-graalvmce-builder-image")
                 .contains("java17");
 
-        assertThat(createConfig("mandrel").getEffectiveBuilderImage()).contains("ubi-quarkus-mandrel-builder-image")
+        assertThat(createConfig("mandrel").builderImage().getEffectiveImage()).contains("ubi-quarkus-mandrel-builder-image")
                 .contains("java17");
-        assertThat(createConfig("Mandrel").getEffectiveBuilderImage()).contains("ubi-quarkus-mandrel-builder-image")
+        assertThat(createConfig("Mandrel").builderImage().getEffectiveImage()).contains("ubi-quarkus-mandrel-builder-image")
                 .contains("java17");
-        assertThat(createConfig("MANDREL").getEffectiveBuilderImage()).contains("ubi-quarkus-mandrel-builder-image")
+        assertThat(createConfig("MANDREL").builderImage().getEffectiveImage()).contains("ubi-quarkus-mandrel-builder-image")
                 .contains("java17");
 
-        assertThat(createConfig("aRandomString").getEffectiveBuilderImage()).isEqualTo("aRandomString");
+        assertThat(createConfig("aRandomString").builderImage().getEffectiveImage()).isEqualTo("aRandomString");
     }
 
     private NativeConfig createConfig(String builderImage) {

core/deployment/src/test/java/io/quarkus/deployment/pkg/TestNativeConfig.java

@@ -8,10 +8,18 @@
 
 public class TestNativeConfig implements NativeConfig {
 
-    private final String builderImage;
+    private final NativeConfig.BuilderImageConfig builderImage;
 
     public TestNativeConfig(String builderImage) {
-        this.builderImage = builderImage;
+        this(builderImage, ImagePullStrategy.ALWAYS);
+    }
+
+    public TestNativeConfig(ImagePullStrategy builderImagePull) {
+        this("mandrel", builderImagePull);
+    }
+
+    public TestNativeConfig(String builderImage, ImagePullStrategy builderImagePull) {
+        this.builderImage = new TestBuildImageConfig(builderImage, builderImagePull);
     }
 
     @Override
@@ -135,7 +143,7 @@ public boolean remoteContainerBuild() {
     }
 
     @Override
-    public String builderImage() {
+    public BuilderImageConfig builderImage() {
         return builderImage;
     }
 
@@ -203,4 +211,24 @@ public boolean enableDashboardDump() {
     public Compression compression() {
         return null;
     }
+
+    private class TestBuildImageConfig implements BuilderImageConfig {
+        private final String image;
+        private final ImagePullStrategy pull;
+
+        TestBuildImageConfig(String image, ImagePullStrategy pull) {
+            this.image = image;
+            this.pull = pull;
+        }
+
+        @Override
+        public String image() {
+            return image;
+        }
+
+        @Override
+        public ImagePullStrategy pull() {
+            return pull;
+        }
+    }
 }

integration-tests/maven/src/test/resources-filtered/projects/platform-properties-overrides/ext/deployment/src/main/java/org/acme/quarkus/sample/extension/deployment/AcmeExtensionProcessor.java

@@ -26,7 +26,7 @@ FeatureBuildItem feature() {
     SyntheticBeanBuildItem syntheticBean(ConfigReportRecorder recorder, NativeConfig nativeConfig) {
        return SyntheticBeanBuildItem.configure(ConfigReport.class)
     		   .scope(Singleton.class)
-    		   .runtimeValue(recorder.configReport(nativeConfig.builderImage()))
+    		   .runtimeValue(recorder.configReport(nativeConfig.builderImage().image()))
     		   .done();
     }
 }