CI yaml fixed after PR 60 changes, Whitelist regexp fix

  FAIL_ON_PERF_REGRESSION: false

.github/workflows/builder_image_tests.yml

@@ -14,6 +14,7 @@ on:
 
 env:
   LANG: en_US.UTF-8
+  FAIL_ON_PERF_REGRESSION: false
 
 jobs:
   local-run:
@@ -22,15 +23,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        quarkus-version: ['2.2.3.Final', '2.4.0.Final']
+        quarkus-version: ['1.11.7.Final', '2.2.3.Final', '2.4.0.Final']
         mandrel-builder-image: [
+            '20.3-java11',
             '21.2-java11',
             '21.3-java11'
         ]
         os: [ubuntu-20.04]
-        # exclude:
-        #   - quarkus-version: '2.2.3.Final'
-        #     mandrel-builder-image: '21.2-java11'
+        exclude:
+          - quarkus-version: '2.2.3.Final'
+            mandrel-builder-image: '20.3-java11'
+          - quarkus-version: '2.4.0.Final'
+            mandrel-builder-image: '20.3-java11'
+          - quarkus-version: '1.11.7.Final'
+            mandrel-builder-image: '21.2-java11'
+          - quarkus-version: '1.11.7.Final'
+            mandrel-builder-image: '21.3-java11'
     steps:
       - uses: actions/checkout@v2
         with:

.github/workflows/local_tests.yml

@@ -14,6 +14,7 @@ on:
 
 env:
   LANG: en_US.UTF-8
+  FAIL_ON_PERF_REGRESSION: false
 
 jobs:
   local-run:
@@ -22,12 +23,18 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        quarkus-version: ['2.2.3.Final', '2.4.0.Final']
-        mandrel-version: ['21.2.0.2-Final', '21.3.0.0-Final']
+        quarkus-version: ['1.11.7.Final', '2.2.3.Final', '2.4.0.Final']
+        mandrel-version: ['20.3.3.0-Final', '21.2.0.2-Final', '21.3.0.0-Final']
         os: [ubuntu-20.04, windows-2019]
-        # exclude:
-        #   - quarkus-version: '1.11.7.Final'
-        #     mandrel-version: '21.2.0.1-Final'
+        exclude:
+          - quarkus-version: '1.11.7.Final'
+            mandrel-version: '21.2.0.2-Final'
+          - quarkus-version: '1.11.7.Final'
+            mandrel-version: '21.3.0.0-Final'
+          - quarkus-version: '2.2.3.Final'
+            mandrel-version: '20.3.3.0-Final'
+          - quarkus-version: '2.4.0.Final'
+            mandrel-version: '20.3.3.0-Final'
     steps:
       - uses: actions/checkout@v2
         with:

builder-image-apps/quarkus-vertx/quarkus_2.4.x.patch

@@ -0,0 +1,13 @@
+diff --git a/builder-image-apps/quarkus-vertx/src/main/docker/Dockerfile.native b/builder-image-apps/quarkus-vertx/src/main/docker/Dockerfile.native
+index 70ec986..2f44460 100644
+--- a/builder-image-apps/quarkus-vertx/src/main/docker/Dockerfile.native
++++ b/builder-image-apps/quarkus-vertx/src/main/docker/Dockerfile.native
+@@ -3,7 +3,7 @@ RUN microdnf --enablerepo=ubi-8-baseos-debug --enablerepo=ubi-8-appstream-debug
+ WORKDIR /work/
+ COPY target/quarkus-runner /work/application
+ COPY target/quarkus-runner.debug /work/application.debug
+-COPY target/sources /work/sources
++COPY target/quarkus-native-image-source-jar/sources /work/sources
+ RUN chmod 775 /work -R
+ EXPOSE 8080
+ CMD ["./application", "-Dquarkus.http.host=0.0.0.0"]

testsuite/src/it/java/org/graalvm/tests/integration/DebugSymbolsTest.java

@@ -206,7 +206,14 @@ public void debugSymbolsQuarkus(TestInfo testInfo) throws IOException, Interrupt
                 writer.write("set confirm off\n");
                 writer.flush();
 
-                writer.write("set directories " + appDir.getAbsolutePath() + "/target/quarkus-native-image-source-jar/sources\n");
+                // See https://github.com/quarkusio/quarkus/pull/20355
+                // TODO: Introduce QuarkusVersion comparable as this "startsWith nonsense won't scale, e.g. 2.5...
+                if (QUARKUS_VERSION.startsWith("2.4")) {
+                    writer.write("set directories " + appDir.getAbsolutePath() + "/target/quarkus-native-image-source-jar/sources\n");
+                } else {
+                    writer.write("set directories " + appDir.getAbsolutePath() + "/target/sources\n");
+                }
+
                 writer.flush();
 
                 carryOutGDBSession(stringBuffer, GDBSession.DEBUG_QUARKUS_FULL_MICROPROFILE, esvc, writer, report, UsedVersion.getVersion(false));
@@ -245,6 +252,11 @@ public void debugSymbolsQuarkusContainer(TestInfo testInfo) throws IOException,
             stopAllRunningContainers();
             Files.createDirectories(Paths.get(appDir.getAbsolutePath() + File.separator + "logs"));
 
+            if (QUARKUS_VERSION.startsWith("2.4")) {
+                runCommand(getRunCommand("git", "apply", "quarkus_2.4.x.patch"),
+                        Path.of(BASE_DIR, Apps.DEBUG_QUARKUS_BUILDER_IMAGE_VERTX.dir).toFile());
+            }
+
             // Build & Run
             processLog = new File(appDir.getAbsolutePath() + File.separator + "logs" + File.separator + "build-and-run.log");
             builderRoutine(app.buildAndRunCmds.cmds.length, app, report, cn, mn, appDir, processLog);
@@ -314,6 +326,10 @@ public void debugSymbolsQuarkusContainer(TestInfo testInfo) throws IOException,
             cleanup(null, cn, mn, report, app, processLog);
             stopAllRunningContainers();
             removeContainers(app.runtimeContainer.name, "quarkus_test_db");
+            if (QUARKUS_VERSION.startsWith("2.4")) {
+                runCommand(getRunCommand("git", "apply", "-R", "quarkus_2.4.x.patch"),
+                        Path.of(BASE_DIR, Apps.DEBUG_QUARKUS_BUILDER_IMAGE_VERTX.dir).toFile());
+            }
         }
     }
 

testsuite/src/it/java/org/graalvm/tests/integration/utils/Commands.java

@@ -75,6 +75,8 @@ public class Commands {
     public static final String QUARKUS_VERSION = getProperty(
             new String[]{"QUARKUS_VERSION", "quarkus.version"},
             "2.2.3.Final");
+    public static final boolean FAIL_ON_PERF_REGRESSION = Boolean.parseBoolean(
+            getProperty(new String[]{"FAIL_ON_PERF_REGRESSION", "fail.on.perf.regression"}, "true"));
     public static final boolean IS_THIS_WINDOWS = System.getProperty("os.name").matches(".*[Ww]indows.*");
     private static final Pattern NUM_PATTERN = Pattern.compile("[ \t]*[0-9]+[ \t]*");
     private static final Pattern ALPHANUMERIC_FIRST = Pattern.compile("([a-z0-9]+).*");

testsuite/src/it/java/org/graalvm/tests/integration/utils/Logs.java

@@ -36,6 +36,7 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
 import static org.graalvm.tests.integration.RuntimesSmokeTest.BASE_DIR;
+import static org.graalvm.tests.integration.utils.Commands.FAIL_ON_PERF_REGRESSION;
 import static org.graalvm.tests.integration.utils.Commands.IS_THIS_WINDOWS;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -90,7 +91,7 @@ public static void checkThreshold(Apps app, String mode, long executableSizeKb,
             final String key = propPrefix + ".executable.size.threshold.kB";
             if (app.thresholdProperties.containsKey(key)) {
                 long executableSizeThresholdKb = app.thresholdProperties.get(key);
-                assertTrue(executableSizeKb <= executableSizeThresholdKb,
+                assertThreshold(executableSizeKb <= executableSizeThresholdKb,
                         "Application " + app + (mode != null ? " in mode " + mode : "") + " executable size is " +
                                 executableSizeKb + " kB, which is over " +
                                 executableSizeThresholdKb + " kB threshold by " + percentageValOverTh(executableSizeKb, executableSizeThresholdKb) + "%.");
@@ -104,7 +105,7 @@ public static void checkThreshold(Apps app, String mode, long executableSizeKb,
             final String key = propPrefix + ".time.to.first.ok.request.threshold.ms";
             if (app.thresholdProperties.containsKey(key)) {
                 long timeToFirstOKRequestThresholdMs = app.thresholdProperties.get(key);
-                assertTrue(timeToFirstOKRequest <= timeToFirstOKRequestThresholdMs,
+                assertThreshold(timeToFirstOKRequest <= timeToFirstOKRequestThresholdMs,
                         "Application " + app + (mode != null ? " in mode " + mode : "") +
                                 " took " + timeToFirstOKRequest + " ms to get the first OK request, which is over " +
                                 timeToFirstOKRequestThresholdMs + " ms threshold by " + percentageValOverTh(timeToFirstOKRequest, timeToFirstOKRequestThresholdMs) + "%.");
@@ -118,7 +119,7 @@ public static void checkThreshold(Apps app, String mode, long executableSizeKb,
             final String key = propPrefix + ".RSS.threshold.kB";
             if (app.thresholdProperties.containsKey(key)) {
                 long rssThresholdKb = app.thresholdProperties.get(key);
-                assertTrue(rssKb <= rssThresholdKb,
+                assertThreshold(rssKb <= rssThresholdKb,
                         "Application " + app + (mode != null ? " in mode " + mode : "") +
                                 " consumed " + rssKb + " kB or RSS memory, which is over " +
                                 rssThresholdKb + " kB threshold by " + percentageValOverTh(rssKb, rssThresholdKb) + "%.");
@@ -132,7 +133,7 @@ public static void checkThreshold(Apps app, String mode, long executableSizeKb,
             final String key = propPrefix + ".time.to.finish.threshold.ms";
             if (app.thresholdProperties.containsKey(key)) {
                 long timeToFinishThresholdMs = app.thresholdProperties.get(key);
-                assertTrue(timeToFinishMs <= timeToFinishThresholdMs,
+                assertThreshold(timeToFinishMs <= timeToFinishThresholdMs,
                         "Application " + app + (mode != null ? " in mode " + mode : "") + " took " +
                                 timeToFinishMs + " ms to finish, which is over " +
                                 timeToFinishThresholdMs + " ms threshold by " + percentageValOverTh(timeToFinishMs, timeToFinishThresholdMs) + "%.");
@@ -143,6 +144,16 @@ public static void checkThreshold(Apps app, String mode, long executableSizeKb,
         }
     }
 
+    public static void assertThreshold(boolean condition, String message) {
+        if (FAIL_ON_PERF_REGRESSION) {
+            assertTrue(condition, message);
+        } else {
+            if (!condition) {
+                LOGGER.error(message);
+            }
+        }
+    }
+
     public static int percentageValOverTh(float value, float threshold) {
         return Math.round(value / (threshold / 100f)) - 100;
     }

testsuite/src/it/java/org/graalvm/tests/integration/utils/WhitelistLogLines.java

@@ -74,7 +74,7 @@ public enum WhitelistLogLines {
             // Windows specific warning, O.K.
             Pattern.compile(".*objcopy executable not found in PATH.*"),
             Pattern.compile(".*That will result in a larger native image.*"),
-            Pattern.compile(".*That also means that resulting native executable is larger*"),
+            Pattern.compile(".*That also means that resulting native executable is larger.*"),
             Pattern.compile(".*contain duplicate files, e.g. javax/activation/ActivationDataFlavor.class.*"),
             Pattern.compile(".*contain duplicate files, e.g. javax/servlet/http/HttpUtils.class.*"),
             Pattern.compile(".*contain duplicate files, e.g. javax/annotation/ManagedBean.class.*"),
@@ -87,6 +87,9 @@ public enum WhitelistLogLines {
             Pattern.compile(".*Unrecognized configuration key.*quarkus.version.*was provided.*"),
             // GitHub workflow Windows executor flaw:
             Pattern.compile(".*Unable to make the Vert.x cache directory.*"),
+            // Not sure, definitely not Mandrel related though
+            Pattern.compile(".*xml-apis:xml-apis:jar:.* has been relocated to xml-apis:xml-apis:jar:.*"),
+            Pattern.compile(".*io.quarkus:quarkus-vertx-web:jar:.* has been relocated to io.quarkus:quarkus-reactive-routes:jar:.*")
     }),
 
     DEBUG_QUARKUS_BUILDER_IMAGE_VERTX(new Pattern[]{
@@ -100,7 +103,10 @@ public enum WhitelistLogLines {
             // Specific Podman version warning about the way we start gdb in an already running container; harmless.
             Pattern.compile(".*The --tty and --interactive flags might not work properly.*"),
             // Expected part of the app log
-            Pattern.compile(".*'table \"fruits\" does not exist, skipping'.*")
+            Pattern.compile(".*'table \"fruits\" does not exist, skipping'.*"),
+            // Not sure, definitely not Mandrel related though
+            Pattern.compile(".*xml-apis:xml-apis:jar:.* has been relocated to xml-apis:xml-apis:jar:.*"),
+            Pattern.compile(".*io.quarkus:quarkus-vertx-web:jar:.* has been relocated to io.quarkus:quarkus-reactive-routes:jar:.*")
     }),
 
     HELIDON_QUICKSTART_SE(new Pattern[]{