FAIL_ON_PERF_REGRESSION: false

Karm · Oct 29, 2021 · b1eb1f0 · b1eb1f0
1 parent 073091f
commit b1eb1f0
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 13 deletions.
diff --git a/.github/workflows/builder_image_tests.yml b/.github/workflows/builder_image_tests.yml
@@ -14,6 +14,7 @@ on:
 
 env:
   LANG: en_US.UTF-8
+  FAIL_ON_PERF_REGRESSION: false
 
 jobs:
   local-run:
@@ -22,15 +23,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        quarkus-version: ['2.2.3.Final', '2.4.0.Final']
+        quarkus-version: ['1.11.7.Final', '2.2.3.Final', '2.4.0.Final']
         mandrel-builder-image: [
+            '20.3-java11',
             '21.2-java11',
             '21.3-java11'
         ]
         os: [ubuntu-20.04]
-        # exclude:
-        #   - quarkus-version: '2.2.3.Final'
-        #     mandrel-builder-image: '21.2-java11'
+        exclude:
+          - quarkus-version: '2.2.3.Final'
+            mandrel-builder-image: '20.3-java11'
+          - quarkus-version: '2.4.0.Final'
+            mandrel-builder-image: '20.3-java11'
+          - quarkus-version: '1.11.7.Final'
+            mandrel-builder-image: '21.2-java11'
+          - quarkus-version: '1.11.7.Final'
+            mandrel-builder-image: '21.3-java11'
     steps:
       - uses: actions/checkout@v2
         with:

diff --git a/.github/workflows/local_tests.yml b/.github/workflows/local_tests.yml
@@ -14,6 +14,7 @@ on:
 
 env:
   LANG: en_US.UTF-8
+  FAIL_ON_PERF_REGRESSION: false
 
 jobs:
   local-run:
@@ -22,12 +23,18 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        quarkus-version: ['2.2.3.Final', '2.4.0.Final']
-        mandrel-version: ['21.2.0.2-Final', '21.3.0.0-Final']
+        quarkus-version: ['1.11.7.Final', '2.2.3.Final', '2.4.0.Final']
+        mandrel-version: ['20.3.3.0-Final', '21.2.0.2-Final', '21.3.0.0-Final']
         os: [ubuntu-20.04, windows-2019]
-        # exclude:
-        #   - quarkus-version: '1.11.7.Final'
-        #     mandrel-version: '21.2.0.1-Final'
+        exclude:
+          - quarkus-version: '1.11.7.Final'
+            mandrel-version: '21.2.0.2-Final'
+          - quarkus-version: '1.11.7.Final'
+            mandrel-version: '21.3.0.0-Final'
+          - quarkus-version: '2.2.3.Final'
+            mandrel-version: '20.3.3.0-Final'
+          - quarkus-version: '2.4.0.Final'
+            mandrel-version: '20.3.3.0-Final'
     steps:
       - uses: actions/checkout@v2
         with:

diff --git a/testsuite/src/it/java/org/graalvm/tests/integration/utils/Commands.java b/testsuite/src/it/java/org/graalvm/tests/integration/utils/Commands.java
@@ -75,6 +75,8 @@ public class Commands {
     public static final String QUARKUS_VERSION = getProperty(
             new String[]{"QUARKUS_VERSION", "quarkus.version"},
             "2.2.3.Final");
+    public static final boolean FAIL_ON_PERF_REGRESSION = Boolean.parseBoolean(
+            getProperty(new String[]{"FAIL_ON_PERF_REGRESSION", "fail.on.perf.regression"}, "true"));
     public static final boolean IS_THIS_WINDOWS = System.getProperty("os.name").matches(".*[Ww]indows.*");
     private static final Pattern NUM_PATTERN = Pattern.compile("[ \t]*[0-9]+[ \t]*");
     private static final Pattern ALPHANUMERIC_FIRST = Pattern.compile("([a-z0-9]+).*");

diff --git a/testsuite/src/it/java/org/graalvm/tests/integration/utils/Logs.java b/testsuite/src/it/java/org/graalvm/tests/integration/utils/Logs.java
@@ -36,6 +36,7 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
 import static org.graalvm.tests.integration.RuntimesSmokeTest.BASE_DIR;
+import static org.graalvm.tests.integration.utils.Commands.FAIL_ON_PERF_REGRESSION;
 import static org.graalvm.tests.integration.utils.Commands.IS_THIS_WINDOWS;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -90,7 +91,7 @@ public static void checkThreshold(Apps app, String mode, long executableSizeKb,
             final String key = propPrefix + ".executable.size.threshold.kB";
             if (app.thresholdProperties.containsKey(key)) {
                 long executableSizeThresholdKb = app.thresholdProperties.get(key);
-                assertTrue(executableSizeKb <= executableSizeThresholdKb,
+                assertThreshold(executableSizeKb <= executableSizeThresholdKb,
                         "Application " + app + (mode != null ? " in mode " + mode : "") + " executable size is " +
                                 executableSizeKb + " kB, which is over " +
                                 executableSizeThresholdKb + " kB threshold by " + percentageValOverTh(executableSizeKb, executableSizeThresholdKb) + "%.");
@@ -104,7 +105,7 @@ public static void checkThreshold(Apps app, String mode, long executableSizeKb,
             final String key = propPrefix + ".time.to.first.ok.request.threshold.ms";
             if (app.thresholdProperties.containsKey(key)) {
                 long timeToFirstOKRequestThresholdMs = app.thresholdProperties.get(key);
-                assertTrue(timeToFirstOKRequest <= timeToFirstOKRequestThresholdMs,
+                assertThreshold(timeToFirstOKRequest <= timeToFirstOKRequestThresholdMs,
                         "Application " + app + (mode != null ? " in mode " + mode : "") +
                                 " took " + timeToFirstOKRequest + " ms to get the first OK request, which is over " +
                                 timeToFirstOKRequestThresholdMs + " ms threshold by " + percentageValOverTh(timeToFirstOKRequest, timeToFirstOKRequestThresholdMs) + "%.");
@@ -118,7 +119,7 @@ public static void checkThreshold(Apps app, String mode, long executableSizeKb,
             final String key = propPrefix + ".RSS.threshold.kB";
             if (app.thresholdProperties.containsKey(key)) {
                 long rssThresholdKb = app.thresholdProperties.get(key);
-                assertTrue(rssKb <= rssThresholdKb,
+                assertThreshold(rssKb <= rssThresholdKb,
                         "Application " + app + (mode != null ? " in mode " + mode : "") +
                                 " consumed " + rssKb + " kB or RSS memory, which is over " +
                                 rssThresholdKb + " kB threshold by " + percentageValOverTh(rssKb, rssThresholdKb) + "%.");
@@ -132,7 +133,7 @@ public static void checkThreshold(Apps app, String mode, long executableSizeKb,
             final String key = propPrefix + ".time.to.finish.threshold.ms";
             if (app.thresholdProperties.containsKey(key)) {
                 long timeToFinishThresholdMs = app.thresholdProperties.get(key);
-                assertTrue(timeToFinishMs <= timeToFinishThresholdMs,
+                assertThreshold(timeToFinishMs <= timeToFinishThresholdMs,
                         "Application " + app + (mode != null ? " in mode " + mode : "") + " took " +
                                 timeToFinishMs + " ms to finish, which is over " +
                                 timeToFinishThresholdMs + " ms threshold by " + percentageValOverTh(timeToFinishMs, timeToFinishThresholdMs) + "%.");
@@ -143,6 +144,16 @@ public static void checkThreshold(Apps app, String mode, long executableSizeKb,
         }
     }
 
+    public static void assertThreshold(boolean condition, String message) {
+        if (FAIL_ON_PERF_REGRESSION) {
+            assertTrue(condition, message);
+        } else {
+            if (!condition) {
+                LOGGER.error(message);
+            }
+        }
+    }
+
     public static int percentageValOverTh(float value, float threshold) {
         return Math.round(value / (threshold / 100f)) - 100;
     }