From f4571d6e240cbfe79e52d28247ef1e4d21998269 Mon Sep 17 00:00:00 2001
From: Falko Modler <famod@users.noreply.github.com>
Date: Sun, 2 Jan 2022 20:52:39 +0100
Subject: [PATCH] Configurable secondary orderer for
 `QuarkusTestProfileAwareClassOrderer`

Resolves #21892
---
 .../QuarkusTestProfileAwareClassOrderer.java  | 65 +++++++++++++++----
 ...arkusTestProfileAwareClassOrdererTest.java | 38 ++++++++++-
 2 files changed, 89 insertions(+), 14 deletions(-)

diff --git a/test-framework/junit5/src/main/java/io/quarkus/test/junit/util/QuarkusTestProfileAwareClassOrderer.java b/test-framework/junit5/src/main/java/io/quarkus/test/junit/util/QuarkusTestProfileAwareClassOrderer.java
index 9ec802da34003..5d9c451f8908a 100644
--- a/test-framework/junit5/src/main/java/io/quarkus/test/junit/util/QuarkusTestProfileAwareClassOrderer.java
+++ b/test-framework/junit5/src/main/java/io/quarkus/test/junit/util/QuarkusTestProfileAwareClassOrderer.java
@@ -3,6 +3,8 @@
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import org.junit.jupiter.api.ClassDescriptor;
 import org.junit.jupiter.api.ClassOrderer;
@@ -26,15 +28,19 @@
  * after tests with profiles and Quarkus*Tests with only unrestricted resources are handled like tests without a profile (come
  * first).
  * <p/>
- * Internally, ordering is based on three prefixes that are prepended to the fully qualified name of the respective class, with
- * the fully qualified class name of the {@link io.quarkus.test.junit.QuarkusTestProfile QuarkusTestProfile} as an infix (if
- * present).
+ * Internally, ordering is based on prefixes that are prepended to a secondary order suffix (by default the fully qualified
+ * name of the respective test class), with the fully qualified class name of the
+ * {@link io.quarkus.test.junit.QuarkusTestProfile QuarkusTestProfile} as an infix (if present).
  * The default prefixes are defined by {@code DEFAULT_ORDER_PREFIX_*} and can be overridden in {@code junit-platform.properties}
  * via {@code CFGKEY_ORDER_PREFIX_*}, e.g. non-Quarkus tests can be run first (not last) by setting
  * {@link #CFGKEY_ORDER_PREFIX_NON_QUARKUS_TEST} to {@code 10_}.
  * <p/>
+ * The secondary order suffix can be changed via {@value #CFGKEY_SECONDARY_ORDERER}, e.g. a value of
+ * {@link org.junit.jupiter.api.ClassOrderer.ClassOrderer.Random} will order the test classes within one group randomly instead
+ * by class name.
+ * <p/>
  * {@link #getCustomOrderKey(ClassDescriptor, ClassOrdererContext)} can be overridden to provide a custom order number for a
- * given test class, e.g. based on {@link org.junit.jupiter.api.Tag}, class name or something else.
+ * given test class, e.g. based on {@link org.junit.jupiter.api.Tag} or something else.
  * <p/>
  * Limitations:
  * <ul>
@@ -53,6 +59,7 @@ public class QuarkusTestProfileAwareClassOrderer implements ClassOrderer {
     static final String CFGKEY_ORDER_PREFIX_QUARKUS_TEST_WITH_PROFILE = "quarkus.test.orderer.prefix.quarkus-test-with-profile";
     static final String CFGKEY_ORDER_PREFIX_QUARKUS_TEST_WITH_RESTRICTED_RES = "quarkus.test.orderer.prefix.quarkus-test-with-restricted-resource";
     static final String CFGKEY_ORDER_PREFIX_NON_QUARKUS_TEST = "quarkus.test.orderer.prefix.non-quarkus-test";
+    static final String CFGKEY_SECONDARY_ORDERER = "quarkus.test.orderer.secondary-orderer";
 
     @Override
     public void orderClasses(ClassOrdererContext context) {
@@ -73,29 +80,50 @@ public void orderClasses(ClassOrdererContext context) {
                 .getConfigurationParameter(CFGKEY_ORDER_PREFIX_NON_QUARKUS_TEST)
                 .orElse(DEFAULT_ORDER_PREFIX_NON_QUARKUS_TEST);
 
-        context.getClassDescriptors().sort(Comparator.comparing(classDescriptor -> {
-            Optional<String> customOrderKey = getCustomOrderKey(classDescriptor, context);
+        // first pass: run secondary orderer first (!), which is easier than running it per "grouping"
+        buildSecondaryOrderer(context).orderClasses(context);
+        var classDecriptors = context.getClassDescriptors();
+        var firstPassIndexMap = IntStream.range(0, classDecriptors.size()).boxed()
+                .collect(Collectors.toMap(classDecriptors::get, i -> String.format("%06d", i)));
+
+        // second pass: apply the actual Quarkus aware ordering logic, using the first pass indices as order key suffixes
+        classDecriptors.sort(Comparator.comparing(classDescriptor -> {
+            var secondaryOrderSuffix = firstPassIndexMap.get(classDescriptor);
+            Optional<String> customOrderKey = getCustomOrderKey(classDescriptor, context, secondaryOrderSuffix)
+                    .or(() -> getCustomOrderKey(classDescriptor, context));
             if (customOrderKey.isPresent()) {
                 return customOrderKey.get();
             }
-            var testClassName = classDescriptor.getTestClass().getName();
             if (classDescriptor.isAnnotated(QuarkusTest.class)
                     || classDescriptor.isAnnotated(QuarkusIntegrationTest.class)
                     || classDescriptor.isAnnotated(QuarkusMainTest.class)) {
                 return classDescriptor.findAnnotation(TestProfile.class)
                         .map(TestProfile::value)
-                        .map(profileClass -> prefixQuarkusTestWithProfile + profileClass.getName() + "@" + testClassName)
+                        .map(profileClass -> prefixQuarkusTestWithProfile + profileClass.getName() + "@" + secondaryOrderSuffix)
                         .orElseGet(() -> {
                             var prefix = hasRestrictedResource(classDescriptor)
                                     ? prefixQuarkusTestWithRestrictedResource
                                     : prefixQuarkusTest;
-                            return prefix + testClassName;
+                            return prefix + secondaryOrderSuffix;
                         });
             }
-            return prefixNonQuarkusTest + testClassName;
+            return prefixNonQuarkusTest + secondaryOrderSuffix;
         }));
     }
 
+    private ClassOrderer buildSecondaryOrderer(ClassOrdererContext context) {
+        return context.getConfigurationParameter(CFGKEY_SECONDARY_ORDERER)
+                .map(fqcn -> {
+                    try {
+                        return (ClassOrderer) Class.forName(fqcn).getDeclaredConstructor().newInstance();
+                    } catch (ReflectiveOperationException e) {
+                        throw new IllegalArgumentException(
+                                "Failed to instantiate " + fqcn + " (defined by " + CFGKEY_SECONDARY_ORDERER + ")", e);
+                    }
+                })
+                .orElseGet(ClassOrderer.ClassName::new);
+    }
+
     private boolean hasRestrictedResource(ClassDescriptor classDescriptor) {
         return classDescriptor.findRepeatableAnnotations(QuarkusTestResource.class).stream()
                 .anyMatch(res -> res.restrictToAnnotatedClass() || isMetaTestResource(res, classDescriptor));
@@ -109,12 +137,27 @@ private boolean isMetaTestResource(QuarkusTestResource resource, ClassDescriptor
 
     /**
      * Template method that provides an optional custom order key for the given {@code classDescriptor}.
-     * 
+     *
      * @param classDescriptor the respective test class
      * @param context for config lookup
      * @return optional custom order key for the given test class
+     * @deprecated use {@link #getCustomOrderKey(ClassDescriptor, ClassOrdererContext, String)} instead
      */
+    @Deprecated(forRemoval = true)
     protected Optional<String> getCustomOrderKey(ClassDescriptor classDescriptor, ClassOrdererContext context) {
         return Optional.empty();
     }
+
+    /**
+     * Template method that provides an optional custom order key for the given {@code classDescriptor}.
+     *
+     * @param classDescriptor the respective test class
+     * @param context for config lookup
+     * @param secondaryOrderSuffix the secondary order suffix that was calculated by the secondary orderer
+     * @return optional custom order key for the given test class
+     */
+    protected Optional<String> getCustomOrderKey(ClassDescriptor classDescriptor, ClassOrdererContext context,
+            String secondaryOrderSuffix) {
+        return Optional.empty();
+    }
 }
diff --git a/test-framework/junit5/src/test/java/io/quarkus/test/junit/util/QuarkusTestProfileAwareClassOrdererTest.java b/test-framework/junit5/src/test/java/io/quarkus/test/junit/util/QuarkusTestProfileAwareClassOrdererTest.java
index 25b974da082e8..8c6303f1d7b83 100644
--- a/test-framework/junit5/src/test/java/io/quarkus/test/junit/util/QuarkusTestProfileAwareClassOrdererTest.java
+++ b/test-framework/junit5/src/test/java/io/quarkus/test/junit/util/QuarkusTestProfileAwareClassOrdererTest.java
@@ -1,6 +1,7 @@
 package io.quarkus.test.junit.util;
 
 import static io.quarkus.test.junit.util.QuarkusTestProfileAwareClassOrderer.CFGKEY_ORDER_PREFIX_NON_QUARKUS_TEST;
+import static io.quarkus.test.junit.util.QuarkusTestProfileAwareClassOrderer.CFGKEY_SECONDARY_ORDERER;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.doReturn;
@@ -14,7 +15,9 @@
 import java.util.Optional;
 
 import org.junit.jupiter.api.ClassDescriptor;
+import org.junit.jupiter.api.ClassOrderer;
 import org.junit.jupiter.api.ClassOrdererContext;
+import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
@@ -92,7 +95,7 @@ void configuredPrefix() {
         List<ClassDescriptor> input = Arrays.asList(quarkusTestDesc, nonQuarkusTestDesc);
         doReturn(input).when(contextMock).getClassDescriptors();
 
-        when(contextMock.getConfigurationParameter(anyString())).thenReturn(Optional.empty());
+        when(contextMock.getConfigurationParameter(anyString())).thenReturn(Optional.empty()); // for strict stubbing
         // prioritize unit tests
         when(contextMock.getConfigurationParameter(CFGKEY_ORDER_PREFIX_NON_QUARKUS_TEST)).thenReturn(Optional.of("01_"));
 
@@ -101,6 +104,33 @@ void configuredPrefix() {
         assertThat(input).containsExactly(nonQuarkusTestDesc, quarkusTestDesc);
     }
 
+    @Test
+    void secondaryOrderer() {
+        ClassDescriptor quarkusTest1Desc = quarkusDescriptorMock(Test01.class, null);
+        ClassDescriptor nonQuarkusTest1Desc = descriptorMock(Test09.class);
+        ClassDescriptor nonQuarkusTest2Desc = descriptorMock(Test10.class);
+        var orderMock = Mockito.mock(Order.class);
+        when(orderMock.value()).thenReturn(1);
+        when(nonQuarkusTest2Desc.findAnnotation(Order.class)).thenReturn(Optional.of(orderMock));
+        List<ClassDescriptor> input = Arrays.asList(
+                nonQuarkusTest1Desc,
+                nonQuarkusTest2Desc,
+                quarkusTest1Desc);
+        doReturn(input).when(contextMock).getClassDescriptors();
+
+        when(contextMock.getConfigurationParameter(anyString())).thenReturn(Optional.empty()); // for strict stubbing
+        // change secondary orderer from ClassName to OrderAnnotation
+        when(contextMock.getConfigurationParameter(CFGKEY_SECONDARY_ORDERER))
+                .thenReturn(Optional.of(ClassOrderer.OrderAnnotation.class.getName()));
+
+        underTest.orderClasses(contextMock);
+
+        assertThat(input).containsExactly(
+                quarkusTest1Desc,
+                nonQuarkusTest2Desc,
+                nonQuarkusTest1Desc);
+    }
+
     @Test
     void customOrderKey() {
         ClassDescriptor quarkusTest1Desc = quarkusDescriptorMock(Test01.class, null);
@@ -110,7 +140,8 @@ void customOrderKey() {
 
         underTest = new QuarkusTestProfileAwareClassOrderer() {
             @Override
-            protected Optional<String> getCustomOrderKey(ClassDescriptor classDescriptor, ClassOrdererContext context) {
+            protected Optional<String> getCustomOrderKey(ClassDescriptor classDescriptor, ClassOrdererContext context,
+                    String secondaryOrderSuffix) {
                 return classDescriptor == quarkusTest2Desc ? Optional.of("00_first") : Optional.empty();
             }
         };
@@ -150,7 +181,8 @@ private ClassDescriptor quarkusDescriptorMock(Class<?> testClass,
     private static class Test01 {
     }
 
-    // this single made-up test class needs an actual annotation since the orderer will have do the meta-check directly because ClassDescriptor does not offer any details whether an annotation is directly annotated or meta-annotated
+    // this single made-up test class needs an actual annotation since the orderer will have to do the meta-check directly
+    // because ClassDescriptor does not offer any details whether an annotation is directly annotated or meta-annotated
     @QuarkusTestResource(Manager3.class)
     private static class Test02 {
     }