Introduce QuarkusTestProfileAwareClassOrderer for efficient @TestProfile ordering

docs/src/main/asciidoc/getting-started-testing.adoc

@@ -484,6 +484,9 @@ a bit slower, as it adds a shutdown/startup cycle to the test time, but gives a
 
 NOTE: In order to reduce the amount of times Quarkus needs to restart it is recommended that you place all tests
 that need a specific profile into their own package, and then run tests alphabetically.
+Alternatively, you can register `io.quarkus.test.junit.util.QuarkusTestProfileAwareClassOrderer` as a global `ClassOrderer`
+in `junit-platform.properties` as described in the
+link:https://junit.org/junit5/docs/current/user-guide/#writing-tests-test-execution-order-classes[JUnit 5 User Guide].
 
 === Writing a Profile
 

test-framework/junit5/pom.xml

@@ -50,6 +50,17 @@
             <!-- Avoid adding this to the BOM -->
             <version>1.4.18</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-junit-jupiter</artifactId>
+            <scope>test</scope>
+        </dependency>
+		<dependency>
+		    <groupId>org.assertj</groupId>
+		    <artifactId>assertj-core</artifactId>
+		    <scope>test</scope>
+		</dependency>
     </dependencies>
 
 </project>

test-framework/junit5/src/main/java/io/quarkus/test/junit/util/QuarkusTestProfileAwareClassOrderer.java

@@ -0,0 +1,87 @@
+package io.quarkus.test.junit.util;
+
+import java.util.Comparator;
+import java.util.Optional;
+
+import org.junit.jupiter.api.ClassDescriptor;
+import org.junit.jupiter.api.ClassOrderer;
+import org.junit.jupiter.api.ClassOrdererContext;
+
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.TestProfile;
+
+/**
+ * A {@link ClassOrderer} that orders {@link QuarkusTest} and {@link QuarkusIntegrationTest} classes for minimum Quarkus
+ * restarts by grouping them by their {@link TestProfile}.
+ * <p/>
+ * By default, Quarkus*Tests not using any profile come first, then classes using a profile (in groups) and then all other
+ * non-Quarkus tests (e.g. plain unit tests).
+ * <p/>
+ * Internally, ordering is based on three prefixes that are prepended to the fully qualified name of the respective class, with
+ * the fully qualified class name of the {@link io.quarkus.test.junit.QuarkusTestProfile QuarkusTestProfile} as an infix (if
+ * present).
+ * The default prefixes are defined by {@code DEFAULT_ORDER_PREFIX_*} and can be overridden in {@code junit-platform.properties}
+ * via {@code CFGKEY_ORDER_PREFIX_*}, e.g. non-Quarkus tests can be run first (not last) by setting
+ * {@link #CFGKEY_ORDER_PREFIX_NON_QUARKUS_TEST} to {@code 10_}.
+ * <p/>
+ * {@link #getCustomOrderKey(ClassDescriptor, ClassOrdererContext)} can be overridden to provide a custom order number for a
+ * given test class, e.g. based on {@link org.junit.jupiter.api.Tag}, class name or something else.
+ * <p/>
+ * Limitations:
+ * <ul>
+ * <li>This orderer does not (yet) consider {@linkplain io.quarkus.test.common.QuarkusTestResource#restrictToAnnotatedClass()
+ * test resources that are restricted to the annotated class}.</li>
+ * <li>Only JUnit5 test classes are subject to ordering, e.g. ArchUnit test classes are not passed to this orderer.</li>
+ * </ul>
+ */
+public class QuarkusTestProfileAwareClassOrderer implements ClassOrderer {
+
+    protected static final String DEFAULT_ORDER_PREFIX_QUARKUS_TEST = "20_";
+    protected static final String DEFAULT_ORDER_PREFIX_QUARKUS_TEST_WITH_PROFILE = "40_";
+    protected static final String DEFAULT_ORDER_PREFIX_NON_QUARKUS_TEST = "60_";
+
+    static final String CFGKEY_ORDER_PREFIX_QUARKUS_TEST = "quarkus.test.orderer.prefix.quarkus-test";
+    static final String CFGKEY_ORDER_PREFIX_QUARKUS_TEST_WITH_PROFILE = "quarkus.test.orderer.prefix.quarkus-test-with-profile";
+    static final String CFGKEY_ORDER_PREFIX_NON_QUARKUS_TEST = "quarkus.test.orderer.prefix.non-quarkus-test";
+
+    @Override
+    public void orderClasses(ClassOrdererContext context) {
+        if (context.getClassDescriptors().size() <= 1) {
+            return;
+        }
+        var prefixQuarkusTest = context.getConfigurationParameter(CFGKEY_ORDER_PREFIX_QUARKUS_TEST)
+                .orElse(DEFAULT_ORDER_PREFIX_QUARKUS_TEST);
+        var prefixQuarkusTestWithProfile = context.getConfigurationParameter(CFGKEY_ORDER_PREFIX_QUARKUS_TEST_WITH_PROFILE)
+                .orElse(DEFAULT_ORDER_PREFIX_QUARKUS_TEST_WITH_PROFILE);
+        var prefixNonQuarkusTest = context.getConfigurationParameter(CFGKEY_ORDER_PREFIX_NON_QUARKUS_TEST)
+                .orElse(DEFAULT_ORDER_PREFIX_NON_QUARKUS_TEST);
+
+        context.getClassDescriptors().sort(Comparator.comparing(classDescriptor -> {
+            Optional<String> customOrderKey = getCustomOrderKey(classDescriptor, context);
+            if (customOrderKey.isPresent()) {
+                return customOrderKey.get();
+            }
+            var testClassName = classDescriptor.getTestClass().getName();
+            if (classDescriptor.isAnnotated(QuarkusTest.class)
+                    || classDescriptor.isAnnotated(QuarkusIntegrationTest.class)) {
+                return classDescriptor.findAnnotation(TestProfile.class)
+                        .map(TestProfile::value)
+                        .map(profileClass -> prefixQuarkusTestWithProfile + profileClass.getName() + "@" + testClassName)
+                        .orElseGet(() -> prefixQuarkusTest + testClassName);
+            }
+            return prefixNonQuarkusTest + testClassName;
+        }));
+    }
+
+    /**
+     * Template method that provides an optional custom order key for the given {@code classDescriptor}.
+     * 
+     * @param classDescriptor the respective test class
+     * @param context for config lookup
+     * @return optional custom order key for the given test class
+     */
+    protected Optional<String> getCustomOrderKey(ClassDescriptor classDescriptor, ClassOrdererContext context) {
+        return Optional.empty();
+    }
+}

test-framework/junit5/src/test/java/io/quarkus/test/junit/util/QuarkusTestProfileAwareClassOrdererTest.java

@@ -0,0 +1,154 @@
+package io.quarkus.test.junit.util;
+
+import static io.quarkus.test.junit.util.QuarkusTestProfileAwareClassOrderer.CFGKEY_ORDER_PREFIX_NON_QUARKUS_TEST;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+
+import org.junit.jupiter.api.ClassDescriptor;
+import org.junit.jupiter.api.ClassOrdererContext;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+
+@ExtendWith(MockitoExtension.class)
+class QuarkusTestProfileAwareClassOrdererTest {
+
+    @Mock
+    ClassOrdererContext contextMock;
+
+    QuarkusTestProfileAwareClassOrderer underTest = new QuarkusTestProfileAwareClassOrderer();
+
+    @Test
+    void singleClass() {
+        doReturn(Arrays.asList(descriptorMock(Test1.class)))
+                .when(contextMock).getClassDescriptors();
+
+        underTest.orderClasses(contextMock);
+
+        verify(contextMock, never()).getConfigurationParameter(anyString());
+    }
+
+    @Test
+    void allVariants() {
+        ClassDescriptor quarkusTest1Desc = quarkusDescriptorMock(Test1.class, null);
+        ClassDescriptor quarkusTest2Desc = quarkusDescriptorMock(Test2.class, null);
+        ClassDescriptor quarkusTestWithProfile1Desc = quarkusDescriptorMock(Test3.class, Profile1.class);
+        ClassDescriptor quarkusTestWithProfile2Test4Desc = quarkusDescriptorMock(Test4.class, Profile2.class);
+        ClassDescriptor quarkusTestWithProfile2Test5Desc = quarkusDescriptorMock(Test5.class, Profile2.class);
+        ClassDescriptor nonQuarkusTest6Desc = descriptorMock(Test6.class);
+        ClassDescriptor nonQuarkusTest7Desc = descriptorMock(Test7.class);
+        List<ClassDescriptor> input = Arrays.asList(
+                nonQuarkusTest7Desc,
+                quarkusTestWithProfile2Test5Desc,
+                quarkusTest2Desc,
+                nonQuarkusTest6Desc,
+                quarkusTest1Desc,
+                quarkusTestWithProfile2Test4Desc,
+                quarkusTestWithProfile1Desc);
+        doReturn(input).when(contextMock).getClassDescriptors();
+
+        underTest.orderClasses(contextMock);
+
+        assertThat(input).containsExactly(
+                quarkusTest1Desc,
+                quarkusTest2Desc,
+                quarkusTestWithProfile1Desc,
+                quarkusTestWithProfile2Test4Desc,
+                quarkusTestWithProfile2Test5Desc,
+                nonQuarkusTest6Desc,
+                nonQuarkusTest7Desc);
+    }
+
+    @Test
+    void configuredPrefix() {
+        ClassDescriptor quarkusTestDesc = quarkusDescriptorMock(Test1.class, null);
+        ClassDescriptor nonQuarkusTestDesc = descriptorMock(Test2.class);
+        List<ClassDescriptor> input = Arrays.asList(quarkusTestDesc, nonQuarkusTestDesc);
+        doReturn(input).when(contextMock).getClassDescriptors();
+
+        when(contextMock.getConfigurationParameter(anyString())).thenReturn(Optional.empty());
+        // prioritize unit tests
+        when(contextMock.getConfigurationParameter(CFGKEY_ORDER_PREFIX_NON_QUARKUS_TEST)).thenReturn(Optional.of("01_"));
+
+        underTest.orderClasses(contextMock);
+
+        assertThat(input).containsExactly(nonQuarkusTestDesc, quarkusTestDesc);
+    }
+
+    @Test
+    void customOrderKey() {
+        ClassDescriptor quarkusTest1Desc = quarkusDescriptorMock(Test1.class, null);
+        ClassDescriptor quarkusTest2Desc = quarkusDescriptorMock(Test2.class, null);
+        List<ClassDescriptor> input = Arrays.asList(quarkusTest1Desc, quarkusTest2Desc);
+        doReturn(input).when(contextMock).getClassDescriptors();
+
+        underTest = new QuarkusTestProfileAwareClassOrderer() {
+            @Override
+            protected Optional<String> getCustomOrderKey(ClassDescriptor classDescriptor, ClassOrdererContext context) {
+                return classDescriptor == quarkusTest2Desc ? Optional.of("00_first") : Optional.empty();
+            }
+        };
+        underTest.orderClasses(contextMock);
+
+        assertThat(input).containsExactly(quarkusTest2Desc, quarkusTest1Desc);
+    }
+
+    private ClassDescriptor descriptorMock(Class<?> testClass) {
+        ClassDescriptor mock = Mockito.mock(ClassDescriptor.class, withSettings().lenient().name(testClass.getSimpleName()));
+        doReturn(testClass).when(mock).getTestClass();
+        return mock;
+    }
+
+    private ClassDescriptor quarkusDescriptorMock(Class<?> testClass, Class<? extends QuarkusTestProfile> profileClass) {
+        ClassDescriptor mock = descriptorMock(testClass);
+        when(mock.isAnnotated(QuarkusTest.class)).thenReturn(true);
+        if (profileClass != null) {
+            TestProfile profileMock = Mockito.mock(TestProfile.class);
+            doReturn(profileClass).when(profileMock).value();
+            when(mock.findAnnotation(TestProfile.class)).thenReturn(Optional.of(profileMock));
+        }
+        return mock;
+    }
+
+    private static class Test1 {
+    };
+
+    private static class Test2 {
+    };
+
+    private static class Test3 {
+    };
+
+    private static class Test4 {
+    };
+
+    private static class Test5 {
+    };
+
+    private static class Test6 {
+    };
+
+    private static class Test7 {
+    };
+
+    private static class Profile1 implements QuarkusTestProfile {
+    }
+
+    private static class Profile2 implements QuarkusTestProfile {
+    }
+}