[SUREFIRE-1584] Add support for rerunFailingTests to JUnit5 Provider

apache · Aug 20, 2019 · 2558e28 · 2558e28
1 parent f7d4310
commit 2558e28
Show file tree

Hide file tree

Showing 6 changed files with 511 additions and 1 deletion.
diff --git a/...ire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java b/...ire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java
@@ -0,0 +1,268 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.apache.maven.surefire.its.fixture.SurefireLauncher;
+import org.junit.Test;
+
+/**
+ * JUnit4 RunListener Integration Test.
+ *
+ * @author <a href="mailto:[email protected]">Qingzhou Luo</a>
+ * @author Matt Coley
+ */
+public class JUnitPlatformRerunFailingTestsIT extends SurefireJUnit4IntegrationTestCase
+{
+    private final static String GOAL = "-Dprovider=surefire-junit-platform";
+    private final static String VERSION = "5.4.2";
+
+    private SurefireLauncher unpack()
+    {
+        return unpack( "/junit-platform-rerun-failing-tests" );
+    }
+
+    @Test
+    public void testRerunFailingErrorTestsWithOneRetry() throws Exception
+    {
+        OutputValidator outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0,
+                0 );
+        verifyFailuresOneRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+                "-DforkCount=2" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+        verifyFailuresOneRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+        verifyFailuresOneRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+        verifyFailuresOneRetryAllClasses( outputValidator );
+    }
+
+    @Test
+    public void testRerunFailingErrorTestsTwoRetry() throws Exception
+    {
+        // Four flakes, both tests have been re-run twice
+        OutputValidator outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=2" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+        verifyFailuresTwoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-DforkCount=3" ).executeTest().assertTestSuiteResults(
+                5, 0, 0, 0, 4 );
+
+        verifyFailuresTwoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+        verifyFailuresTwoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+        verifyFailuresTwoRetryAllClasses( outputValidator );
+    }
+
+    @Test
+    public void testRerunFailingErrorTestsFalse() throws Exception
+    {
+        OutputValidator outputValidator = unpack().addGoal( GOAL ).setJUnitVersion(
+                VERSION ).maven().withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+        verifyFailuresNoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-DforkCount=3" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+        verifyFailuresNoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+        verifyFailuresNoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+        verifyFailuresNoRetryAllClasses( outputValidator );
+    }
+
+    @Test
+    public void testRerunOneTestClass() throws Exception
+    {
+        OutputValidator outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneClass( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneClass( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneClass( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneClass( outputValidator );
+    }
+
+    @Test
+    public void testRerunOneTestMethod() throws Exception
+    {
+        OutputValidator outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1,
+                0, 0 );
+
+        verifyFailuresOneRetryOneMethod( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1,
+                0, 0 );
+
+        verifyFailuresOneRetryOneMethod( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1,
+                0, 0 );
+
+        verifyFailuresOneRetryOneMethod( outputValidator );
+
+        outputValidator = unpack().addGoal( GOAL ).setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1,
+                0, 0 );
+
+        verifyFailuresOneRetryOneMethod( outputValidator );
+    }
+
+    private void verifyFailuresOneRetryAllClasses( OutputValidator outputValidator )
+    {
+        verifyFailuresOneRetry( outputValidator, 5, 1, 1, 0 );
+    }
+
+    private void verifyFailuresTwoRetryAllClasses( OutputValidator outputValidator )
+    {
+        verifyFailuresTwoRetry( outputValidator, 5, 0, 0, 2 );
+    }
+
+    private void verifyFailuresNoRetryAllClasses( OutputValidator outputValidator )
+    {
+        verifyFailuresNoRetry( outputValidator, 5, 1, 1, 0 );
+    }
+
+    private void verifyFailuresOneRetryOneClass( OutputValidator outputValidator )
+    {
+        verifyFailuresOneRetry( outputValidator, 3, 1, 1, 0 );
+    }
+
+    private void verifyFailuresOneRetryOneMethod( OutputValidator outputValidator )
+    {
+        verifyOnlyFailuresOneRetry( outputValidator, 1, 1, 0, 0 );
+    }
+
+    private void verifyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+    {
+        outputValidator.verifyTextInLog( "Failures:" );
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+        outputValidator.verifyTextInLog( "Errors:" );
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+        verifyStatistics( outputValidator, run, failures, errors, flakes );
+    }
+
+    private void verifyOnlyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+    {
+        outputValidator.verifyTextInLog( "Failures:" );
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+        verifyStatistics( outputValidator, run, failures, errors, flakes );
+    }
+
+    private void verifyFailuresTwoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+    {
+        outputValidator.verifyTextInLog( "Flakes:" );
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "Run 3: PASS" );
+
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+        verifyStatistics( outputValidator, run, failures, errors, flakes );
+    }
+
+    private void verifyFailuresNoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+    {
+        outputValidator.verifyTextInLog( "Failures:" );
+        outputValidator.verifyTextInLog( "junitplatform.FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "ERROR" );
+        outputValidator.verifyTextInLog( "junitplatform.FlakyFirstTimeTest.testErrorTestOne" );
+
+        verifyStatistics( outputValidator, run, failures, errors, flakes );
+    }
+
+    private void verifyStatistics( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+    {
+        if ( flakes > 0 )
+        {
+            outputValidator.verifyTextInLog(
+                    "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0, Flakes: " + flakes );
+        }
+        else
+        {
+            outputValidator.verifyTextInLog(
+                    "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0" );
+        }
+    }
+}
diff --git a/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/pom.xml b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.apache.maven.plugins.surefire</groupId>
+    <artifactId>junit-platform-rerun-failing-tests</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <name>Test for rerun failing tests in JUnit 5/Platform</name>
+
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>${surefire.version}</version>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
diff --git a/...es/junit-platform-rerun-failing-tests/src/test/java/junitplatform/FlakyFirstTimeTest.java b/...es/junit-platform-rerun-failing-tests/src/test/java/junitplatform/FlakyFirstTimeTest.java
@@ -0,0 +1,63 @@
+package junitplatform;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+public class FlakyFirstTimeTest
+{
+    private static int failingCount = 0;
+
+    private static int errorCount = 0;
+
+
+    @Test
+    public void testFailingTestOne()
+    {
+        System.out.println( "Failing test" );
+        // This test will fail with only one retry, but will pass with two
+        if ( failingCount < 2 )
+        {
+            failingCount++;
+            fail( "Failing test" );
+        }
+    }
+
+    @Test
+    public void testErrorTestOne() throws Exception
+    {
+        System.out.println( "Error test" );
+        // This test will error out with only one retry, but will pass with two
+        if ( errorCount < 2 )
+        {
+            errorCount++;
+            throw new IllegalArgumentException( "..." );
+        }
+    }
+
+    @Test
+    public void testPassingTest() throws Exception
+    {
+        System.out.println( "Passing test" );
+    }
+}