[SUREFIRE-2095] Fork crash doesn't fail build with -Dmaven.test.failu…

…re.ignore=true when run with failsafe (#545)

Verify goal should fail when test summary indicates a SurefireBooterForkException occurred

maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/VerifyMojo.java

@@ -33,6 +33,7 @@
 import org.apache.maven.plugins.annotations.Parameter;
 import org.apache.maven.surefire.api.cli.CommandLineOption;
 import org.apache.maven.surefire.api.suite.RunResult;
+import org.apache.maven.surefire.booter.SurefireBooterForkException;
 import org.codehaus.plexus.logging.Logger;
 
 import java.io.File;
@@ -198,10 +199,27 @@ public void execute()
                 throw new MojoExecutionException( e.getMessage(), e );
             }
 
-            reportExecution( this, summary, getConsoleLogger(), null );
+            reportExecution( this, summary, getConsoleLogger(), getBooterForkException( summary ) );
         }
     }
 
+    private Exception getBooterForkException( RunResult summary )
+    {
+        String firstForkExceptionFailureMessage =
+            String.format( "%s: " , SurefireBooterForkException.class.getName() );
+        if ( summary.getFailure() != null && summary.getFailure().contains( firstForkExceptionFailureMessage ) )
+        {
+            return new SurefireBooterForkException(
+                summary.getFailure().substring( firstForkExceptionFailureMessage.length() ) );
+        }
+        return null;
+    }
+
+    void setLogger( Logger logger )
+    {
+        this.logger = logger;
+    }
+
     private PluginConsoleLogger getConsoleLogger()
     {
         if ( consoleLogger == null )
@@ -359,6 +377,16 @@ public void setReportsDirectory( File reportsDirectory )
         this.reportsDirectory = reportsDirectory;
     }
 
+    public File getSummaryFile()
+    {
+        return summaryFile;
+    }
+
+    public void setSummaryFile( File summaryFile )
+    {
+        this.summaryFile = summaryFile;
+    }
+
     @Override
     public boolean getFailIfNoTests()
     {
@@ -383,6 +411,16 @@ public void setFailOnFlakeCount( int failOnFlakeCount )
         this.failOnFlakeCount = failOnFlakeCount;
     }
 
+    public MavenSession getSession()
+    {
+        return session;
+    }
+
+    public void setSession( MavenSession session )
+    {
+        this.session = session;
+    }
+
     private boolean existsSummaryFile()
     {
         return summaryFile != null && summaryFile.isFile();

maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/JUnit4SuiteTest.java

@@ -34,7 +34,8 @@
 @SuiteClasses( {
         IntegrationTestMojoTest.class,
         MarshallerUnmarshallerTest.class,
-        RunResultTest.class
+        RunResultTest.class,
+        VerifyMojoTest.class
 } )
 @RunWith( Suite.class )
 public class JUnit4SuiteTest

maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/VerifyMojoTest.java

@@ -0,0 +1,108 @@
+package org.apache.maven.plugin.failsafe;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.UnsupportedEncodingException;
+import java.net.URL;
+import java.net.URLDecoder;
+
+import org.apache.maven.execution.MavenExecutionRequest;
+import org.apache.maven.execution.MavenSession;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugin.MojoFailureException;
+import org.codehaus.plexus.logging.Logger;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ */
+public class VerifyMojoTest
+{
+    private VerifyMojo mojo;
+
+    @Rule
+    public TemporaryFolder tempFolder = new TemporaryFolder();
+
+    @Before
+    public void init() throws UnsupportedEncodingException
+    {
+        mojo = new VerifyMojo();
+        mojo.setTestClassesDirectory( tempFolder.getRoot() );
+        mojo.setReportsDirectory( getTestBaseDir() );
+    }
+
+    private void setupExecuteMocks()
+    {
+        Logger logger = mock( Logger.class );
+        when( logger.isErrorEnabled() ).thenReturn( true );
+        when( logger.isWarnEnabled() ).thenReturn( true );
+        when( logger.isInfoEnabled() ).thenReturn( true );
+        when( logger.isDebugEnabled() ).thenReturn( false );
+        mojo.setLogger( logger );
+
+        MavenSession session = mock( MavenSession.class );
+        MavenExecutionRequest request = mock ( MavenExecutionRequest.class );
+        when( request.isShowErrors() ).thenReturn( true );
+        when( request.getReactorFailureBehavior() ).thenReturn( null );
+        when( session.getRequest() ).thenReturn( request );
+        mojo.setSession( session );
+    }
+
+    private File getTestBaseDir()
+        throws UnsupportedEncodingException
+    {
+        URL resource = getClass().getResource( "/verify-mojo" );
+        // URLDecoder.decode necessary for JDK 1.5+, where spaces are escaped to %20
+        return new File( URLDecoder.decode( resource.getPath(), "UTF-8" ) ).getAbsoluteFile();
+    }
+
+    @Test( expected = MojoExecutionException.class )
+    public void executeForForkError() throws MojoExecutionException, MojoFailureException, UnsupportedEncodingException
+    {
+        setupExecuteMocks();
+        mojo.setSummaryFile( new File( getTestBaseDir(), "failsafe-summary-booter-fork-error.xml" ) );
+        mojo.execute();
+    }
+
+    @Test( expected = MojoExecutionException.class )
+    public void executeForForkErrorTestFailureIgnore() throws MojoExecutionException, MojoFailureException,
+        UnsupportedEncodingException
+    {
+        setupExecuteMocks();
+        mojo.setSummaryFile( new File( getTestBaseDir(), "failsafe-summary-booter-fork-error.xml" ) );
+        mojo.setTestFailureIgnore( true );
+        mojo.execute();
+    }
+
+    @Test
+    public void executeForPassingTests() throws MojoExecutionException, MojoFailureException,
+        UnsupportedEncodingException
+    {
+        setupExecuteMocks();
+        mojo.setSummaryFile( new File( getTestBaseDir(), "failsafe-summary-success.xml" ) );
+        mojo.execute();
+    }
+}

maven-failsafe-plugin/src/test/resources/verify-mojo/failsafe-summary-booter-fork-error.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<failsafe-summary xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://maven.apache.org/surefire/maven-surefire-plugin/xsd/failsafe-summary.xsd" result="254" timeout="false">
+    <completed>0</completed>
+    <errors>0</errors>
+    <failures>0</failures>
+    <skipped>0</skipped>
+    <failureMessage>org.apache.maven.surefire.booter.SurefireBooterForkException: The forked VM terminated without properly saying goodbye. VM crash or System.exit called?
+Command was /bin/sh -c cd &apos;/Users/aaron.braunstein/git/apache/maven-surefire/surefire-its/target/TestClass_testMethod&apos; &amp;&amp; &apos;/Library/Java/JavaVirtualMachines/temurin-11.jdk/Contents/Home/bin/java&apos; &apos;-Dfile.encoding=UTF-8&apos; &apos;-Duser.language=en&apos; &apos;-XFakeUnrecognizedOptionThatWillCrashJVM&apos; &apos;-Duser.region=US&apos; &apos;-showversion&apos; &apos;-Xmx6g&apos; &apos;-Xms2g&apos; &apos;-XX:+PrintGCDetails&apos; &apos;-jar&apos; &apos;/Users/aaron.braunstein/git/apache/maven-surefire/surefire-its/target/TestClass_testMethod/target/surefire/surefirebooter-20220606220315261_3.jar&apos; &apos;/Users/aaron.braunstein/git/apache/maven-surefire/surefire-its/target/TestClass_testMethod/target/surefire&apos; &apos;2022-06-06T22-03-11_910-jvmRun1&apos; &apos;surefire-20220606220315261_1tmp&apos; &apos;surefire_0-20220606220315261_2tmp&apos;
+Error occurred in starting fork, check output in log
+Process Exit Code: 1
+	at org.apache.maven.plugin.surefire.booterclient.ForkStarter.fork(ForkStarter.java:714)
+	at org.apache.maven.plugin.surefire.booterclient.ForkStarter.run(ForkStarter.java:311)
+	at org.apache.maven.plugin.surefire.booterclient.ForkStarter.run(ForkStarter.java:268)
+	at org.apache.maven.plugin.surefire.AbstractSurefireMojo.executeProvider(AbstractSurefireMojo.java:1334)
+	at org.apache.maven.plugin.surefire.AbstractSurefireMojo.executeAfterPreconditionsChecked(AbstractSurefireMojo.java:1167)
+	at org.apache.maven.plugin.surefire.AbstractSurefireMojo.execute(AbstractSurefireMojo.java:931)
+	at org.apache.maven.plugin.DefaultBuildPluginManager.executeMojo(DefaultBuildPluginManager.java:137)
+	at org.apache.maven.lifecycle.internal.MojoExecutor.doExecute(MojoExecutor.java:301)
+	at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:211)
+	at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:165)
+	at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:157)
+	at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:121)
+	at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:81)
+	at org.apache.maven.lifecycle.internal.builder.singlethreaded.SingleThreadedBuilder.build(SingleThreadedBuilder.java:56)
+	at org.apache.maven.lifecycle.internal.LifecycleStarter.execute(LifecycleStarter.java:127)
+	at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:294)
+	at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:192)
+	at org.apache.maven.DefaultMaven.execute(DefaultMaven.java:105)
+	at org.apache.maven.cli.MavenCli.execute(MavenCli.java:960)
+	at org.apache.maven.cli.MavenCli.doMain(MavenCli.java:293)
+	at org.apache.maven.cli.MavenCli.main(MavenCli.java:196)
+	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
+	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
+	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
+	at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced(Launcher.java:282)
+	at org.codehaus.plexus.classworlds.launcher.Launcher.launch(Launcher.java:225)
+	at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode(Launcher.java:406)
+	at org.codehaus.plexus.classworlds.launcher.Launcher.main(Launcher.java:347)
+</failureMessage>
+</failsafe-summary>

maven-failsafe-plugin/src/test/resources/verify-mojo/failsafe-summary-success.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<failsafe-summary xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://maven.apache.org/surefire/maven-surefire-plugin/xsd/failsafe-summary.xsd" result="254" timeout="false">
+    <completed>1</completed>
+    <errors>0</errors>
+    <failures>0</failures>
+    <skipped>0</skipped>
+    <failureMessage xsi:nil="true" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"/>
+</failsafe-summary>

surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2095FailsafeJvmCrashShouldNotBeIgnoredIT.java

@@ -0,0 +1,48 @@
+package org.apache.maven.surefire.its.jiras;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.it.VerificationException;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.Test;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Test https://issues.apache.org/jira/browse/SUREFIRE-2095
+ *
+ */
+public class Surefire2095FailsafeJvmCrashShouldNotBeIgnoredIT
+    extends SurefireJUnit4IntegrationTestCase
+{
+    @Test
+    public void mavenShouldFail() throws VerificationException
+    {
+        // Run failsafe with testFailureIgnore=true and an unknown JVM option that will cause a crash
+        unpack( "surefire-2095-failsafe-jvm-crash" )
+            .maven()
+            .withFailure()
+            .debugLogging()
+            .executeVerify()
+            .assertThatLogLine( containsString( "BUILD SUCCESS" ), is( 0 ) )
+            .verifyTextInLog( "BUILD FAILURE" );
+    }
+}

surefire-its/src/test/resources/surefire-2095-failsafe-jvm-crash/pom.xml

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.apache.maven.plugins.surefire</groupId>
+  <artifactId>SUREFIRE-2095</artifactId>
+  <version>1.0-SNAPSHOT</version>
+  <name>SUREFIRE-2095</name>
+
+  <properties>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.target>1.8</maven.compiler.target>
+  </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+        <version>${surefire.version}</version>
+        <configuration>
+          <argLine>-Dfile.encoding=UTF-8 -Duser.language=en -XFakeUnrecognizedOptionThatWillCrashJVM -Duser.region=US -showversion -Xmx6g -Xms2g -XX:+PrintGCDetails</argLine>
+          <testFailureIgnore>true</testFailureIgnore>
+        </configuration>
+        <executions>
+          <execution>
+            <id>integration-test</id>
+            <goals>
+              <goal>integration-test</goal>
+            </goals>
+          </execution>
+          <execution>
+            <id>verify</id>
+            <goals>
+              <goal>verify</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>

surefire-its/src/test/resources/surefire-2095-failsafe-jvm-crash/src/test/java/PojoIT.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+public class PojoIT
+{
+    private static int calls;
+
+    public void setUp()
+    {
+        System.out.println( "setUp called " + ++calls );
+    }
+
+    public void tearDown()
+    {
+        System.out.println( "tearDown called " + calls );
+    }
+
+    public void testSuccess()
+    {
+        assert true;
+    }
+
+    public void testFailure()
+    {
+        assert false;
+    }
+
+}