[fix][build] Dump Jacoco coverage data to file with JMX interface in TestNG listener

buildtools/pom.xml

@@ -176,7 +176,7 @@
           <properties>
             <property>
               <name>listener</name>
-              <value>org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier</value>
+              <value>org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.JacocoDumpListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier</value>
             </property>
           </properties>
           <argLine>

buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java

@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.tests;
+
+import java.lang.management.ManagementFactory;
+import java.util.concurrent.TimeUnit;
+import javax.management.InstanceNotFoundException;
+import javax.management.MBeanServer;
+import javax.management.MBeanServerInvocationHandler;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import org.testng.IExecutionListener;
+import org.testng.ISuite;
+import org.testng.ISuiteListener;
+
+/**
+ * A TestNG listener that dumps Jacoco coverage data to file using the Jacoco JMX interface.
+ *
+ * This ensures that coverage data is dumped even if the shutdown sequence of the Test JVM gets stuck. Coverage
+ * data will be dumped every 2 minutes by default and once all test suites have been run.
+ * Each test class runs in its own suite when run with maven-surefire-plugin.
+ */
+public class JacocoDumpListener implements ISuiteListener, IExecutionListener {
+    private final MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer();
+    private final ObjectName jacocoObjectName;
+    private final JacocoProxy jacocoProxy;
+    private final boolean enabled;
+
+    private long lastDumpTime;
+
+    private static final long DUMP_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(120);
+
+    public JacocoDumpListener() {
+        try {
+            jacocoObjectName = new ObjectName("org.jacoco:type=Runtime");
+        } catch (MalformedObjectNameException e) {
+            // this won't happen since the ObjectName is static and valid
+            throw new RuntimeException(e);
+        }
+        enabled = checkEnabled();
+        if (enabled) {
+            jacocoProxy = MBeanServerInvocationHandler.newProxyInstance(platformMBeanServer, jacocoObjectName,
+                    JacocoProxy.class, false);
+        } else {
+            jacocoProxy = null;
+        }
+        lastDumpTime = System.currentTimeMillis();
+    }
+
+    private boolean checkEnabled() {
+        try {
+            platformMBeanServer.getObjectInstance(jacocoObjectName);
+        } catch (InstanceNotFoundException e) {
+            // jacoco jmx is not enabled
+            return false;
+        }
+        return true;
+    }
+
+    public void onFinish(ISuite suite) {
+        // dump jacoco coverage data to file using the Jacoco JMX interface if more than DUMP_INTERVAL_MILLIS has passed
+        // since the last dump
+        if (enabled && System.currentTimeMillis() - lastDumpTime > DUMP_INTERVAL_MILLIS) {
+            // dump jacoco coverage data to file using the Jacoco JMX interface
+            triggerJacocoDump();
+        }
+    }
+    @Override
+    public void onExecutionFinish() {
+        if (enabled) {
+            // dump jacoco coverage data to file using the Jacoco JMX interface when all tests have finished
+            triggerJacocoDump();
+        }
+    }
+
+    private void triggerJacocoDump() {
+        System.out.println("Dumping Jacoco coverage data to file...");
+        long start = System.currentTimeMillis();
+        jacocoProxy.dump(true);
+        lastDumpTime = System.currentTimeMillis();
+        System.out.println("Completed in " + (lastDumpTime - start) + "ms.");
+    }
+
+    public interface JacocoProxy {
+        void dump(boolean reset);
+    }
+}

pom.xml

@@ -1531,7 +1531,7 @@ flexible messaging model and an intuitive client API.</description>
           <properties>
             <property>
               <name>listener</name>
-              <value>org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener,org.apache.pulsar.tests.SingletonCleanerListener</value>
+              <value>org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.JacocoDumpListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener,org.apache.pulsar.tests.SingletonCleanerListener</value>
             </property>
           </properties>
         </configuration>
@@ -1995,6 +1995,7 @@ flexible messaging model and an intuitive client API.</description>
                   <!-- use unique jacoco exec files for every forked test process -->
                   <destFile>${project.build.directory}/jacoco_${maven.build.timestamp}_${surefire.forkNumber}.exec</destFile>
                   <append>true</append>
+                  <jmx>true</jmx>
                   <includes>
                     <include>org.apache.pulsar.*</include>
                     <include>org.apache.bookkeeper.mledger.*</include>