From 3d25ba9fee91b92374305710a9d335429ef23167 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 28 Mar 2023 11:16:18 +0300 Subject: [PATCH 1/2] [fix][build] Dump Jacoco coverage data to file with JMX interface in TestNG listener - sometimes the default Jacoco shutdown hook doesn't run and there's no coverage data --- buildtools/pom.xml | 2 +- .../pulsar/tests/JacocoDumpListener.java | 102 ++++++++++++++++++ pom.xml | 3 +- 3 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java diff --git a/buildtools/pom.xml b/buildtools/pom.xml index de52ac0930a33..b22abf286f693 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -176,7 +176,7 @@ listener - org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier + org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.JacocoDumpListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java new file mode 100644 index 0000000000000..ad9f976454b53 --- /dev/null +++ b/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests; + +import java.lang.management.ManagementFactory; +import java.util.concurrent.TimeUnit; +import javax.management.InstanceNotFoundException; +import javax.management.MBeanServer; +import javax.management.MBeanServerInvocationHandler; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import org.testng.ISuite; +import org.testng.ISuiteListener; +import org.testng.ITestContext; +import org.testng.ITestListener; + +/** + * A TestNG listener that dumps Jacoco coverage data to file using the Jacoco JMX interface. + * + * This ensures that coverage data is dumped even if the shutdown sequence of the Test JVM gets stuck. Coverage + * data will be dumped every 2 minutes by default and at the end of the test suite. + */ +public class JacocoDumpListener implements ITestListener, ISuiteListener { + private final MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer(); + private final ObjectName jacocoObjectName; + private final JacocoProxy jacocoProxy; + private final boolean enabled; + + private long lastDumpTime; + + private static final long DUMP_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(120); + + public JacocoDumpListener() { + try { + jacocoObjectName = new ObjectName("org.jacoco:type=Runtime"); + } catch (MalformedObjectNameException e) { + // this won't happen since the ObjectName is static and valid + throw new RuntimeException(e); + } + enabled = checkEnabled(); + if (enabled) { + jacocoProxy = MBeanServerInvocationHandler.newProxyInstance(platformMBeanServer, jacocoObjectName, + JacocoProxy.class, false); + } else { + jacocoProxy = null; + } + } + + private boolean checkEnabled() { + try { + platformMBeanServer.getObjectInstance(jacocoObjectName); + } catch (InstanceNotFoundException e) { + // jacoco jmx is not enabled + return false; + } + return true; + } + + public void onFinish(ITestContext context) { + // dump jacoco coverage data to file using the Jacoco JMX interface if more than DUMP_INTERVAL_MILLIS has passed + // since the last dump + if (enabled && (lastDumpTime == 0L || System.currentTimeMillis() - lastDumpTime > DUMP_INTERVAL_MILLIS)) { + // dump jacoco coverage data to file using the Jacoco JMX interface + triggerJacocoDump(); + } + } + @Override + public void onFinish(ISuite suite) { + if (enabled) { + // dump jacoco coverage data to file using the Jacoco JMX interface when all tests have finished + triggerJacocoDump(); + } + } + + private void triggerJacocoDump() { + System.out.println("Dumping Jacoco coverage data to file..."); + long start = System.currentTimeMillis(); + jacocoProxy.dump(true); + lastDumpTime = System.currentTimeMillis(); + System.out.println("Completed in " + (lastDumpTime - start) + "ms."); + } + + public interface JacocoProxy { + void dump(boolean reset); + } +} diff --git a/pom.xml b/pom.xml index 3f33069b8c252..ac640f1a6a05f 100644 --- a/pom.xml +++ b/pom.xml @@ -1531,7 +1531,7 @@ flexible messaging model and an intuitive client API. listener - org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener,org.apache.pulsar.tests.SingletonCleanerListener + org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.JacocoDumpListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener,org.apache.pulsar.tests.SingletonCleanerListener @@ -1995,6 +1995,7 @@ flexible messaging model and an intuitive client API. ${project.build.directory}/jacoco_${maven.build.timestamp}_${surefire.forkNumber}.exec true + true org.apache.pulsar.* org.apache.bookkeeper.mledger.* From c5672e180463fb3f457786945337aa06626c11d3 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 28 Mar 2023 14:59:10 +0300 Subject: [PATCH 2/2] Revisit dumping logic so that dumps every 2 minutes or after all tests have been executed --- .../apache/pulsar/tests/JacocoDumpListener.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java index ad9f976454b53..2c49d5118ae52 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java @@ -25,18 +25,18 @@ import javax.management.MBeanServerInvocationHandler; import javax.management.MalformedObjectNameException; import javax.management.ObjectName; +import org.testng.IExecutionListener; import org.testng.ISuite; import org.testng.ISuiteListener; -import org.testng.ITestContext; -import org.testng.ITestListener; /** * A TestNG listener that dumps Jacoco coverage data to file using the Jacoco JMX interface. * * This ensures that coverage data is dumped even if the shutdown sequence of the Test JVM gets stuck. Coverage - * data will be dumped every 2 minutes by default and at the end of the test suite. + * data will be dumped every 2 minutes by default and once all test suites have been run. + * Each test class runs in its own suite when run with maven-surefire-plugin. */ -public class JacocoDumpListener implements ITestListener, ISuiteListener { +public class JacocoDumpListener implements ISuiteListener, IExecutionListener { private final MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer(); private final ObjectName jacocoObjectName; private final JacocoProxy jacocoProxy; @@ -60,6 +60,7 @@ public JacocoDumpListener() { } else { jacocoProxy = null; } + lastDumpTime = System.currentTimeMillis(); } private boolean checkEnabled() { @@ -72,16 +73,16 @@ private boolean checkEnabled() { return true; } - public void onFinish(ITestContext context) { + public void onFinish(ISuite suite) { // dump jacoco coverage data to file using the Jacoco JMX interface if more than DUMP_INTERVAL_MILLIS has passed // since the last dump - if (enabled && (lastDumpTime == 0L || System.currentTimeMillis() - lastDumpTime > DUMP_INTERVAL_MILLIS)) { + if (enabled && System.currentTimeMillis() - lastDumpTime > DUMP_INTERVAL_MILLIS) { // dump jacoco coverage data to file using the Jacoco JMX interface triggerJacocoDump(); } } @Override - public void onFinish(ISuite suite) { + public void onExecutionFinish() { if (enabled) { // dump jacoco coverage data to file using the Jacoco JMX interface when all tests have finished triggerJacocoDump();