HBASE-27720 TestClusterRestartFailover is flakey due to metrics asser…

…tion

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
@@ -131,10 +132,17 @@ public void test() throws Exception {
       .getServerNode(SERVER_FOR_TEST);
     assertNull("serverNode should be deleted after SCP finished", serverNode);
 
+    // this test has been flakey. When it is rerun by surefire, the underlying minicluster isn't
+    // completely cleaned. specifically, the metics system isn't reset. The result is an otherwise
+    // successful re-run is failed because there's 8 or 12 SCPcounts instead of the 4 that a
+    // single run of the test would otherwise produce. Thus, assert of a multiple of 4.
     MetricsMasterSource masterSource =
       UTIL.getHBaseCluster().getMaster().getMasterMetrics().getMetricsSource();
-    metricsHelper.assertCounter(MetricsMasterSource.SERVER_CRASH_METRIC_PREFIX + "SubmittedCount",
-      4, masterSource);
+    final String ScpCountMetricName =
+      MetricsMasterSource.SERVER_CRASH_METRIC_PREFIX + "SubmittedCount";
+    final long scpMetricValue = metricsHelper.getCounter(ScpCountMetricName, masterSource);
+    assertEquals("Metric " + ScpCountMetricName + " has unexpected value " + scpMetricValue, 0,
+      scpMetricValue % 4);
   }
 
   private void setupCluster() throws Exception {