From a196c17e852d74e69bfb7e4a7146e86f4cdd0a08 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Fri, 18 Mar 2022 15:52:09 +0800 Subject: [PATCH] HBASE-26861 Fix flaky TestSnapshotFromMaster.testSnapshotHFileArchiving --- .../cleaner/TestSnapshotFromMaster.java | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java index 08a7b5ce9ec2..5fc60a7c0908 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.Waiter.ExplainingPredicate; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; @@ -488,14 +489,27 @@ public void testSnapshotHFileArchiving() throws Exception { } } // run the cleaner again - LOG.debug("Running hfile cleaners"); - ensureHFileCleanersRun(); - LOG.info("After delete snapshot cleaners run File-System state"); - CommonFSUtils.logFileSystemState(fs, rootDir, LOG); + // In SnapshotFileCache, we use lock.tryLock to avoid being blocked by snapshot operation, if we + // fail to get the lock, we will not archive any files. This is not a big deal in real world, so + // here we will try to run the cleaner multiple times to make the test more stable. + // See HBASE-26861 + UTIL.waitFor(60000, 1000, new ExplainingPredicate() { + + @Override + public boolean evaluate() throws Exception { + LOG.debug("Running hfile cleaners"); + ensureHFileCleanersRun(); + LOG.info("After delete snapshot cleaners run File-System state"); + CommonFSUtils.logFileSystemState(fs, rootDir, LOG); + return getHFiles(archiveDir, fs, TABLE_NAME).isEmpty(); + } - archives = getHFiles(archiveDir, fs, TABLE_NAME); - assertEquals("Still have some hfiles in the archive, when their snapshot has been deleted.", 0, - archives.size()); + @Override + public String explainFailure() throws Exception { + return "Still have some hfiles in the archive, when their snapshot has been deleted: " + + getHFiles(archiveDir, fs, TABLE_NAME); + } + }); } /**