From 251423e4abf92b9ad3080799ed86131100a3496e Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Fri, 18 Mar 2022 23:28:30 +0800 Subject: [PATCH] HBASE-26861 Fix flaky TestSnapshotFromMaster.testSnapshotHFileArchiving (#4242) Signed-off-by: Xiaolin Ha (cherry picked from commit 520ad638d9dbc3cdf0d3f0706610dcf3f0a43f1e) (cherry picked from commit 7acfbc0bea83e5d2ad9929175c33cdffeec9fef3) Change-Id: I0f53154968db8fe3c58dac309dde1f63911fe2cf --- .../cleaner/TestSnapshotFromMaster.java | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java index fd309ec69775..f2d1afa8ea9c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.Waiter.ExplainingPredicate; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; @@ -488,14 +489,27 @@ public void testSnapshotHFileArchiving() throws Exception { } } // run the cleaner again - LOG.debug("Running hfile cleaners"); - ensureHFileCleanersRun(); - LOG.info("After delete snapshot cleaners run File-System state"); - CommonFSUtils.logFileSystemState(fs, rootDir, LOG); + // In SnapshotFileCache, we use lock.tryLock to avoid being blocked by snapshot operation, if we + // fail to get the lock, we will not archive any files. This is not a big deal in real world, so + // here we will try to run the cleaner multiple times to make the test more stable. + // See HBASE-26861 + UTIL.waitFor(60000, 1000, new ExplainingPredicate() { + + @Override + public boolean evaluate() throws Exception { + LOG.debug("Running hfile cleaners"); + ensureHFileCleanersRun(); + LOG.info("After delete snapshot cleaners run File-System state"); + CommonFSUtils.logFileSystemState(fs, rootDir, LOG); + return getHFiles(archiveDir, fs, TABLE_NAME).isEmpty(); + } - archives = getHFiles(archiveDir, fs, TABLE_NAME); - assertEquals("Still have some hfiles in the archive, when their snapshot has been deleted.", 0, - archives.size()); + @Override + public String explainFailure() throws Exception { + return "Still have some hfiles in the archive, when their snapshot has been deleted: " + + getHFiles(archiveDir, fs, TABLE_NAME); + } + }); } /**