From f9fa857d12e39a02fffaf5fccdfefc85e9c990b2 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Fri, 18 Mar 2022 13:38:51 -0700 Subject: [PATCH] HBASE-26826 Backport StoreFileTracker (HBASE-26067, HBASE-26584, and others) to branch-2.5 Signed-off-by: Josh Elser Reviewed-by: Wellington Ramos Chevreuil --- .../org/apache/hadoop/hbase/client/HBaseAdmin.java | 8 ++++---- .../hadoop/hbase/client/TableDescriptorBuilder.java | 5 ++--- .../IntegrationTestFileBasedSFTBulkLoad.java | 3 ++- .../hadoop/hbase/regionserver/AbstractMemStore.java | 3 +-- .../hbase/regionserver/CompactingMemStore.java | 2 +- .../hadoop/hbase/regionserver/HRegionServer.java | 3 ++- .../hbase/regionserver/SecureBulkLoadManager.java | 2 +- .../hbase/snapshot/RestoreSnapshotHelper.java | 2 -- .../hadoop/hbase/regionserver/TestBulkloadBase.java | 11 +++++++---- .../regionserver/TestMergesSplitsAddToTracker.java | 1 - .../regionserver/TestSecureBulkloadListener.java | 13 ++++++++----- .../hbase/regionserver/TestStripeStoreEngine.java | 1 - .../compactions/TestDateTieredCompactor.java | 1 - .../compactions/TestStripeCompactor.java | 1 - .../hbase/snapshot/MobSnapshotTestingUtils.java | 11 ++++++----- 15 files changed, 34 insertions(+), 33 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index b87874e1994a..6cf018468ebb 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -2868,10 +2868,10 @@ private Future internalRestoreSnapshotAsync(final String snapshotName, @Override protected RestoreSnapshotResponse rpcCall() throws Exception { final RestoreSnapshotRequest.Builder builder = RestoreSnapshotRequest.newBuilder() - .setSnapshot(snapshot) - .setNonceGroup(nonceGroup) - .setNonce(nonce) - .setRestoreACL(restoreAcl); + .setSnapshot(snapshot) + .setNonceGroup(nonceGroup) + .setNonce(nonce) + .setRestoreACL(restoreAcl); if (customSFT != null) { builder.setCustomSFT(customSFT); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java index d9a15154a313..c84388dcbbf0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java @@ -818,9 +818,8 @@ public ModifyableTableDescriptor removeValue(Bytes key) { /** * Remove metadata represented by the key from the {@link #values} map * - * @param key Key whose key and value we're to remove from TableDescriptor - * parameters. - * @return the modifyable TD + * @param key Key whose key and value we're to remove from TableDescriptor parameters + * @return the modifiable TD */ public ModifyableTableDescriptor removeValue(final byte[] key) { return removeValue(new Bytes(key)); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestFileBasedSFTBulkLoad.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestFileBasedSFTBulkLoad.java index 93e51fff8aab..ca36ff8effa4 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestFileBasedSFTBulkLoad.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestFileBasedSFTBulkLoad.java @@ -66,7 +66,8 @@ @Category(IntegrationTests.class) public class IntegrationTestFileBasedSFTBulkLoad extends IntegrationTestBulkLoad { - private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestFileBasedSFTBulkLoad.class); + private static final Logger LOG = + LoggerFactory.getLogger(IntegrationTestFileBasedSFTBulkLoad.class); private static String NUM_MAPS_KEY = "hbase.IntegrationTestBulkLoad.numMaps"; private static String NUM_IMPORT_ROUNDS_KEY = "hbase.IntegrationTestBulkLoad.numImportRounds"; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java index 56dab21baf2e..0cef4a91fb7d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java @@ -232,8 +232,7 @@ public long timeOfOldestEdit() { } /** - * This method is protected under {@link HStore#lock} write lock,
- * and this method is used by {@link HStore#updateStorefiles} after flushing is completed.
+ * This method is protected under HStore write lock.
* The passed snapshot was successfully persisted; it can be let go. * @param id Id of the snapshot to clean out. * @see MemStore#snapshot() diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index eac5fc110c57..cd3caa9c3ebd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -395,7 +395,7 @@ public String getFamilyName() { } /** - * This method is protected under {@link HStore#lock} read lock. + * This method is protected under HStore read lock. */ @Override public List getScanners(long readPt) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 2bef9388021f..990b4da46f6b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -2262,7 +2262,8 @@ private void initializeThreads() { double brokenStoreFileCleanerDelayJitter = conf.getDouble( BrokenStoreFileCleaner.BROKEN_STOREFILE_CLEANER_DELAY_JITTER, BrokenStoreFileCleaner.DEFAULT_BROKEN_STOREFILE_CLEANER_DELAY_JITTER); - double jitterRate = (ThreadLocalRandom.current().nextDouble() - 0.5D) * brokenStoreFileCleanerDelayJitter; + double jitterRate = (ThreadLocalRandom.current().nextDouble() - 0.5D) * + brokenStoreFileCleanerDelayJitter; long jitterValue = Math.round(brokenStoreFileCleanerDelay * jitterRate); this.brokenStoreFileCleaner = new BrokenStoreFileCleaner((int) (brokenStoreFileCleanerDelay + jitterValue), diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java index dbc5f72e37b6..ee028f1487a5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java @@ -364,7 +364,7 @@ public SecureBulkLoadListener(FileSystem fs, String stagingDir, Configuration co @Override public String prepareBulkLoad(final byte[] family, final String srcPath, boolean copyFile, - String customStaging ) throws IOException { + String customStaging) throws IOException { Path p = new Path(srcPath); //store customStaging for failedBulkLoad diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index 17406445fc3a..e4928e8fa777 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hbase.snapshot; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkloadBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkloadBase.java index 86e41ede8294..fbf71e29b3e1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkloadBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkloadBase.java @@ -59,10 +59,10 @@ import org.junit.Rule; import org.junit.rules.TemporaryFolder; import org.junit.rules.TestName; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; @RunWith(Parameterized.class) public class TestBulkloadBase { @@ -135,7 +135,8 @@ protected HRegion testRegionWithFamiliesAndSpecifiedTableName(TableName tableNam } protected HRegion testRegionWithFamilies(byte[]... families) throws IOException { - TableName tableName = TableName.valueOf(name.getMethodName().substring(0, name.getMethodName().indexOf("["))); + TableName tableName = + TableName.valueOf(name.getMethodName().substring(0, name.getMethodName().indexOf("["))); return testRegionWithFamiliesAndSpecifiedTableName(tableName, families); } @@ -175,7 +176,9 @@ private String createHFileForFamilies(byte[] family) throws IOException { private static String generateUniqueName(final String suffix) { String name = UUID.randomUUID().toString().replaceAll("-", ""); - if (suffix != null) name += suffix; + if (suffix != null) { + name += suffix; + } return name; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java index 703d6193e5e5..b0520e353d1b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java @@ -234,7 +234,6 @@ private void validateDaughterRegionsFiles(HRegion region, String orignalFileName List infos = region.getRegionFileSystem().getStoreFiles("info"); final MutableBoolean foundLink = new MutableBoolean(false); infos.stream().forEach(i -> { - i.getActiveFileName().contains(orignalFileName); if(i.getActiveFileName().contains(untrackedFile)){ fail(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSecureBulkloadListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSecureBulkloadListener.java index bf3732a821da..7a6e411dfab0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSecureBulkloadListener.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSecureBulkloadListener.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hbase.regionserver; +import java.io.IOException; +import java.util.Random; +import java.util.UUID; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -42,9 +45,6 @@ import org.junit.experimental.categories.Category; import org.junit.rules.TemporaryFolder; import org.junit.rules.TestName; -import java.io.IOException; -import java.util.Random; -import java.util.UUID; /** * Tests for failedBulkLoad logic to make sure staged files are returned to their original location @@ -232,7 +232,8 @@ public void testDeletedStagedFile() throws Exception { private String createHFileForFamilies(byte[] family) throws IOException { HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(conf); - Path testDir = new Path(dfs.getWorkingDirectory() , new Path(name.getMethodName(), Bytes.toString(family))); + Path testDir = new Path(dfs.getWorkingDirectory(), + new Path(name.getMethodName(), Bytes.toString(family))); if(!dfs.exists(testDir)){ dfs.mkdirs(testDir); } @@ -257,7 +258,9 @@ private String createHFileForFamilies(byte[] family) throws IOException { private static String generateUniqueName(final String suffix) { String name = UUID.randomUUID().toString().replaceAll("-", ""); - if (suffix != null) name += suffix; + if (suffix != null) { + name += suffix; + } return name; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java index e30c9d9f5015..24cf3e96ce75 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java index dd844080aa82..dfbda26147de 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfoBuilder; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java index 0ae584c5aa9b..e834d66112c4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfoBuilder; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java index 4fb1ae2e1f04..cb0fad6c8fe2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java @@ -59,8 +59,9 @@ public static void createMobTable(final HBaseTestingUtility util, final TableNam storeFileTracker, families); } - public static void createPreSplitMobTable(final HBaseTestingUtility util, final TableName tableName, - int nRegions, final byte[]... families) throws IOException, InterruptedException { + public static void createPreSplitMobTable(final HBaseTestingUtility util, + final TableName tableName, int nRegions, final byte[]... families) + throws IOException, InterruptedException { createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(nRegions), 1, families); } @@ -77,9 +78,9 @@ public static void createMobTable(final HBaseTestingUtility util, final TableNam createMobTable(util, tableName, splitKeys, regionReplication, storeFileTracker, null, families); } - public static void createMobTable(HBaseTestingUtility util, TableName tableName, byte[][] splitKeys, - int regionReplication, String storeFileTracker, String cpClassName, byte[]... families) - throws IOException, InterruptedException { + public static void createMobTable(HBaseTestingUtility util, TableName tableName, + byte[][] splitKeys, int regionReplication, String storeFileTracker, String cpClassName, + byte[]... families) throws IOException, InterruptedException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(regionReplication); for (byte[] family : families) {