From c393e5b9baa28162d8adb97b9c00261304047ae6 Mon Sep 17 00:00:00 2001 From: ddupg Date: Fri, 15 Apr 2022 20:03:10 +0800 Subject: [PATCH] HBASE-26956 ExportSnapshot tool supports removing TTL --- .../hadoop/hbase/snapshot/ExportSnapshot.java | 26 ++++++--- .../hbase/snapshot/TestExportSnapshot.java | 56 +++++++++++++++++-- .../snapshot/TestExportSnapshotAdjunct.java | 4 +- .../TestExportSnapshotV1NoCluster.java | 2 +- 4 files changed, 73 insertions(+), 15 deletions(-) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java index 54c92c5ab6a0..b60af6994318 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java @@ -157,6 +157,8 @@ static final class Options { "Number of mappers to use during the copy (mapreduce.job.maps)."); static final Option BANDWIDTH = new Option(null, "bandwidth", true, "Limit bandwidth to this value in MB/second."); + static final Option RESET_TTL = new Option(null, "reset-ttl", false, + "Do not copy TTL for the snapshot"); } // Export Map-Reduce Counters, to keep track of the progress @@ -929,6 +931,7 @@ private void setPermissionParallel(final FileSystem outputFs, final short filesM private int bandwidthMB = Integer.MAX_VALUE; private int filesMode = 0; private int mappers = 0; + private boolean resetTtl = false; @Override protected void processOptions(CommandLine cmd) { @@ -950,6 +953,7 @@ protected void processOptions(CommandLine cmd) { verifyChecksum = !cmd.hasOption(Options.NO_CHECKSUM_VERIFY.getLongOpt()); verifyTarget = !cmd.hasOption(Options.NO_TARGET_VERIFY.getLongOpt()); verifySource = !cmd.hasOption(Options.NO_SOURCE_VERIFY.getLongOpt()); + resetTtl = cmd.hasOption(Options.RESET_TTL.getLongOpt()); } /** @@ -1081,14 +1085,19 @@ public int doWork() throws IOException { } } - // Write a new .snapshotinfo if the target name is different from the source name - if (!targetName.equals(snapshotName)) { - SnapshotDescription snapshotDesc = - SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir) - .toBuilder() - .setName(targetName) - .build(); - SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, initialOutputSnapshotDir, outputFs); + // Write a new .snapshotinfo if the target name is different from the source name or we want to + // reset TTL for target snapshot. + if (!targetName.equals(snapshotName) || resetTtl) { + SnapshotDescription.Builder snapshotDescBuilder = + SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir).toBuilder(); + if (!targetName.equals(snapshotName)) { + snapshotDescBuilder.setName(targetName); + } + if (resetTtl) { + snapshotDescBuilder.setTtl(HConstants.DEFAULT_SNAPSHOT_TTL); + } + SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDescBuilder.build(), + initialOutputSnapshotDir, outputFs); if (filesUser != null || filesGroup != null) { outputFs.setOwner(new Path(initialOutputSnapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), filesUser, filesGroup); @@ -1164,6 +1173,7 @@ protected void printUsage() { addOption(Options.CHMOD); addOption(Options.MAPPERS); addOption(Options.BANDWIDTH); + addOption(Options.RESET_TTL); } public static void main(String[] args) { diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java index 02aae1b341bd..7e2a6fb8183d 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java @@ -24,9 +24,12 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; @@ -230,6 +233,38 @@ public void testExportWithTargetName() throws Exception { testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles); } + @Test + public void testExportWithResetTtl() throws Exception { + String name = "testExportWithResetTtl"; + TableName tableName = TableName.valueOf(name); + String snapshotName = "snaptb-" + name; + Long ttl = 100000L; + + try { + // create Table + createTable(tableName); + SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); + int tableNumFiles = admin.getRegions(tableName).size(); + // take a snapshot with TTL + Map props = new HashMap<>(); + props.put("TTL", ttl); + admin.snapshot(snapshotName, tableName, props); + Optional ttlOpt = + admin.listSnapshots().stream().filter(s -> s.getName().equals(snapshotName)) + .map(org.apache.hadoop.hbase.client.SnapshotDescription::getTtl) + .findAny(); + assertTrue(ttlOpt.isPresent()); + assertEquals(ttl, ttlOpt.get()); + + testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, + getHdfsDestinationDir(), false, true); + } catch (Exception e) { + throw e; + } finally { + TEST_UTIL.deleteTable(tableName); + } + } + private void testExportFileSystemState(final TableName tableName, final String snapshotName, final String targetName, int filesExpected) throws Exception { testExportFileSystemState(tableName, snapshotName, targetName, @@ -239,9 +274,16 @@ private void testExportFileSystemState(final TableName tableName, final String s protected void testExportFileSystemState(final TableName tableName, final String snapshotName, final String targetName, int filesExpected, Path copyDir, boolean overwrite) throws Exception { + testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir, + overwrite, false); + } + + protected void testExportFileSystemState(final TableName tableName, + final String snapshotName, final String targetName, int filesExpected, + Path copyDir, boolean overwrite, boolean resetTtl) throws Exception { testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName, filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, - overwrite, getBypassRegionPredicate(), true); + overwrite, resetTtl, getBypassRegionPredicate(), true); } /** @@ -249,7 +291,7 @@ protected void testExportFileSystemState(final TableName tableName, */ protected static void testExportFileSystemState(final Configuration conf, final TableName tableName, final String snapshotName, final String targetName, final int filesExpected, - final Path srcDir, Path rawTgtDir, final boolean overwrite, + final Path srcDir, Path rawTgtDir, final boolean overwrite, final boolean resetTtl, final RegionPredicate bypassregionPredicate, boolean success) throws Exception { FileSystem tgtFs = rawTgtDir.getFileSystem(conf); FileSystem srcFs = srcDir.getFileSystem(conf); @@ -268,6 +310,9 @@ protected static void testExportFileSystemState(final Configuration conf, final if (overwrite) { opts.add("--overwrite"); } + if (resetTtl) { + opts.add("--reset-ttl"); + } // Export Snapshot int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()])); @@ -296,7 +341,7 @@ protected static void testExportFileSystemState(final Configuration conf, final final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName); verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir)); Set snapshotFiles = verifySnapshot(conf, tgtFs, tgtDir, tableName, - targetName, bypassregionPredicate); + targetName, resetTtl, bypassregionPredicate); assertEquals(filesExpected, snapshotFiles.size()); } @@ -313,7 +358,7 @@ protected static void verifySnapshotDir(final FileSystem fs1, final Path root1, */ protected static Set verifySnapshot(final Configuration conf, final FileSystem fs, final Path rootDir, final TableName tableName, final String snapshotName, - final RegionPredicate bypassregionPredicate) throws IOException { + final boolean resetTtl, final RegionPredicate bypassregionPredicate) throws IOException { final Path exportedSnapshot = new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName)); final Set snapshotFiles = new HashSet<>(); @@ -355,6 +400,9 @@ private void verifyNonEmptyFile(final Path path) throws IOException { SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot); assertTrue(desc.getName().equals(snapshotName)); assertTrue(desc.getTable().equals(tableName.getNameAsString())); + if (resetTtl) { + assertEquals(HConstants.DEFAULT_SNAPSHOT_TTL, desc.getTtl()); + } return snapshotFiles; } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java index 6569767ea3fc..4685fcb79feb 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java @@ -153,7 +153,7 @@ public void testExportRetry() throws Exception { conf.setInt("mapreduce.map.maxattempts", 3); TestExportSnapshot.testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), - copyDir, true, null, true); + copyDir, true, false, null, true); } /** @@ -170,6 +170,6 @@ public void testExportFailure() throws Exception { conf.setInt("mapreduce.map.maxattempts", 3); TestExportSnapshot.testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), - copyDir, true, null, false); + copyDir, true, false, null, false); } } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java index dd5ed0cc9655..ae857d5048ac 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java @@ -126,7 +126,7 @@ static void testSnapshotWithRefsExportFileSystemState(FileSystem fs, TableName tableName = builder.getTableDescriptor().getTableName(); TestExportSnapshot.testExportFileSystemState(testUtil.getConfiguration(), tableName, snapshotName, snapshotName, snapshotFilesCount, - testDir, getDestinationDir(fs, testUtil, testDir), false, null, true); + testDir, getDestinationDir(fs, testUtil, testDir), false, false, null, true); } static Path getDestinationDir(FileSystem fs, HBaseCommonTestingUtil hctu, Path testDir)