From f4a9985826474f3af32f6ec5cc9a56a279016bbf Mon Sep 17 00:00:00 2001 From: xicm Date: Mon, 14 Mar 2022 16:30:43 +0800 Subject: [PATCH 1/2] HBASE-26720, validate the source snapshot before copying files --- .../apache/hadoop/hbase/snapshot/ExportSnapshot.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java index 17086579aa1c..d9a955a70432 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java @@ -143,6 +143,8 @@ static final class Options { "Do not verify checksum, use name+length only."); static final Option NO_TARGET_VERIFY = new Option(null, "no-target-verify", false, "Do not verify the integrity of the exported snapshot."); + static final Option NO_SOURCE_VERIFY = new Option(null, "no-source-verify", false, + "Do not verify the source of the snapshot."); static final Option OVERWRITE = new Option(null, "overwrite", false, "Rewrite the snapshot manifest if already exists."); static final Option CHUSER = new Option(null, "chuser", true, @@ -915,6 +917,7 @@ private void setPermissionParallel(final FileSystem outputFs, final short filesM } private boolean verifyTarget = true; + private boolean verifySource = true; private boolean verifyChecksum = true; private String snapshotName = null; private String targetName = null; @@ -946,6 +949,7 @@ protected void processOptions(CommandLine cmd) { // And verifyChecksum and verifyTarget with values read from old args in processOldArgs(...). verifyChecksum = !cmd.hasOption(Options.NO_CHECKSUM_VERIFY.getLongOpt()); verifyTarget = !cmd.hasOption(Options.NO_TARGET_VERIFY.getLongOpt()); + verifySource = !cmd.hasOption(Options.NO_SOURCE_VERIFY.getLongOpt()); } /** @@ -996,6 +1000,12 @@ public int doWork() throws IOException { LOG.debug("outputFs={}, outputRoot={}, skipTmp={}, initialOutputSnapshotDir={}", outputFs, outputRoot.toString(), skipTmp, initialOutputSnapshotDir); + // Verify snapshot source before copying files + if (verifySource) { + LOG.info("Verify snapshot source."); + verifySnapshot(srcConf, inputFs, inputRoot, snapshotDir); + } + // Find the necessary directory which need to change owner and group Path needSetOwnerDir = SnapshotDescriptionUtils.getSnapshotRootDir(outputRoot); if (outputFs.exists(needSetOwnerDir)) { @@ -1146,6 +1156,7 @@ protected void printUsage() { addOption(Options.TARGET_NAME); addOption(Options.NO_CHECKSUM_VERIFY); addOption(Options.NO_TARGET_VERIFY); + addOption(Options.NO_SOURCE_VERIFY); addOption(Options.OVERWRITE); addOption(Options.CHUSER); addOption(Options.CHGROUP); From ed390135cc6505dd9e025705fa8a1f6078cb091c Mon Sep 17 00:00:00 2001 From: xicm Date: Wed, 16 Mar 2022 10:31:18 +0800 Subject: [PATCH 2/2] update log about check of snapshot source --- .../java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java index d9a955a70432..54c92c5ab6a0 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java @@ -1002,7 +1002,8 @@ public int doWork() throws IOException { // Verify snapshot source before copying files if (verifySource) { - LOG.info("Verify snapshot source."); + LOG.info("Verify snapshot source, inputFs={}, inputRoot={}, snapshotDir={}.", + inputFs.getUri(), inputRoot, snapshotDir); verifySnapshot(srcConf, inputFs, inputRoot, snapshotDir); }