From 6ee403ef99f88d755f51160acccec1b974f3eb3c Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Thu, 18 Jul 2024 07:45:42 -0400 Subject: [PATCH] HBASE-28686 MapReduceBackupCopyJob should support custom DistCp options (#6017) Co-authored-by: Ray Mattingly Signed-off-by: Duo Zhang Signed-off-by: Nick Dimiduk --- .../mapreduce/MapReduceBackupCopyJob.java | 37 ++++++++++++- .../mapreduce/TestMapReduceBackupCopyJob.java | 55 +++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/mapreduce/TestMapReduceBackupCopyJob.java diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyJob.java index 51a276df4c5a..747bd3e217d9 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyJob.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyJob.java @@ -22,6 +22,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.math.BigDecimal; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -45,6 +46,7 @@ import org.apache.hadoop.tools.CopyListingFileStatus; import org.apache.hadoop.tools.DistCp; import org.apache.hadoop.tools.DistCpConstants; +import org.apache.hadoop.tools.DistCpOptionSwitch; import org.apache.hadoop.tools.DistCpOptions; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -58,6 +60,10 @@ @InterfaceAudience.Private public class MapReduceBackupCopyJob implements BackupCopyJob { public static final String NUMBER_OF_LEVELS_TO_PRESERVE_KEY = "num.levels.preserve"; + + // This prefix specifies the DistCp options to be used during backup copy + public static final String BACKUP_COPY_OPTION_PREFIX = "hbase.backup.copy."; + private static final Logger LOG = LoggerFactory.getLogger(MapReduceBackupCopyJob.class); private Configuration conf; @@ -394,7 +400,15 @@ public int copy(BackupInfo context, BackupManager backupManager, Configuration c if (!destfs.exists(dest)) { destfs.mkdirs(dest); } - res = distcp.run(newOptions); + + List distCpOptionsFromConf = parseDistCpOptions(conf); + String[] finalOptions = new String[newOptions.length + distCpOptionsFromConf.size()]; + for (int i = 0; i < distCpOptionsFromConf.size(); i++) { + finalOptions[i] = distCpOptionsFromConf.get(i); + } + System.arraycopy(newOptions, 0, finalOptions, distCpOptionsFromConf.size(), + newOptions.length); + res = distcp.run(finalOptions); } return res; @@ -425,4 +439,25 @@ public void cancel(String jobId) throws IOException { } } + protected static List parseDistCpOptions(Configuration conf) { + List extraArgsFromConf = new ArrayList<>(); + + for (DistCpOptionSwitch optionSwitch : DistCpOptionSwitch.values()) { + String configLabel = BACKUP_COPY_OPTION_PREFIX + optionSwitch.getConfigLabel(); + if (conf.get(configLabel) != null) { + if (optionSwitch.getOption().hasArg()) { + extraArgsFromConf.add("-" + optionSwitch.getOption().getOpt()); + extraArgsFromConf.add(conf.get(configLabel)); + } else { + boolean value = conf.getBoolean(configLabel, false); + if (value) { + extraArgsFromConf.add("-" + optionSwitch.getOption().getOpt()); + } + } + } + } + + return extraArgsFromConf; + } + } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/mapreduce/TestMapReduceBackupCopyJob.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/mapreduce/TestMapReduceBackupCopyJob.java new file mode 100644 index 000000000000..6e35815c84f8 --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/mapreduce/TestMapReduceBackupCopyJob.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup.mapreduce; + +import static org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupCopyJob.BACKUP_COPY_OPTION_PREFIX; +import static org.apache.hadoop.tools.DistCpConstants.CONF_LABEL_DIRECT_WRITE; +import static org.apache.hadoop.tools.DistCpConstants.CONF_LABEL_MAX_MAPS; +import static org.junit.Assert.assertEquals; + +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; + +@Category(SmallTests.class) +public class TestMapReduceBackupCopyJob { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMapReduceBackupCopyJob.class); + + @Test + public void testDistCpOptionParsing() { + Configuration conf = new Configuration(); + conf.setInt(BACKUP_COPY_OPTION_PREFIX + CONF_LABEL_MAX_MAPS, 1000); + conf.setBoolean(BACKUP_COPY_OPTION_PREFIX + CONF_LABEL_DIRECT_WRITE, true); + List args = MapReduceBackupCopyJob.parseDistCpOptions(conf); + + List expectedArgs = + ImmutableList. builder().add("-m", "1000").add("-direct").build(); + + assertEquals(args, expectedArgs); + } + +}