diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java index 91c609557d5d..f0c93db4b4c2 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -34,7 +33,6 @@ import org.apache.hadoop.hbase.backup.BackupObserver; import org.apache.hadoop.hbase.backup.BackupRestoreConstants; import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage; import org.apache.hadoop.hbase.backup.master.BackupLogCleaner; import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; import org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager; @@ -267,85 +265,6 @@ public void setBackupInfo(BackupInfo backupInfo) { this.backupInfo = backupInfo; } - /** - * Get direct ancestors of the current backup. - * @param backupInfo The backup info for the current backup - * @return The ancestors for the current backup - * @throws IOException exception - */ - public ArrayList getAncestors(BackupInfo backupInfo) throws IOException { - LOG.debug("Getting the direct ancestors of the current backup {}", backupInfo.getBackupId()); - - ArrayList ancestors = new ArrayList<>(); - - // full backup does not have ancestor - if (backupInfo.getType() == BackupType.FULL) { - LOG.debug("Current backup is a full backup, no direct ancestor for it."); - return ancestors; - } - - Set tablesToCover = new HashSet<>(backupInfo.getTables()); - - // get all backup history list in descending order (newest to oldest) - ArrayList allHistoryList = getBackupHistory(true); - for (BackupInfo backup : allHistoryList) { - - BackupImage.Builder builder = BackupImage.newBuilder(); - - BackupImage image = builder.withBackupId(backup.getBackupId()).withType(backup.getType()) - .withRootDir(backup.getBackupRootDir()).withTableList(backup.getTableNames()) - .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build(); - - // If the image has a different rootDir, it cannot be an ancestor. - if (!image.getRootDir().equals(backupInfo.getBackupRootDir())) { - continue; - } - - // The ancestors consist of the most recent FULL backups that cover the list of tables - // required in the new backup and all INCREMENTAL backups that came after one of those FULL - // backups. - if (backup.getType().equals(BackupType.INCREMENTAL)) { - ancestors.add(image); - LOG.debug("Dependent incremental backup image: {BackupID={}}", image.getBackupId()); - } else { - if (tablesToCover.removeAll(image.getTableNames())) { - ancestors.add(image); - LOG.debug("Dependent full backup image: {BackupID={}}", image.getBackupId()); - - if (tablesToCover.isEmpty()) { - LOG.debug("Got {} ancestors for the current backup.", ancestors.size()); - return ancestors; - } - } - } - } - - throw new IllegalStateException( - "Unable to find full backup that contains tables: " + tablesToCover); - } - - /** - * Get the direct ancestors of this backup for one table involved. - * @param backupInfo backup info - * @param table table - * @return backupImages on the dependency list - * @throws IOException exception - */ - public ArrayList getAncestors(BackupInfo backupInfo, TableName table) - throws IOException { - ArrayList ancestors = getAncestors(backupInfo); - ArrayList tableAncestors = new ArrayList<>(); - for (BackupImage image : ancestors) { - if (image.hasTable(table)) { - tableAncestors.add(image); - if (image.getType() == BackupType.FULL) { - break; - } - } - } - return tableAncestors; - } - /* * backup system table operations */ diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java index fee2e825728e..06dad8880b5b 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java @@ -190,7 +190,7 @@ public void execute() throws IOException { backupManager.writeBackupStartCode(newStartCode); // backup complete - completeBackup(conn, backupInfo, backupManager, BackupType.FULL, conf); + completeBackup(conn, backupInfo, BackupType.FULL, conf); } catch (Exception e) { failBackup(conn, backupInfo, backupManager, e, "Unexpected BackupException : ", BackupType.FULL, conf); diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index 211e9f96c89c..b7d1c4a95cc6 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -310,7 +310,7 @@ public void execute() throws IOException { handleBulkLoad(backupInfo.getTableNames()); // backup complete - completeBackup(conn, backupInfo, backupManager, BackupType.INCREMENTAL, conf); + completeBackup(conn, backupInfo, BackupType.INCREMENTAL, conf); } catch (IOException e) { failBackup(conn, backupInfo, backupManager, e, "Unexpected Exception : ", diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java index e758ced3f846..6b2ec938a457 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java @@ -19,8 +19,11 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -271,8 +274,8 @@ public static void cleanupAndRestoreBackupSystem(Connection conn, BackupInfo bac * @param backupInfo The current backup info * @throws IOException exception */ - protected void addManifest(BackupInfo backupInfo, BackupManager backupManager, BackupType type, - Configuration conf) throws IOException { + protected void addManifest(BackupInfo backupInfo, BackupType type, Configuration conf) + throws IOException { // set the overall backup phase : store manifest backupInfo.setPhase(BackupPhase.STORE_MANIFEST); @@ -281,13 +284,65 @@ protected void addManifest(BackupInfo backupInfo, BackupManager backupManager, B // set the table region server start and end timestamps for incremental backup manifest.setIncrTimestampMap(backupInfo.getIncrTimestampMap()); } - ArrayList ancestors = backupManager.getAncestors(backupInfo); + List ancestors = getAncestors(backupInfo); for (BackupImage image : ancestors) { manifest.addDependentImage(image); } manifest.store(conf); } + /** + * Gets the direct ancestors of the currently being created backup. + * @param backupInfo The backup info for the backup being created + */ + protected List getAncestors(BackupInfo backupInfo) throws IOException { + LOG.debug("Getting the direct ancestors of the current backup {}", backupInfo.getBackupId()); + + // Full backups do not have ancestors + if (backupInfo.getType() == BackupType.FULL) { + LOG.debug("Current backup is a full backup, no direct ancestor for it."); + return Collections.emptyList(); + } + + List ancestors = new ArrayList<>(); + Set tablesToCover = new HashSet<>(backupInfo.getTables()); + + // Go over the backup history list in descending order (newest to oldest) + List allHistoryList = backupManager.getBackupHistory(true); + for (BackupInfo backup : allHistoryList) { + // If the image has a different rootDir, it cannot be an ancestor. + if (!backup.getBackupRootDir().equals(backupInfo.getBackupRootDir())) { + continue; + } + + BackupImage.Builder builder = BackupImage.newBuilder(); + BackupImage image = builder.withBackupId(backup.getBackupId()).withType(backup.getType()) + .withRootDir(backup.getBackupRootDir()).withTableList(backup.getTableNames()) + .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build(); + + // The ancestors consist of the most recent FULL backups that cover the list of tables + // required in the new backup and all INCREMENTAL backups that came after one of those FULL + // backups. + if (backup.getType().equals(BackupType.INCREMENTAL)) { + ancestors.add(image); + LOG.debug("Dependent incremental backup image: {BackupID={}}", image.getBackupId()); + } else { + if (tablesToCover.removeAll(image.getTableNames())) { + ancestors.add(image); + LOG.debug("Dependent full backup image: {BackupID={}}", image.getBackupId()); + + if (tablesToCover.isEmpty()) { + LOG.debug("Got {} ancestors for the current backup.", ancestors.size()); + return ancestors; + } + } + } + } + + throw new IllegalStateException( + "Unable to find full backup that contains tables: " + tablesToCover); + } + /** * Get backup request meta data dir as string. * @param backupInfo backup info @@ -312,15 +367,15 @@ protected String obtainBackupMetaDataStr(BackupInfo backupInfo) { * @param backupInfo backup info * @throws IOException exception */ - protected void completeBackup(final Connection conn, BackupInfo backupInfo, - BackupManager backupManager, BackupType type, Configuration conf) throws IOException { + protected void completeBackup(final Connection conn, BackupInfo backupInfo, BackupType type, + Configuration conf) throws IOException { // set the complete timestamp of the overall backup backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime()); // set overall backup status: complete backupInfo.setState(BackupState.COMPLETE); backupInfo.setProgress(100); // add and store the manifest for the backup - addManifest(backupInfo, backupManager, type, conf); + addManifest(backupInfo, type, conf); // compose the backup complete data String backupCompleteData = diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java index 7b5095a897e2..e9c1cfd9c323 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java @@ -160,7 +160,7 @@ public void execute() throws IOException { failStageIf(Stage.stage_4); // backup complete - completeBackup(conn, backupInfo, backupManager, BackupType.INCREMENTAL, conf); + completeBackup(conn, backupInfo, BackupType.INCREMENTAL, conf); } catch (Exception e) { failBackup(conn, backupInfo, backupManager, e, "Unexpected Exception : ", @@ -244,7 +244,7 @@ public void execute() throws IOException { backupManager.writeBackupStartCode(newStartCode); failStageIf(Stage.stage_4); // backup complete - completeBackup(conn, backupInfo, backupManager, BackupType.FULL, conf); + completeBackup(conn, backupInfo, BackupType.FULL, conf); } catch (Exception e) {