HBASE-28643 An unbounded backup failure message can cause an irrecoverable state for the given backup

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java

@@ -44,6 +44,7 @@
 @InterfaceAudience.Private
 public class BackupInfo implements Comparable<BackupInfo> {
   private static final Logger LOG = LoggerFactory.getLogger(BackupInfo.class);
+  private static final int MAX_FAILED_MESSAGE_LENGTH = 1024;
 
   public interface Filter {
     /**
@@ -253,6 +254,9 @@ public String getFailedMsg() {
   }
 
   public void setFailedMsg(String failedMsg) {
+    if (failedMsg.length() > MAX_FAILED_MESSAGE_LENGTH) {
+      failedMsg = failedMsg.substring(0, MAX_FAILED_MESSAGE_LENGTH);
+    }
     this.failedMsg = failedMsg;
   }
 

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java

@@ -293,7 +293,19 @@ public void updateBackupInfo(BackupInfo info) throws IOException {
     }
     try (Table table = connection.getTable(tableName)) {
       Put put = createPutForBackupInfo(info);
-      table.put(put);
+      try {
+        table.put(put);
+      } catch (Exception e) {
+        // If the BackupInfo update can't be processed, then we should fall back to
+        // the previous BackupInfo, but also update it to reflect the failure.
+        LOG.error("Failed to update BackupInfo for {}. Marking as failed", info.getBackupId(), e);
+        BackupInfo legacyInfo = readBackupInfo(info.getBackupId());
+        if (legacyInfo != null) {
+          legacyInfo.setFailedMsg("Failed to update BackupInfo. Error: " + e.getMessage());
+          table.put(createPutForBackupInfo(legacyInfo));
+        }
+        throw e;
+      }
     }
   }