[PLAT-16291] Restore YBDB releases from S3 storage bucket

Summary: As part of restoring a continuous backup we should also restore any releases that are present in the S3 bucket but not locally. This diff downloads the remote releases to a local directory inside of /opt/yugabyte/releases/<version> as part of every restore continuous backup execution.

Test Plan:
remove local release that was uploaded to s3 bucket
run restore continuous backup, ensure releases get downloaded

Reviewers: dshubin, svarshney, vkumar

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D40573

managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RestoreContinuousBackup.java

@@ -13,17 +13,22 @@
 import static play.mvc.Http.Status.INTERNAL_SERVER_ERROR;
 
 import com.google.inject.Inject;
+import com.typesafe.config.Config;
 import com.yugabyte.yw.commissioner.AbstractTaskBase;
 import com.yugabyte.yw.commissioner.BaseTaskDependencies;
 import com.yugabyte.yw.common.CloudUtil;
 import com.yugabyte.yw.common.CloudUtilFactory;
 import com.yugabyte.yw.common.PlatformServiceException;
+import com.yugabyte.yw.common.ReleaseContainer;
+import com.yugabyte.yw.common.ReleaseManager;
 import com.yugabyte.yw.common.Util;
 import com.yugabyte.yw.common.ha.PlatformReplicationHelper;
 import com.yugabyte.yw.common.ha.PlatformReplicationManager;
 import com.yugabyte.yw.forms.AbstractTaskParams;
 import com.yugabyte.yw.models.configs.CustomerConfig;
 import java.io.File;
+import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import lombok.extern.slf4j.Slf4j;
 
@@ -33,17 +38,23 @@ public class RestoreContinuousBackup extends AbstractTaskBase {
   private final CloudUtilFactory cloudUtilFactory;
   private final PlatformReplicationHelper replicationHelper;
   private final PlatformReplicationManager replicationManager;
+  private final Config appConfig;
+  private final ReleaseManager releaseManager;
 
   @Inject
   protected RestoreContinuousBackup(
       BaseTaskDependencies baseTaskDependencies,
       PlatformReplicationHelper replicationHelper,
       PlatformReplicationManager replicationManager,
-      CloudUtilFactory cloudUtilFactory) {
+      CloudUtilFactory cloudUtilFactory,
+      Config appConfig,
+      ReleaseManager releaseManager) {
     super(baseTaskDependencies);
     this.replicationHelper = replicationHelper;
     this.replicationManager = replicationManager;
     this.cloudUtilFactory = cloudUtilFactory;
+    this.appConfig = appConfig;
+    this.releaseManager = releaseManager;
   }
 
   public static class Params extends AbstractTaskParams {
@@ -80,6 +91,23 @@ public void run() {
       throw new PlatformServiceException(
           INTERNAL_SERVER_ERROR, "Could not download YBA backup from cloud storage.");
     }
+
+    // Restore any missing YBDB releases from remote storage
+    Set<String> toDownloadReleases =
+        cloudUtil.getRemoteReleaseVersions(customerConfig.getDataObject(), taskParams.backupDir);
+    Map<String, ReleaseContainer> localReleaseContainers =
+        releaseManager.getAllLocalReleaseContainersByVersion();
+    toDownloadReleases.removeAll(localReleaseContainers.keySet());
+    if (!cloudUtil.downloadRemoteReleases(
+        customerConfig.getDataObject(),
+        toDownloadReleases,
+        appConfig.getString(Util.YB_RELEASES_PATH),
+        taskParams.backupDir)) {
+      log.warn(
+          "Error downloading YBDB releases from remote storage location, some universe operations"
+              + " may not be permitted.");
+    }
+
     if (!replicationManager.restoreBackup(backup)) {
       throw new PlatformServiceException(INTERNAL_SERVER_ERROR, "Error restoring backup to YBA");
     }

managed/src/main/java/com/yugabyte/yw/common/AWSUtil.java

@@ -373,6 +373,102 @@ public Set<String> getRemoteReleaseVersions(CustomerConfigData configData, Strin
     return new HashSet<>();
   }
 
+  // Best effort to download YBDB releases matching releaseVersions specified. Downloads both x86
+  // and aarch64 releases. false only if Exception, true even if release version not found
+  @Override
+  public boolean downloadRemoteReleases(
+      CustomerConfigData configData,
+      Set<String> releaseVersions,
+      String releasesPath,
+      String backupDir) {
+
+    for (String version : releaseVersions) {
+      // Create the local directory if necessary inside of yb.storage.path/releases
+      Path versionPath;
+      try {
+        versionPath = Files.createDirectories(Path.of(releasesPath, version));
+      } catch (Exception e) {
+        log.error(
+            "Error creating local releases directory for version {}: {}", version, e.getMessage());
+        return false;
+      }
+
+      // Find the exact S3 file paths (x86 and aarch64) matching the version
+      try {
+        maybeDisableCertVerification();
+        CustomerConfigStorageS3Data s3Data = (CustomerConfigStorageS3Data) configData;
+        AmazonS3 client = createS3Client(s3Data);
+        CloudLocationInfo cLInfo =
+            getCloudLocationInfo(YbcBackupUtil.DEFAULT_REGION_STRING, s3Data, "");
+
+        // Get all the backups in the specified bucket/directory
+        String nextContinuationToken = null;
+
+        do {
+          ListObjectsV2Result listObjectsResult;
+          // List objects with prefix matching version number
+          ListObjectsV2Request request =
+              new ListObjectsV2Request()
+                  .withBucketName(cLInfo.bucket)
+                  .withPrefix(backupDir + "/" + YBDB_RELEASES + "/" + version);
+
+          if (StringUtils.isNotBlank(nextContinuationToken)) {
+            request.withContinuationToken(nextContinuationToken);
+          }
+
+          listObjectsResult = client.listObjectsV2(request);
+
+          if (listObjectsResult.getKeyCount() == 0) {
+            // No releases found for a specified version (best effort, might work later)
+            break;
+          }
+
+          nextContinuationToken =
+              listObjectsResult.isTruncated() ? listObjectsResult.getNextContinuationToken() : null;
+
+          List<S3ObjectSummary> releases = listObjectsResult.getObjectSummaries();
+          // Download found releases individually (same version, different arch)
+          for (S3ObjectSummary release : releases) {
+
+            // Name the local file same as S3 key basename (yugabyte-version-arch.tar.gz)
+            File localRelease =
+                versionPath
+                    .resolve(release.getKey().substring(release.getKey().lastIndexOf('/') + 1))
+                    .toFile();
+
+            log.info("Attempting to download from S3 {} to {}", release.getKey(), localRelease);
+
+            GetObjectRequest getRequest = new GetObjectRequest(cLInfo.bucket, release.getKey());
+            try (S3Object releaseObject = client.getObject(getRequest);
+                InputStream inputStream = releaseObject.getObjectContent();
+                FileOutputStream outputStream = new FileOutputStream(localRelease)) {
+              byte[] buffer = new byte[1024];
+              int bytesRead;
+              while ((bytesRead = inputStream.read(buffer)) != -1) {
+                outputStream.write(buffer, 0, bytesRead);
+              }
+
+            } catch (Exception e) {
+              log.error(
+                  "Error downloading {} to {}: {}", release.getKey(), localRelease, e.getMessage());
+              return false;
+            }
+          }
+        } while (nextContinuationToken != null);
+
+      } catch (AmazonS3Exception e) {
+        log.error("AWS Error occurred while downloading releases from S3: {}", e.getErrorMessage());
+        return false;
+      } catch (Exception e) {
+        log.error("Unexpected exception while downloading releases from S3: {}", e.getMessage());
+        return false;
+      } finally {
+        maybeEnableCertVerification();
+      }
+    }
+    return true;
+  }
+
   @Override
   public File downloadYbaBackup(CustomerConfigData configData, String backupDir, Path localDir) {
     try {

managed/src/main/java/com/yugabyte/yw/common/CloudUtil.java

@@ -140,6 +140,14 @@ public default Set<String> getRemoteReleaseVersions(
     return new HashSet<>();
   }
 
+  public default boolean downloadRemoteReleases(
+      CustomerConfigData configData,
+      Set<String> releaseVersions,
+      String releasesPath,
+      String backupDir) {
+    return false;
+  }
+
   public default RestorePreflightResponse
       generateYBBackupRestorePreflightResponseWithoutBackupObject(
           AdvancedRestorePreflightParams preflightParams, CustomerConfigData configData) {