From 60fae6aeae776f98e1a1c963cd7301c1cd526d97 Mon Sep 17 00:00:00 2001 From: Muthu Chidambaram Date: Fri, 27 Dec 2024 22:57:11 +0000 Subject: [PATCH] [PLAT-16291] Restore YBDB releases from S3 storage bucket Summary: As part of restoring a continuous backup we should also restore any releases that are present in the S3 bucket but not locally. This diff downloads the remote releases to a local directory inside of /opt/yugabyte/releases/ as part of every restore continuous backup execution. Test Plan: remove local release that was uploaded to s3 bucket run restore continuous backup, ensure releases get downloaded Reviewers: dshubin, svarshney, vkumar Subscribers: yugaware Differential Revision: https://phorge.dev.yugabyte.com/D40573 --- .../tasks/RestoreContinuousBackup.java | 30 +++++- .../java/com/yugabyte/yw/common/AWSUtil.java | 96 +++++++++++++++++++ .../com/yugabyte/yw/common/CloudUtil.java | 8 ++ 3 files changed, 133 insertions(+), 1 deletion(-) diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RestoreContinuousBackup.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RestoreContinuousBackup.java index d2d5ea14c6d8..5e0faf6a58ea 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RestoreContinuousBackup.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/RestoreContinuousBackup.java @@ -13,17 +13,22 @@ import static play.mvc.Http.Status.INTERNAL_SERVER_ERROR; import com.google.inject.Inject; +import com.typesafe.config.Config; import com.yugabyte.yw.commissioner.AbstractTaskBase; import com.yugabyte.yw.commissioner.BaseTaskDependencies; import com.yugabyte.yw.common.CloudUtil; import com.yugabyte.yw.common.CloudUtilFactory; import com.yugabyte.yw.common.PlatformServiceException; +import com.yugabyte.yw.common.ReleaseContainer; +import com.yugabyte.yw.common.ReleaseManager; import com.yugabyte.yw.common.Util; import com.yugabyte.yw.common.ha.PlatformReplicationHelper; import com.yugabyte.yw.common.ha.PlatformReplicationManager; import com.yugabyte.yw.forms.AbstractTaskParams; import com.yugabyte.yw.models.configs.CustomerConfig; import java.io.File; +import java.util.Map; +import java.util.Set; import java.util.UUID; import lombok.extern.slf4j.Slf4j; @@ -33,17 +38,23 @@ public class RestoreContinuousBackup extends AbstractTaskBase { private final CloudUtilFactory cloudUtilFactory; private final PlatformReplicationHelper replicationHelper; private final PlatformReplicationManager replicationManager; + private final Config appConfig; + private final ReleaseManager releaseManager; @Inject protected RestoreContinuousBackup( BaseTaskDependencies baseTaskDependencies, PlatformReplicationHelper replicationHelper, PlatformReplicationManager replicationManager, - CloudUtilFactory cloudUtilFactory) { + CloudUtilFactory cloudUtilFactory, + Config appConfig, + ReleaseManager releaseManager) { super(baseTaskDependencies); this.replicationHelper = replicationHelper; this.replicationManager = replicationManager; this.cloudUtilFactory = cloudUtilFactory; + this.appConfig = appConfig; + this.releaseManager = releaseManager; } public static class Params extends AbstractTaskParams { @@ -80,6 +91,23 @@ public void run() { throw new PlatformServiceException( INTERNAL_SERVER_ERROR, "Could not download YBA backup from cloud storage."); } + + // Restore any missing YBDB releases from remote storage + Set toDownloadReleases = + cloudUtil.getRemoteReleaseVersions(customerConfig.getDataObject(), taskParams.backupDir); + Map localReleaseContainers = + releaseManager.getAllLocalReleaseContainersByVersion(); + toDownloadReleases.removeAll(localReleaseContainers.keySet()); + if (!cloudUtil.downloadRemoteReleases( + customerConfig.getDataObject(), + toDownloadReleases, + appConfig.getString(Util.YB_RELEASES_PATH), + taskParams.backupDir)) { + log.warn( + "Error downloading YBDB releases from remote storage location, some universe operations" + + " may not be permitted."); + } + if (!replicationManager.restoreBackup(backup)) { throw new PlatformServiceException(INTERNAL_SERVER_ERROR, "Error restoring backup to YBA"); } diff --git a/managed/src/main/java/com/yugabyte/yw/common/AWSUtil.java b/managed/src/main/java/com/yugabyte/yw/common/AWSUtil.java index b4a48481883a..80834f5a9733 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/AWSUtil.java +++ b/managed/src/main/java/com/yugabyte/yw/common/AWSUtil.java @@ -373,6 +373,102 @@ public Set getRemoteReleaseVersions(CustomerConfigData configData, Strin return new HashSet<>(); } + // Best effort to download YBDB releases matching releaseVersions specified. Downloads both x86 + // and aarch64 releases. false only if Exception, true even if release version not found + @Override + public boolean downloadRemoteReleases( + CustomerConfigData configData, + Set releaseVersions, + String releasesPath, + String backupDir) { + + for (String version : releaseVersions) { + // Create the local directory if necessary inside of yb.storage.path/releases + Path versionPath; + try { + versionPath = Files.createDirectories(Path.of(releasesPath, version)); + } catch (Exception e) { + log.error( + "Error creating local releases directory for version {}: {}", version, e.getMessage()); + return false; + } + + // Find the exact S3 file paths (x86 and aarch64) matching the version + try { + maybeDisableCertVerification(); + CustomerConfigStorageS3Data s3Data = (CustomerConfigStorageS3Data) configData; + AmazonS3 client = createS3Client(s3Data); + CloudLocationInfo cLInfo = + getCloudLocationInfo(YbcBackupUtil.DEFAULT_REGION_STRING, s3Data, ""); + + // Get all the backups in the specified bucket/directory + String nextContinuationToken = null; + + do { + ListObjectsV2Result listObjectsResult; + // List objects with prefix matching version number + ListObjectsV2Request request = + new ListObjectsV2Request() + .withBucketName(cLInfo.bucket) + .withPrefix(backupDir + "/" + YBDB_RELEASES + "/" + version); + + if (StringUtils.isNotBlank(nextContinuationToken)) { + request.withContinuationToken(nextContinuationToken); + } + + listObjectsResult = client.listObjectsV2(request); + + if (listObjectsResult.getKeyCount() == 0) { + // No releases found for a specified version (best effort, might work later) + break; + } + + nextContinuationToken = + listObjectsResult.isTruncated() ? listObjectsResult.getNextContinuationToken() : null; + + List releases = listObjectsResult.getObjectSummaries(); + // Download found releases individually (same version, different arch) + for (S3ObjectSummary release : releases) { + + // Name the local file same as S3 key basename (yugabyte-version-arch.tar.gz) + File localRelease = + versionPath + .resolve(release.getKey().substring(release.getKey().lastIndexOf('/') + 1)) + .toFile(); + + log.info("Attempting to download from S3 {} to {}", release.getKey(), localRelease); + + GetObjectRequest getRequest = new GetObjectRequest(cLInfo.bucket, release.getKey()); + try (S3Object releaseObject = client.getObject(getRequest); + InputStream inputStream = releaseObject.getObjectContent(); + FileOutputStream outputStream = new FileOutputStream(localRelease)) { + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + + } catch (Exception e) { + log.error( + "Error downloading {} to {}: {}", release.getKey(), localRelease, e.getMessage()); + return false; + } + } + } while (nextContinuationToken != null); + + } catch (AmazonS3Exception e) { + log.error("AWS Error occurred while downloading releases from S3: {}", e.getErrorMessage()); + return false; + } catch (Exception e) { + log.error("Unexpected exception while downloading releases from S3: {}", e.getMessage()); + return false; + } finally { + maybeEnableCertVerification(); + } + } + return true; + } + @Override public File downloadYbaBackup(CustomerConfigData configData, String backupDir, Path localDir) { try { diff --git a/managed/src/main/java/com/yugabyte/yw/common/CloudUtil.java b/managed/src/main/java/com/yugabyte/yw/common/CloudUtil.java index 92eca9e455dc..e9468cc60414 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/CloudUtil.java +++ b/managed/src/main/java/com/yugabyte/yw/common/CloudUtil.java @@ -140,6 +140,14 @@ public default Set getRemoteReleaseVersions( return new HashSet<>(); } + public default boolean downloadRemoteReleases( + CustomerConfigData configData, + Set releaseVersions, + String releasesPath, + String backupDir) { + return false; + } + public default RestorePreflightResponse generateYBBackupRestorePreflightResponseWithoutBackupObject( AdvancedRestorePreflightParams preflightParams, CustomerConfigData configData) {