[Az.RecoveryServices.Backup] Adding Cross Region Restore feature

src/RecoveryServices/RecoveryServices.Backup.Models/CmdletParamEnums.cs

@@ -80,6 +80,12 @@ public enum RestoreWLBackupItemParams
         WLRecoveryConfig
     }
 
+    public enum CRRParams
+    {
+        UseSecondaryRegion,
+        SecondaryRegion
+    }
+
     public enum WorkloadRecoveryConfigParams
     {
         PointInTime,

src/RecoveryServices/RecoveryServices.Backup.Providers/AzureWorkloadProviderHelper.cs

@@ -102,7 +102,8 @@ public List<ProtectedItemResource> ListProtectedItemsByContainer(
             CmdletModel.ContainerBase container,
             CmdletModel.PolicyBase policy,
             string backupManagementType,
-            string dataSourceType)
+            string dataSourceType, 
+            bool UseSecondaryRegion = false)
         {
             ODataQuery<ProtectedItemQueryObject> queryParams = policy != null ?
                 new ODataQuery<ProtectedItemQueryObject>(
@@ -113,16 +114,30 @@ public List<ProtectedItemResource> ListProtectedItemsByContainer(
                 new ODataQuery<ProtectedItemQueryObject>(
                     q => q.BackupManagementType
                             == backupManagementType &&
-                         q.ItemType == dataSourceType);
+                         q.ItemType == dataSourceType);            
 
             List<ProtectedItemResource> protectedItems = new List<ProtectedItemResource>();
             string skipToken = null;
-            var listResponse = ServiceClientAdapter.ListProtectedItem(
+
+            // fetching backup items from secondary region
+            if (UseSecondaryRegion)
+            {
+                var listResponse = ServiceClientAdapter.ListCrrProtectedItem(
                 queryParams,
                 skipToken,
                 vaultName: vaultName,
                 resourceGroupName: resourceGroupName);
-            protectedItems.AddRange(listResponse);
+                protectedItems.AddRange(listResponse);
+            }
+            else
+            {
+                var listResponse = ServiceClientAdapter.ListProtectedItem(
+                queryParams,
+                skipToken,
+                vaultName: vaultName,
+                resourceGroupName: resourceGroupName);
+                protectedItems.AddRange(listResponse);
+            }            
 
             if (container != null)
             {
@@ -200,7 +215,6 @@ public List<ProtectedItemResource> ListProtectedItemsByContainer(
                 }
             }
 
-
             List<CmdletModel.ItemBase> itemModels = ConversionHelpers.GetItemModelList(protectedItems);
 
             if (!string.IsNullOrEmpty(itemName))
@@ -279,7 +293,7 @@ public void ValidateSQLSchedulePolicy(CmdletModel.SchedulePolicyBase policy)
             // call validation
             policy.Validate();
         }
-                
+
         public void ValidateLongTermRetentionPolicy(CmdletModel.RetentionPolicyBase policy, string backupManagementType = "")
         {
             if (policy == null || policy.GetType() != typeof(CmdletModel.LongTermRetentionPolicy))
@@ -357,6 +371,7 @@ public List<RecoveryPointBase> ListRecoveryPoints(Dictionary<Enum, object> Provi
             DateTime endDate = (DateTime)(ProviderData[RecoveryPointParams.EndDate]);
             string restorePointQueryType = ProviderData.ContainsKey(RecoveryPointParams.RestorePointQueryType) ?
                 (string)ProviderData[RecoveryPointParams.RestorePointQueryType] : "All";
+            bool secondaryRegion = (bool)ProviderData[CRRParams.UseSecondaryRegion];
 
             ItemBase item = ProviderData[RecoveryPointParams.Item] as ItemBase;
 
@@ -394,12 +409,27 @@ public List<RecoveryPointBase> ListRecoveryPoints(Dictionary<Enum, object> Provi
             ODataQuery<BMSRPQueryObject> queryFilter = new ODataQuery<BMSRPQueryObject>();
             queryFilter.Filter = queryFilterString;
 
-            List<RecoveryPointResource> rpListResponse = ServiceClientAdapter.GetRecoveryPoints(
+            List<RecoveryPointResource> rpListResponse; 
+            if (secondaryRegion)
+            {
+                //fetch recovery points from secondary region
+                rpListResponse = ServiceClientAdapter.GetRecoveryPointsFromSecondaryRegion(
                 containerUri,
                 protectedItemName,
                 queryFilter,
                 vaultName: vaultName,
                 resourceGroupName: resourceGroupName);
+            }
+            else
+            {
+                rpListResponse = ServiceClientAdapter.GetRecoveryPoints(
+                containerUri,
+                protectedItemName,
+                queryFilter,
+                vaultName: vaultName,
+                resourceGroupName: resourceGroupName);
+            }
+
             return RecoveryPointConversions.GetPSAzureRecoveryPoints(rpListResponse, item);
         }
 

src/RecoveryServices/RecoveryServices.Backup.Providers/Providers/AzureWorkloadPsBackupProvider.cs

@@ -265,6 +265,7 @@ public List<ItemBase> ListProtectedItems()
                 (ItemProtectionState)ProviderData[ItemParams.ProtectionState];
             CmdletModel.WorkloadType workloadType =
                 (CmdletModel.WorkloadType)ProviderData[ItemParams.WorkloadType];
+            bool UseSecondaryRegion = (bool)ProviderData[CRRParams.UseSecondaryRegion];
             PolicyBase policy = (PolicyBase)ProviderData[PolicyParams.ProtectionPolicy];
 
             // 1. Filter by container
@@ -274,7 +275,8 @@ public List<ItemBase> ListProtectedItems()
                 container,
                 policy,
                 ServiceClientModel.BackupManagementType.AzureWorkload,
-                DataSourceType.SQLDataBase);
+                DataSourceType.SQLDataBase,
+                UseSecondaryRegion);
 
             List<ProtectedItemResource> protectedItemGetResponses =
                 new List<ProtectedItemResource>();
@@ -403,6 +405,9 @@ public RestAzureNS.AzureOperationResponse TriggerRestore()
                 (AzureWorkloadRecoveryConfig)ProviderData[RestoreWLBackupItemParams.WLRecoveryConfig];
             RestoreRequestResource triggerRestoreRequest = new RestoreRequestResource();
 
+            bool useSecondaryRegion = (bool)ProviderData[CRRParams.UseSecondaryRegion];
+            String secondaryRegion = useSecondaryRegion ? (string)ProviderData[CRRParams.SecondaryRegion] : null;
+
             if (wLRecoveryConfig.RecoveryPoint.ContainerName != null && wLRecoveryConfig.FullRP == null)
             {
                 AzureWorkloadSQLRestoreRequest azureWorkloadSQLRestoreRequest =
@@ -481,14 +486,39 @@ public RestAzureNS.AzureOperationResponse TriggerRestore()
                 triggerRestoreRequest.Properties = azureWorkloadSQLPointInTimeRestoreRequest;
             }
 
-            var response = ServiceClientAdapter.RestoreDisk(
+            if (useSecondaryRegion)
+            {
+                AzureRecoveryPoint rp = (AzureRecoveryPoint)wLRecoveryConfig.RecoveryPoint;
+
+                // get access token
+                CrrAccessToken accessToken = ServiceClientAdapter.GetCRRAccessToken(rp, secondaryRegion, vaultName: vaultName, resourceGroupName: resourceGroupName);
+
+                // AzureWorkload  CRR Request
+                Logger.Instance.WriteDebug("Triggering Restore to secondary region: " + secondaryRegion);
+
+                CrossRegionRestoreRequest crrRestoreRequest = new CrossRegionRestoreRequest();
+                crrRestoreRequest.CrossRegionRestoreAccessDetails = accessToken;
+                crrRestoreRequest.RestoreRequest = triggerRestoreRequest.Properties;
+
+                // storage account location isn't required in case of workload restore
+                var response = ServiceClientAdapter.RestoreDiskSecondryRegion(
+                    rp,
+                    triggerCRRRestoreRequest: crrRestoreRequest,
+                    secondaryRegion: secondaryRegion);
+                return response;
+            }
+            else
+            {
+                var response = ServiceClientAdapter.RestoreDisk(
                 (AzureRecoveryPoint)wLRecoveryConfig.RecoveryPoint,
                 "LocationNotRequired",
                 triggerRestoreRequest,
                 vaultName: vaultName,
                 resourceGroupName: resourceGroupName,
                 vaultLocation: vaultLocation);
-            return response;
+                return response;
+            }          
+
         }
 
         private RestAzureNS.AzureOperationResponse<ProtectionPolicyResource> CreateorModifyPolicy()

src/RecoveryServices/RecoveryServices.Backup.Providers/Providers/IaasVmPsBackupProvider.cs

@@ -19,7 +19,6 @@
 using Microsoft.Azure.Commands.RecoveryServices.Backup.Properties;
 using Microsoft.Azure.Management.Internal.Resources.Models;
 using Microsoft.Azure.Management.RecoveryServices.Backup.Models;
-using Microsoft.Rest;
 using Microsoft.Rest.Azure.OData;
 using System;
 using System.Collections.Generic;
@@ -391,6 +390,8 @@ public RestAzureNS.AzureOperationResponse TriggerRestore()
             SwitchParameter restoreAsUnmanagedDisks = (SwitchParameter)ProviderData[RestoreVMBackupItemParams.RestoreAsUnmanagedDisks];
             String DiskEncryptionSetId = ProviderData.ContainsKey(RestoreVMBackupItemParams.DiskEncryptionSetId) ?
                 (string)ProviderData[RestoreVMBackupItemParams.DiskEncryptionSetId].ToString() : null;
+            bool useSecondaryRegion = (bool)ProviderData[CRRParams.UseSecondaryRegion];                        
+            String secondaryRegion = useSecondaryRegion ? (string)ProviderData[CRRParams.SecondaryRegion]: null;
 
             Dictionary<UriEnums, string> uriDict = HelperUtils.ParseUri(rp.Id);
             string containerUri = HelperUtils.GetContainerUri(uriDict, rp.Id);
@@ -453,15 +454,40 @@ public RestAzureNS.AzureOperationResponse TriggerRestore()
 
             RestoreRequestResource triggerRestoreRequest = new RestoreRequestResource();
             triggerRestoreRequest.Properties = restoreRequest;
-
-            var response = ServiceClientAdapter.RestoreDisk(
-                rp,
-                storageAccountResource.Location,
-                triggerRestoreRequest,
-                vaultName: vaultName,
-                resourceGroupName: resourceGroupName,
-                vaultLocation: vaultLocation ?? ServiceClientAdapter.BmsAdapter.GetResourceLocation());
-            return response;
+
+            if (useSecondaryRegion)
+            {
+                // get access token
+                CrrAccessToken accessToken = ServiceClientAdapter.GetCRRAccessToken(rp, secondaryRegion, vaultName: vaultName, resourceGroupName: resourceGroupName);
+
+                // Iaas VM CRR Request
+                Logger.Instance.WriteDebug("Triggering Restore to secondary region: " + secondaryRegion);
+                restoreRequest.Region = secondaryRegion;
+                restoreRequest.AffinityGroup = "";                 
+
+                CrossRegionRestoreRequest crrRestoreRequest = new CrossRegionRestoreRequest();
+                crrRestoreRequest.CrossRegionRestoreAccessDetails = accessToken;
+                crrRestoreRequest.RestoreRequest = restoreRequest;
+
+                var response = ServiceClientAdapter.RestoreDiskSecondryRegion(
+                    rp,
+                    crrRestoreRequest,
+                    storageAccountResource.Location,
+                    secondaryRegion: secondaryRegion);
+
+                return response;
+            }
+            else
+            {
+                var response = ServiceClientAdapter.RestoreDisk(
+                    rp,
+                    storageAccountResource.Location,
+                    triggerRestoreRequest,
+                    vaultName: vaultName,
+                    resourceGroupName: resourceGroupName,
+                    vaultLocation: vaultLocation ?? ServiceClientAdapter.BmsAdapter.GetResourceLocation());
+                return response;
+            }    
         }
 
         public ProtectedItemResource GetProtectedItem()
@@ -827,7 +853,7 @@ public List<ItemBase> ListProtectedItems()
                 (CmdletModel.WorkloadType)ProviderData[ItemParams.WorkloadType];
             ItemDeleteState deleteState =
                (ItemDeleteState)ProviderData[ItemParams.DeleteState];
-
+            bool UseSecondaryRegion = (bool)ProviderData[CRRParams.UseSecondaryRegion];            
             PolicyBase policy = (PolicyBase)ProviderData[PolicyParams.ProtectionPolicy];
 
             // 1. Filter by container
@@ -837,7 +863,8 @@ public List<ItemBase> ListProtectedItems()
                 container,
                 policy,
                 ServiceClientModel.BackupManagementType.AzureIaasVM,
-                DataSourceType.VM);
+                DataSourceType.VM,
+                UseSecondaryRegion);
 
             // 2. Filter by item name
             List<ItemBase> itemModels = AzureWorkloadProviderHelper.ListProtectedItemsByItemName(

src/RecoveryServices/RecoveryServices.Backup.ServiceClientAdapter/BMSAPIs/ItemAPIs.cs

@@ -122,6 +122,35 @@ public List<ProtectedItemResource> ListProtectedItem(
             return HelperUtils.GetPagedList(listAsync, listNextAsync);
         }
 
+        /// <summary>
+        /// List protected items protected from secondary region by the Recovery Services vault according to the query params 
+        /// and pagination params.
+        /// </summary>
+        /// <param name="queryFilter">Query params</param>
+        /// <param name="skipToken">Skip token used for pagination</param>
+        /// <returns>List of protected items</returns>
+        public List<ProtectedItemResource> ListCrrProtectedItem(
+            ODataQuery<ProtectedItemQueryObject> queryFilter,
+            string skipToken = default(string),
+            string vaultName = null,
+            string resourceGroupName = null)
+        {
+            Func<RestAzureNS.IPage<ProtectedItemResource>> listAsync =
+                () => BmsAdapter.Client.BackupProtectedItemsCrr.ListWithHttpMessagesAsync(
+                    vaultName ?? BmsAdapter.GetResourceName(),
+                    resourceGroupName ?? BmsAdapter.GetResourceGroupName(),
+                    queryFilter,
+                    skipToken,
+                    cancellationToken: BmsAdapter.CmdletCancellationToken).Result.Body;
+
+            Func<string, RestAzureNS.IPage<ProtectedItemResource>> listNextAsync =
+                nextLink => BmsAdapter.Client.BackupProtectedItemsCrr.ListNextWithHttpMessagesAsync(
+                    nextLink,
+                    cancellationToken: BmsAdapter.CmdletCancellationToken).Result.Body;
+
+            return HelperUtils.GetPagedList(listAsync, listNextAsync);
+        }       
+
         /// <summary>
         /// Triggers backup on the specified item
         /// </summary>

src/RecoveryServices/RecoveryServices.Backup.ServiceClientAdapter/BMSAPIs/JobAPIs.cs

@@ -40,6 +40,51 @@ public JobResource GetJob(
                 cancellationToken: BmsAdapter.CmdletCancellationToken).Result.Body;
         }
 
+        /// <summary>
+        /// Gets CRR job details
+        /// </summary>
+        /// <param name="secondaryRegion">secondaryRegion for the vault </param>
+        /// <param name="jobRequest">JobId, ResourceId for the Job to be fetched </param>
+        /// <returns>Job response returned by the service</returns>
+        public JobResource GetCRRJobDetails(
+            string secondaryRegion,
+            CrrJobRequest jobRequest
+            )
+        {
+            return BmsAdapter.Client.BackupCrrJobDetails.GetWithHttpMessagesAsync(secondaryRegion, jobRequest).Result.Body;
+        }
+
+        public List<JobResource> GetCrrJobs(string vaultId,
+            string jobId,
+            string status,
+            string operation,
+            DateTime startTime,
+            DateTime endTime,
+            string backupManagementType, 
+            string azureRegion = null)
+        {
+            ODataQuery<JobQueryObject> queryFilter = GetQueryObject(
+                backupManagementType,
+                startTime,
+                endTime,
+                jobId,
+                status, 
+                operation);
+
+            CrrJobRequest crrJobRequest = new CrrJobRequest();
+            crrJobRequest.ResourceId = vaultId;
+
+            Func<RestAzureNS.IPage<JobResource>> listAsync =
+                () => BmsAdapter.Client.BackupCrrJobs.ListWithHttpMessagesAsync(azureRegion, crrJobRequest, queryFilter,  cancellationToken: BmsAdapter.CmdletCancellationToken).Result.Body;   //  , crrJobRequest , queryFilter
+
+            Func<string, RestAzureNS.IPage<JobResource>> listNextAsync =
+                nextLink => BmsAdapter.Client.BackupJobs.ListNextWithHttpMessagesAsync(
+                    nextLink,
+                    cancellationToken: BmsAdapter.CmdletCancellationToken).Result.Body;
+
+            return HelperUtils.GetPagedList(listAsync, listNextAsync);
+        }
+
         /// <summary>
         /// Lists jobs according to the parameters
         /// </summary>

src/RecoveryServices/RecoveryServices.Backup.ServiceClientAdapter/BMSAPIs/OperationStatusAPIs.cs

@@ -38,6 +38,21 @@ public RestAzureNS.AzureOperationResponse<ServiceClientModel.OperationStatus>
                 operationId).Result;
         }
 
+        /// <summary>
+        /// Gets status of a generic operation on the protected item using the operation ID
+        /// </summary>
+        /// <param name="operationId">ID of the operation in progress</param>
+        /// <returns>Operation status response returned by the service</returns>
+        public RestAzureNS.AzureOperationResponse<ServiceClientModel.OperationStatus>
+            GetCrrOperationStatus(
+                string secondaryRegion,
+                string operationId)
+        {
+            return BmsAdapter.Client.CrrOperationStatus.GetWithHttpMessagesAsync(
+                secondaryRegion,
+                operationId).Result;
+        }
+
         /// <summary>
         /// Gets result of a generic operation on the protection policy using the operation ID
         /// </summary>
@@ -136,6 +151,7 @@ public ServiceClientModel.PrepareDataMoveResponse
 
             var prepareResponseSerialized = JsonConvert.SerializeObject(prepareResponseBase);
             PrepareDataMoveResponse prepareResponseDerived = JsonConvert.DeserializeObject<PrepareDataMoveResponse>(prepareResponseSerialized);
+
             return prepareResponseDerived;
         }