Add option for Azure cross subscription backups (#1895)

* add option for different subscription id

Signed-off-by: Moritz Schmitz von Hülst <[email protected]>

changelogs/unreleased/1895-boxcee

@@ -0,0 +1 @@
+Azure: add support for cross-subscription backups

pkg/cloudprovider/azure/object_store.go

@@ -35,6 +35,7 @@ import (
 
 const (
 	storageAccountConfigKey = "storageAccount"
+	subscriptionIdConfigKey = "subscriptionId"
 )
 
 type containerGetter interface {
@@ -146,22 +147,28 @@ func getStorageAccountKey(config map[string]string) (string, error) {
 		return "", errors.Wrap(err, "unable to get all required environment variables")
 	}
 
-	// 2. we need config["resourceGroup"], config["storageAccount"]
+	// 2. check whether a different subscription ID was set for backups in config["subscriptionId"]
+	subscriptionId := envVars[subscriptionIDEnvVar]
+	if val := config[subscriptionIdConfigKey]; val != "" {
+		subscriptionId = val
+	}
+
+	// 3. we need config["resourceGroup"], config["storageAccount"]
 	if _, err := getRequiredValues(mapLookup(config), resourceGroupConfigKey, storageAccountConfigKey); err != nil {
 		return "", errors.Wrap(err, "unable to get all required config values")
 	}
 
-	// 3. get SPT
+	// 4. get SPT
 	spt, err := newServicePrincipalToken(envVars[tenantIDEnvVar], envVars[clientIDEnvVar], envVars[clientSecretEnvVar], azure.PublicCloud.ResourceManagerEndpoint)
 	if err != nil {
 		return "", errors.Wrap(err, "error getting service principal token")
 	}
 
-	// 4. get storageAccountsClient
-	storageAccountsClient := storagemgmt.NewAccountsClient(envVars[subscriptionIDEnvVar])
+	// 5. get storageAccountsClient
+	storageAccountsClient := storagemgmt.NewAccountsClient(subscriptionId)
 	storageAccountsClient.Authorizer = autorest.NewBearerAuthorizer(spt)
 
-	// 5. get storage key
+	// 6. get storage key
 	res, err := storageAccountsClient.ListKeys(context.TODO(), config[resourceGroupConfigKey], config[storageAccountConfigKey])
 	if err != nil {
 		return "", errors.WithStack(err)
@@ -194,7 +201,11 @@ func mapLookup(data map[string]string) func(string) string {
 }
 
 func (o *ObjectStore) Init(config map[string]string) error {
-	if err := cloudprovider.ValidateObjectStoreConfigKeys(config, resourceGroupConfigKey, storageAccountConfigKey); err != nil {
+	if err := cloudprovider.ValidateObjectStoreConfigKeys(config,
+		resourceGroupConfigKey,
+		storageAccountConfigKey,
+		subscriptionIdConfigKey,
+	); err != nil {
 		return err
 	}
 

pkg/cloudprovider/azure/volume_snapshotter.go

@@ -51,7 +51,8 @@ type VolumeSnapshotter struct {
 	log                logrus.FieldLogger
 	disks              *disk.DisksClient
 	snaps              *disk.SnapshotsClient
-	subscription       string
+	disksSubscription  string
+	snapsSubscription  string
 	disksResourceGroup string
 	snapsResourceGroup string
 	apiTimeout         time.Duration
@@ -72,7 +73,7 @@ func NewVolumeSnapshotter(logger logrus.FieldLogger) *VolumeSnapshotter {
 }
 
 func (b *VolumeSnapshotter) Init(config map[string]string) error {
-	if err := cloudprovider.ValidateVolumeSnapshotterConfigKeys(config, resourceGroupConfigKey, apiTimeoutConfigKey); err != nil {
+	if err := cloudprovider.ValidateVolumeSnapshotterConfigKeys(config, resourceGroupConfigKey, apiTimeoutConfigKey, subscriptionIdConfigKey); err != nil {
 		return err
 	}
 
@@ -87,7 +88,17 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error {
 		return errors.Wrap(err, "unable to get all required environment variables")
 	}
 
-	// 2. if config["apiTimeout"] is empty, default to 2m; otherwise, parse it
+	// 2. set a different subscriptionId for snapshots if specified
+	snapshotsSubscriptionId := envVars[subscriptionIDEnvVar]
+	if val := config[subscriptionIdConfigKey]; val != "" {
+		// if subscription was set in config, it is required to also set the resource group
+		if _, err := getRequiredValues(mapLookup(config), resourceGroupConfigKey); err != nil {
+			return errors.Wrap(err, "resourceGroup not specified, but is a requirement when backing up to a different subscription")
+		}
+		snapshotsSubscriptionId = val
+	}
+
+	// 3. if config["apiTimeout"] is empty, default to 2m; otherwise, parse it
 	var apiTimeout time.Duration
 	if val := config[apiTimeoutConfigKey]; val == "" {
 		apiTimeout = 2 * time.Minute
@@ -98,15 +109,15 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error {
 		}
 	}
 
-	// 3. get SPT
+	// 4. get SPT
 	spt, err := newServicePrincipalToken(envVars[tenantIDEnvVar], envVars[clientIDEnvVar], envVars[clientSecretEnvVar], azure.PublicCloud.ResourceManagerEndpoint)
 	if err != nil {
 		return errors.Wrap(err, "error getting service principal token")
 	}
 
-	// 4. set up clients
+	// 5. set up clients
 	disksClient := disk.NewDisksClient(envVars[subscriptionIDEnvVar])
-	snapsClient := disk.NewSnapshotsClient(envVars[subscriptionIDEnvVar])
+	snapsClient := disk.NewSnapshotsClient(snapshotsSubscriptionId)
 
 	disksClient.PollingDelay = 5 * time.Second
 	snapsClient.PollingDelay = 5 * time.Second
@@ -117,7 +128,8 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error {
 
 	b.disks = &disksClient
 	b.snaps = &snapsClient
-	b.subscription = envVars[subscriptionIDEnvVar]
+	b.disksSubscription = envVars[subscriptionIDEnvVar]
+	b.snapsSubscription = snapshotsSubscriptionId
 	b.disksResourceGroup = envVars[resourceGroupEnvVar]
 	b.snapsResourceGroup = config[resourceGroupConfigKey]
 
@@ -205,7 +217,7 @@ func (b *VolumeSnapshotter) CreateSnapshot(volumeID, volumeAZ string, tags map[s
 		return "", errors.WithStack(err)
 	}
 
-	fullDiskName := getComputeResourceName(b.subscription, b.disksResourceGroup, disksResource, volumeID)
+	fullDiskName := getComputeResourceName(b.disksSubscription, b.disksResourceGroup, disksResource, volumeID)
 	// snapshot names must be <= 80 characters long
 	var snapshotName string
 	suffix := "-" + uuid.NewV4().String()
@@ -242,7 +254,7 @@ func (b *VolumeSnapshotter) CreateSnapshot(volumeID, volumeAZ string, tags map[s
 		return "", errors.WithStack(err)
 	}
 
-	return getComputeResourceName(b.subscription, b.snapsResourceGroup, snapshotsResource, snapshotName), nil
+	return getComputeResourceName(b.snapsSubscription, b.snapsResourceGroup, snapshotsResource, snapshotName), nil
 }
 
 func getSnapshotTags(veleroTags map[string]string, diskTags map[string]*string) map[string]*string {
@@ -374,7 +386,7 @@ func (b *VolumeSnapshotter) SetVolumeID(unstructuredPV runtime.Unstructured, vol
 	}
 
 	pv.Spec.AzureDisk.DiskName = volumeID
-	pv.Spec.AzureDisk.DataDiskURI = getComputeResourceName(b.subscription, b.disksResourceGroup, disksResource, volumeID)
+	pv.Spec.AzureDisk.DataDiskURI = getComputeResourceName(b.disksSubscription, b.disksResourceGroup, disksResource, volumeID)
 
 	res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pv)
 	if err != nil {

pkg/cloudprovider/azure/volume_snapshotter_test.go

@@ -57,7 +57,7 @@ func TestGetVolumeID(t *testing.T) {
 func TestSetVolumeID(t *testing.T) {
 	b := &VolumeSnapshotter{
 		disksResourceGroup: "rg",
-		subscription:       "sub",
+		disksSubscription:  "sub",
 	}
 
 	pv := &unstructured.Unstructured{

site/docs/master/api-types/backupstoragelocation.md

@@ -64,6 +64,7 @@ The configurable parameters are as follows:
 | --- | --- | --- | --- |
 | `resourceGroup` | string | Required Field | Name of the resource group containing the storage account for this backup storage location. |
 | `storageAccount` | string | Required Field | Name of the storage account for this backup storage location. |
+| `subscriptionId` | string | Optional Field | ID of the subscription for this backup storage location. |
 
 #### GCP
 

site/docs/master/api-types/volumesnapshotlocation.md

@@ -51,6 +51,7 @@ The configurable parameters are as follows:
 | --- | --- | --- | --- |
 | `apiTimeout` | metav1.Duration | 2m0s | How long to wait for an Azure API request to complete before timeout. |
 | `resourceGroup` | string | Optional | The name of the resource group where volume snapshots should be stored, if different from the cluster's resource group. |
+| `subscriptionId` | string | Optional | The ID of the subscription where volume snapshots should be stored, if different from the cluster's subscription. Requires `resourceGroup`to be set.
 
 #### GCP
 

site/docs/master/azure-config.md

@@ -38,6 +38,29 @@ of the Velero repository is under active development and is not guaranteed to be
 
 1. Move the `velero` binary from the Velero directory to somewhere in your PATH.
 
+## (Optional) Change to the Azure subscription you want to create your backups in
+
+By default, Velero will store backups in the same Subscription as your VMs and disks and will
+not allow you to restore backups to a Resource Group in a different Subscription. To enable backups/restore
+across Subscriptions you will need to specify the Subscription ID to backup to.
+
+Use `az` to switch to the Subscription the backups should be created in.
+
+First, find the Subscription ID by name.
+
+```bash
+AZURE_BACKUP_SUBSCRIPTION_NAME=<NAME_OF_TARGET_SUBSCRIPTION>
+AZURE_BACKUP_SUBSCRIPTION_ID=$(az account list --query="[?name=='$AZURE_BACKUP_SUBSCRIPTION_NAME'].id | [0]" -o tsv)
+```
+
+Second, change the Subscription.
+
+```bash
+az account set -s $AZURE_BACKUP_SUBSCRIPTION_ID
+```
+
+Execute the next step – creating an storage account and blob container – using the active Subscription.
+
 ## Create Azure storage account and blob container
 
 Velero requires a storage account and blob container in which to store backups.
@@ -82,6 +105,9 @@ az storage container create -n $BLOB_CONTAINER --public-access off --account-nam
 
 ## Get resource group for persistent volume snapshots
 
+_(Optional) If you decided to backup to a different Subscription, make sure you change back to the Subscription
+of your cluster's resources before continuing._
+
 1. Set the name of the Resource Group that contains your Kubernetes cluster's virtual machines/disks.
 
     **WARNING**: If you're using [AKS][22], `AZURE_RESOURCE_GROUP` must be set to the name of the auto-generated resource group that is created
@@ -115,9 +141,13 @@ To integrate Velero with Azure, you must create a Velero-specific [service princ
     If you'll be using Velero to backup multiple clusters with multiple blob containers, it may be desirable to create a unique username per cluster rather than the default `velero`.
 
     Create service principal and let the CLI generate a password for you. Make sure to capture the password.
+    
+    _(Optional) If you are using a different Subscription for backups and cluster resources, make sure to specify both subscriptions
+    in the `az` command using `--scopes`._
 
     ```bash
-    AZURE_CLIENT_SECRET=`az ad sp create-for-rbac --name "velero" --role "Contributor" --query 'password' -o tsv`
+    AZURE_CLIENT_SECRET=`az ad sp create-for-rbac --name "velero" --role "Contributor" --query 'password' -o tsv \
+      [--scopes /subscriptions/$AZURE_BACKUP_SUBSCRIPTION_ID /subscriptions/$AZURE_SUBSCRIPTION_ID]`
     ```
 
     After creating the service principal, obtain the client id.
@@ -147,8 +177,8 @@ velero install \
     --provider azure \
     --bucket $BLOB_CONTAINER \
     --secret-file ./credentials-velero \
-    --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID \
-    --snapshot-location-config apiTimeout=<YOUR_TIMEOUT>
+    --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] \
+    --snapshot-location-config apiTimeout=<YOUR_TIMEOUT>[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID]
 ```
 
 Additionally, you can specify `--use-restic` to enable restic support, and `--wait` to wait for the deployment to be ready.

site/docs/master/locations.md

@@ -154,7 +154,7 @@ velero backup create full-cluster-backup
 
 1. If you're using Azure's AKS, you may want to store your volume snapshots outside of the "infrastructure" resource group that is automatically created when you create your AKS cluster. This is possible using a `VolumeSnapshotLocation`, by specifying a `resourceGroup` under the `config` section of the snapshot location. See the [Azure volume snapshot location documentation][3] for details.
 
-1. If you're using Azure, you may want to store your Velero backups across multiple storage accounts and/or resource groups. This is possible using a `BackupStorageLocation`, by specifying a `storageAccount` and/or `resourceGroup`, respectively, under the `config` section of the backup location. See the [Azure backup storage location documentation][4] for details.
+1. If you're using Azure, you may want to store your Velero backups across multiple storage accounts and/or resource groups/subscriptions. This is possible using a `BackupStorageLocation`, by specifying a `storageAccount`, `resourceGroup` and/or `subscriptionId`, respectively, under the `config` section of the backup location. See the [Azure backup storage location documentation][4] for details.