From 98e01c4ca9d35b34344cf63d0b6c30232e4cb983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Schmitz=20von=20H=C3=BClst?= Date: Fri, 20 Sep 2019 17:04:55 +0200 Subject: [PATCH 1/5] add option for different subscription id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Moritz Schmitz von Hülst --- pkg/cloudprovider/azure/object_store.go | 20 +++++++++++++++----- site/docs/master/azure-config.md | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/pkg/cloudprovider/azure/object_store.go b/pkg/cloudprovider/azure/object_store.go index 4d341eb150..889e408632 100644 --- a/pkg/cloudprovider/azure/object_store.go +++ b/pkg/cloudprovider/azure/object_store.go @@ -35,6 +35,7 @@ import ( const ( storageAccountConfigKey = "storageAccount" + subscriptionIdConfigKey = "subscriptionId" ) type containerGetter interface { @@ -146,22 +147,31 @@ func getStorageAccountKey(config map[string]string) (string, error) { return "", errors.Wrap(err, "unable to get all required environment variables") } - // 2. we need config["resourceGroup"], config["storageAccount"] + // 2. check whether a different subscription ID was set for backups in config["subscriptionId"] + var subscriptionId string + if _, err := getRequiredValues(mapLookup(config), subscriptionIdConfigKey); err != nil { + subscriptionId = envVars[subscriptionIDEnvVar] + } + if subscriptionId == "" { + subscriptionId = config[subscriptionIdConfigKey] + } + + // 3. we need config["resourceGroup"], config["storageAccount"] if _, err := getRequiredValues(mapLookup(config), resourceGroupConfigKey, storageAccountConfigKey); err != nil { return "", errors.Wrap(err, "unable to get all required config values") } - // 3. get SPT + // 4. get SPT spt, err := newServicePrincipalToken(envVars[tenantIDEnvVar], envVars[clientIDEnvVar], envVars[clientSecretEnvVar], azure.PublicCloud.ResourceManagerEndpoint) if err != nil { return "", errors.Wrap(err, "error getting service principal token") } - // 4. get storageAccountsClient - storageAccountsClient := storagemgmt.NewAccountsClient(envVars[subscriptionIDEnvVar]) + // 5. get storageAccountsClient + storageAccountsClient := storagemgmt.NewAccountsClient(subscriptionId) storageAccountsClient.Authorizer = autorest.NewBearerAuthorizer(spt) - // 5. get storage key + // 6. get storage key res, err := storageAccountsClient.ListKeys(context.TODO(), config[resourceGroupConfigKey], config[storageAccountConfigKey]) if err != nil { return "", errors.WithStack(err) diff --git a/site/docs/master/azure-config.md b/site/docs/master/azure-config.md index 0a8faf37a1..08188db2be 100644 --- a/site/docs/master/azure-config.md +++ b/site/docs/master/azure-config.md @@ -147,7 +147,7 @@ velero install \ --provider azure \ --bucket $BLOB_CONTAINER \ --secret-file ./credentials-velero \ - --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID \ + --backup-location-config subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID \ --snapshot-location-config apiTimeout= ``` From 17db66cc5dad3ccb1923e35328e4b9f2e016369e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Schmitz=20von=20H=C3=BClst?= Date: Mon, 23 Sep 2019 13:55:33 +0200 Subject: [PATCH 2/5] Extend azure-config.md by adding additional information regarding backing up to a different Azure Subscription than the cluster's resources. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Moritz Schmitz von Hülst --- site/docs/master/azure-config.md | 34 ++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/site/docs/master/azure-config.md b/site/docs/master/azure-config.md index 08188db2be..630538b8ad 100644 --- a/site/docs/master/azure-config.md +++ b/site/docs/master/azure-config.md @@ -38,6 +38,29 @@ of the Velero repository is under active development and is not guaranteed to be 1. Move the `velero` binary from the Velero directory to somewhere in your PATH. +## (Optional) Change to the Azure subscription you want to create your backups in + +By default, Velero will store backups in the same Subscription as your VMs and disks and will +not allow you to restore backups to a Resource Group in a different Subscription. To enable backups/restore +across Subscriptions you will need to specify the Subscription ID to backup to. + +Use `az` to switch to the Subscription the backups should be created in. + +First, find the Subscription ID by name. + +```bash +AZURE_BACKUP_SUBSCRIPTION_NAME= +AZURE_BACKUP_SUBSCRIPTION_ID=$(az account list --query="[?name=='$AZURE_BACKUP_SUBSCRIPTION_NAME'].id | [0]" -o tsv) +``` + +Second, change the Subscription. + +```bash +az account set -s $AZURE_BACKUP_SUBSCRIPTION_ID +``` + +Execute the next step – creating an storage account and blob container – using the active Subscription. + ## Create Azure storage account and blob container Velero requires a storage account and blob container in which to store backups. @@ -82,6 +105,9 @@ az storage container create -n $BLOB_CONTAINER --public-access off --account-nam ## Get resource group for persistent volume snapshots +_(Optional) If you decided to backup to a different Subscription, make sure you change back to the Subscription +of your cluster's resources before continuing._ + 1. Set the name of the Resource Group that contains your Kubernetes cluster's virtual machines/disks. **WARNING**: If you're using [AKS][22], `AZURE_RESOURCE_GROUP` must be set to the name of the auto-generated resource group that is created @@ -115,9 +141,13 @@ To integrate Velero with Azure, you must create a Velero-specific [service princ If you'll be using Velero to backup multiple clusters with multiple blob containers, it may be desirable to create a unique username per cluster rather than the default `velero`. Create service principal and let the CLI generate a password for you. Make sure to capture the password. + + _(Optional) If you are using a different Subscription for backups and cluster resources, make sure to specify both subscriptions + in the `az` command using `--scopes`._ ```bash - AZURE_CLIENT_SECRET=`az ad sp create-for-rbac --name "velero" --role "Contributor" --query 'password' -o tsv` + AZURE_CLIENT_SECRET=`az ad sp create-for-rbac --name "velero" --role "Contributor" --query 'password' -o tsv \ + [--scopes /subscriptions/$AZURE_BACKUP_SUBSCRIPTION_ID /subscriptions/$AZURE_SUBSCRIPTION_ID]` ``` After creating the service principal, obtain the client id. @@ -147,7 +177,7 @@ velero install \ --provider azure \ --bucket $BLOB_CONTAINER \ --secret-file ./credentials-velero \ - --backup-location-config subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID \ + --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] \ --snapshot-location-config apiTimeout= ``` From c159ffdaee1a7243b9052067ca69fbca8b5413c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Schmitz=20von=20H=C3=BClst?= Date: Mon, 23 Sep 2019 13:55:33 +0200 Subject: [PATCH 3/5] refactor regarding to comment, add more documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Moritz Schmitz von Hülst --- site/docs/master/azure-config.md | 34 ++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/site/docs/master/azure-config.md b/site/docs/master/azure-config.md index 08188db2be..630538b8ad 100644 --- a/site/docs/master/azure-config.md +++ b/site/docs/master/azure-config.md @@ -38,6 +38,29 @@ of the Velero repository is under active development and is not guaranteed to be 1. Move the `velero` binary from the Velero directory to somewhere in your PATH. +## (Optional) Change to the Azure subscription you want to create your backups in + +By default, Velero will store backups in the same Subscription as your VMs and disks and will +not allow you to restore backups to a Resource Group in a different Subscription. To enable backups/restore +across Subscriptions you will need to specify the Subscription ID to backup to. + +Use `az` to switch to the Subscription the backups should be created in. + +First, find the Subscription ID by name. + +```bash +AZURE_BACKUP_SUBSCRIPTION_NAME= +AZURE_BACKUP_SUBSCRIPTION_ID=$(az account list --query="[?name=='$AZURE_BACKUP_SUBSCRIPTION_NAME'].id | [0]" -o tsv) +``` + +Second, change the Subscription. + +```bash +az account set -s $AZURE_BACKUP_SUBSCRIPTION_ID +``` + +Execute the next step – creating an storage account and blob container – using the active Subscription. + ## Create Azure storage account and blob container Velero requires a storage account and blob container in which to store backups. @@ -82,6 +105,9 @@ az storage container create -n $BLOB_CONTAINER --public-access off --account-nam ## Get resource group for persistent volume snapshots +_(Optional) If you decided to backup to a different Subscription, make sure you change back to the Subscription +of your cluster's resources before continuing._ + 1. Set the name of the Resource Group that contains your Kubernetes cluster's virtual machines/disks. **WARNING**: If you're using [AKS][22], `AZURE_RESOURCE_GROUP` must be set to the name of the auto-generated resource group that is created @@ -115,9 +141,13 @@ To integrate Velero with Azure, you must create a Velero-specific [service princ If you'll be using Velero to backup multiple clusters with multiple blob containers, it may be desirable to create a unique username per cluster rather than the default `velero`. Create service principal and let the CLI generate a password for you. Make sure to capture the password. + + _(Optional) If you are using a different Subscription for backups and cluster resources, make sure to specify both subscriptions + in the `az` command using `--scopes`._ ```bash - AZURE_CLIENT_SECRET=`az ad sp create-for-rbac --name "velero" --role "Contributor" --query 'password' -o tsv` + AZURE_CLIENT_SECRET=`az ad sp create-for-rbac --name "velero" --role "Contributor" --query 'password' -o tsv \ + [--scopes /subscriptions/$AZURE_BACKUP_SUBSCRIPTION_ID /subscriptions/$AZURE_SUBSCRIPTION_ID]` ``` After creating the service principal, obtain the client id. @@ -147,7 +177,7 @@ velero install \ --provider azure \ --bucket $BLOB_CONTAINER \ --secret-file ./credentials-velero \ - --backup-location-config subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID \ + --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] \ --snapshot-location-config apiTimeout= ``` From 96c1f580b43929f10426fdf973548190cf3db77c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Schmitz=20von=20H=C3=BClst?= Date: Fri, 27 Sep 2019 15:44:33 +0200 Subject: [PATCH 4/5] add option to snapshot to different subscription MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Moritz Schmitz von Hülst --- changelogs/unreleased/1895-boxcee | 1 + pkg/cloudprovider/azure/object_store.go | 6 +++- pkg/cloudprovider/azure/volume_snapshotter.go | 32 +++++++++++++------ .../api-types/volumesnapshotlocation.md | 1 + site/docs/master/azure-config.md | 2 +- 5 files changed, 30 insertions(+), 12 deletions(-) create mode 100644 changelogs/unreleased/1895-boxcee diff --git a/changelogs/unreleased/1895-boxcee b/changelogs/unreleased/1895-boxcee new file mode 100644 index 0000000000..cc25a7e38a --- /dev/null +++ b/changelogs/unreleased/1895-boxcee @@ -0,0 +1 @@ +Azure: add support for cross-subscription backups diff --git a/pkg/cloudprovider/azure/object_store.go b/pkg/cloudprovider/azure/object_store.go index 8d989a39b2..38c1bc513c 100644 --- a/pkg/cloudprovider/azure/object_store.go +++ b/pkg/cloudprovider/azure/object_store.go @@ -201,7 +201,11 @@ func mapLookup(data map[string]string) func(string) string { } func (o *ObjectStore) Init(config map[string]string) error { - if err := cloudprovider.ValidateObjectStoreConfigKeys(config, resourceGroupConfigKey, storageAccountConfigKey); err != nil { + if err := cloudprovider.ValidateObjectStoreConfigKeys(config, + resourceGroupConfigKey, + storageAccountConfigKey, + subscriptionIdConfigKey, + ); err != nil { return err } diff --git a/pkg/cloudprovider/azure/volume_snapshotter.go b/pkg/cloudprovider/azure/volume_snapshotter.go index 91e20dad62..56282ce47e 100644 --- a/pkg/cloudprovider/azure/volume_snapshotter.go +++ b/pkg/cloudprovider/azure/volume_snapshotter.go @@ -51,7 +51,8 @@ type VolumeSnapshotter struct { log logrus.FieldLogger disks *disk.DisksClient snaps *disk.SnapshotsClient - subscription string + disksSubscription string + snapsSubscription string disksResourceGroup string snapsResourceGroup string apiTimeout time.Duration @@ -72,7 +73,7 @@ func NewVolumeSnapshotter(logger logrus.FieldLogger) *VolumeSnapshotter { } func (b *VolumeSnapshotter) Init(config map[string]string) error { - if err := cloudprovider.ValidateVolumeSnapshotterConfigKeys(config, resourceGroupConfigKey, apiTimeoutConfigKey); err != nil { + if err := cloudprovider.ValidateVolumeSnapshotterConfigKeys(config, resourceGroupConfigKey, apiTimeoutConfigKey, subscriptionIdConfigKey); err != nil { return err } @@ -87,7 +88,17 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error { return errors.Wrap(err, "unable to get all required environment variables") } - // 2. if config["apiTimeout"] is empty, default to 2m; otherwise, parse it + // 2. set a different subscriptionId for snapshots if specified + snapshotsSubscriptionId := envVars[subscriptionIDEnvVar] + if val := config[subscriptionIdConfigKey]; val != "" { + // if subscription was set in config, it is required to also set the resource group + if _, err := getRequiredValues(mapLookup(config), resourceGroupConfigKey); err != nil { + return errors.Wrap(err, "resourceGroup not specified, but is a requirement when backing up to a different subscription") + } + snapshotsSubscriptionId = val + } + + // 3. if config["apiTimeout"] is empty, default to 2m; otherwise, parse it var apiTimeout time.Duration if val := config[apiTimeoutConfigKey]; val == "" { apiTimeout = 2 * time.Minute @@ -98,15 +109,15 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error { } } - // 3. get SPT + // 4. get SPT spt, err := newServicePrincipalToken(envVars[tenantIDEnvVar], envVars[clientIDEnvVar], envVars[clientSecretEnvVar], azure.PublicCloud.ResourceManagerEndpoint) if err != nil { return errors.Wrap(err, "error getting service principal token") } - // 4. set up clients + // 5. set up clients disksClient := disk.NewDisksClient(envVars[subscriptionIDEnvVar]) - snapsClient := disk.NewSnapshotsClient(envVars[subscriptionIDEnvVar]) + snapsClient := disk.NewSnapshotsClient(snapshotsSubscriptionId) disksClient.PollingDelay = 5 * time.Second snapsClient.PollingDelay = 5 * time.Second @@ -117,7 +128,8 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error { b.disks = &disksClient b.snaps = &snapsClient - b.subscription = envVars[subscriptionIDEnvVar] + b.disksSubscription = envVars[subscriptionIDEnvVar] + b.snapsSubscription = snapshotsSubscriptionId b.disksResourceGroup = envVars[resourceGroupEnvVar] b.snapsResourceGroup = config[resourceGroupConfigKey] @@ -205,7 +217,7 @@ func (b *VolumeSnapshotter) CreateSnapshot(volumeID, volumeAZ string, tags map[s return "", errors.WithStack(err) } - fullDiskName := getComputeResourceName(b.subscription, b.disksResourceGroup, disksResource, volumeID) + fullDiskName := getComputeResourceName(b.disksSubscription, b.disksResourceGroup, disksResource, volumeID) // snapshot names must be <= 80 characters long var snapshotName string suffix := "-" + uuid.NewV4().String() @@ -242,7 +254,7 @@ func (b *VolumeSnapshotter) CreateSnapshot(volumeID, volumeAZ string, tags map[s return "", errors.WithStack(err) } - return getComputeResourceName(b.subscription, b.snapsResourceGroup, snapshotsResource, snapshotName), nil + return getComputeResourceName(b.snapsSubscription, b.snapsResourceGroup, snapshotsResource, snapshotName), nil } func getSnapshotTags(veleroTags map[string]string, diskTags map[string]*string) map[string]*string { @@ -374,7 +386,7 @@ func (b *VolumeSnapshotter) SetVolumeID(unstructuredPV runtime.Unstructured, vol } pv.Spec.AzureDisk.DiskName = volumeID - pv.Spec.AzureDisk.DataDiskURI = getComputeResourceName(b.subscription, b.disksResourceGroup, disksResource, volumeID) + pv.Spec.AzureDisk.DataDiskURI = getComputeResourceName(b.disksSubscription, b.disksResourceGroup, disksResource, volumeID) res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pv) if err != nil { diff --git a/site/docs/master/api-types/volumesnapshotlocation.md b/site/docs/master/api-types/volumesnapshotlocation.md index 8e6ad15c2c..f6fcd0d4b9 100644 --- a/site/docs/master/api-types/volumesnapshotlocation.md +++ b/site/docs/master/api-types/volumesnapshotlocation.md @@ -51,6 +51,7 @@ The configurable parameters are as follows: | --- | --- | --- | --- | | `apiTimeout` | metav1.Duration | 2m0s | How long to wait for an Azure API request to complete before timeout. | | `resourceGroup` | string | Optional | The name of the resource group where volume snapshots should be stored, if different from the cluster's resource group. | +| `subscriptionId` | string | Optional | The ID of the subscription where volume snapshots should be stored, if different from the cluster's subscription. Requires `resourceGroup`to be set. #### GCP diff --git a/site/docs/master/azure-config.md b/site/docs/master/azure-config.md index 630538b8ad..5c9c0e651f 100644 --- a/site/docs/master/azure-config.md +++ b/site/docs/master/azure-config.md @@ -178,7 +178,7 @@ velero install \ --bucket $BLOB_CONTAINER \ --secret-file ./credentials-velero \ --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] \ - --snapshot-location-config apiTimeout= + --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] ``` Additionally, you can specify `--use-restic` to enable restic support, and `--wait` to wait for the deployment to be ready. From fcc8076b369e1f8b250ab7635ebbe43f70319944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Schmitz=20von=20H=C3=BClst?= Date: Mon, 30 Sep 2019 16:59:26 +0200 Subject: [PATCH 5/5] fix tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Moritz Schmitz von Hülst --- pkg/cloudprovider/azure/volume_snapshotter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudprovider/azure/volume_snapshotter_test.go b/pkg/cloudprovider/azure/volume_snapshotter_test.go index 1df1ba7d84..f010850406 100644 --- a/pkg/cloudprovider/azure/volume_snapshotter_test.go +++ b/pkg/cloudprovider/azure/volume_snapshotter_test.go @@ -57,7 +57,7 @@ func TestGetVolumeID(t *testing.T) { func TestSetVolumeID(t *testing.T) { b := &VolumeSnapshotter{ disksResourceGroup: "rg", - subscription: "sub", + disksSubscription: "sub", } pv := &unstructured.Unstructured{