Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add option for Azure cross subscription backups #1895

Merged
merged 8 commits into from
Oct 3, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelogs/unreleased/1895-boxcee
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Azure: add support for cross-subscription backups
23 changes: 17 additions & 6 deletions pkg/cloudprovider/azure/object_store.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (

const (
storageAccountConfigKey = "storageAccount"
subscriptionIdConfigKey = "subscriptionId"
)

type containerGetter interface {
Expand Down Expand Up @@ -146,22 +147,28 @@ func getStorageAccountKey(config map[string]string) (string, error) {
return "", errors.Wrap(err, "unable to get all required environment variables")
}

// 2. we need config["resourceGroup"], config["storageAccount"]
// 2. check whether a different subscription ID was set for backups in config["subscriptionId"]
subscriptionId := envVars[subscriptionIDEnvVar]
if val := config[subscriptionIdConfigKey]; val != "" {
subscriptionId = val
}

// 3. we need config["resourceGroup"], config["storageAccount"]
if _, err := getRequiredValues(mapLookup(config), resourceGroupConfigKey, storageAccountConfigKey); err != nil {
return "", errors.Wrap(err, "unable to get all required config values")
}

// 3. get SPT
// 4. get SPT
spt, err := newServicePrincipalToken(envVars[tenantIDEnvVar], envVars[clientIDEnvVar], envVars[clientSecretEnvVar], azure.PublicCloud.ResourceManagerEndpoint)
if err != nil {
return "", errors.Wrap(err, "error getting service principal token")
}

// 4. get storageAccountsClient
storageAccountsClient := storagemgmt.NewAccountsClient(envVars[subscriptionIDEnvVar])
// 5. get storageAccountsClient
storageAccountsClient := storagemgmt.NewAccountsClient(subscriptionId)
storageAccountsClient.Authorizer = autorest.NewBearerAuthorizer(spt)

// 5. get storage key
// 6. get storage key
res, err := storageAccountsClient.ListKeys(context.TODO(), config[resourceGroupConfigKey], config[storageAccountConfigKey])
if err != nil {
return "", errors.WithStack(err)
Expand Down Expand Up @@ -194,7 +201,11 @@ func mapLookup(data map[string]string) func(string) string {
}

func (o *ObjectStore) Init(config map[string]string) error {
if err := cloudprovider.ValidateObjectStoreConfigKeys(config, resourceGroupConfigKey, storageAccountConfigKey); err != nil {
if err := cloudprovider.ValidateObjectStoreConfigKeys(config,
resourceGroupConfigKey,
storageAccountConfigKey,
subscriptionIdConfigKey,
); err != nil {
return err
}

Expand Down
32 changes: 22 additions & 10 deletions pkg/cloudprovider/azure/volume_snapshotter.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@ type VolumeSnapshotter struct {
log logrus.FieldLogger
disks *disk.DisksClient
snaps *disk.SnapshotsClient
subscription string
disksSubscription string
snapsSubscription string
disksResourceGroup string
snapsResourceGroup string
apiTimeout time.Duration
Expand All @@ -72,7 +73,7 @@ func NewVolumeSnapshotter(logger logrus.FieldLogger) *VolumeSnapshotter {
}

func (b *VolumeSnapshotter) Init(config map[string]string) error {
if err := cloudprovider.ValidateVolumeSnapshotterConfigKeys(config, resourceGroupConfigKey, apiTimeoutConfigKey); err != nil {
if err := cloudprovider.ValidateVolumeSnapshotterConfigKeys(config, resourceGroupConfigKey, apiTimeoutConfigKey, subscriptionIdConfigKey); err != nil {
return err
}

Expand All @@ -87,7 +88,17 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error {
return errors.Wrap(err, "unable to get all required environment variables")
}

// 2. if config["apiTimeout"] is empty, default to 2m; otherwise, parse it
// 2. set a different subscriptionId for snapshots if specified
snapshotsSubscriptionId := envVars[subscriptionIDEnvVar]
if val := config[subscriptionIdConfigKey]; val != "" {
// if subscription was set in config, it is required to also set the resource group
if _, err := getRequiredValues(mapLookup(config), resourceGroupConfigKey); err != nil {
return errors.Wrap(err, "resourceGroup not specified, but is a requirement when backing up to a different subscription")
}
snapshotsSubscriptionId = val
}

// 3. if config["apiTimeout"] is empty, default to 2m; otherwise, parse it
var apiTimeout time.Duration
if val := config[apiTimeoutConfigKey]; val == "" {
apiTimeout = 2 * time.Minute
Expand All @@ -98,15 +109,15 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error {
}
}

// 3. get SPT
// 4. get SPT
spt, err := newServicePrincipalToken(envVars[tenantIDEnvVar], envVars[clientIDEnvVar], envVars[clientSecretEnvVar], azure.PublicCloud.ResourceManagerEndpoint)
if err != nil {
return errors.Wrap(err, "error getting service principal token")
}

// 4. set up clients
// 5. set up clients
disksClient := disk.NewDisksClient(envVars[subscriptionIDEnvVar])
snapsClient := disk.NewSnapshotsClient(envVars[subscriptionIDEnvVar])
snapsClient := disk.NewSnapshotsClient(snapshotsSubscriptionId)

disksClient.PollingDelay = 5 * time.Second
snapsClient.PollingDelay = 5 * time.Second
Expand All @@ -117,7 +128,8 @@ func (b *VolumeSnapshotter) Init(config map[string]string) error {

b.disks = &disksClient
b.snaps = &snapsClient
b.subscription = envVars[subscriptionIDEnvVar]
b.disksSubscription = envVars[subscriptionIDEnvVar]
b.snapsSubscription = snapshotsSubscriptionId
b.disksResourceGroup = envVars[resourceGroupEnvVar]
b.snapsResourceGroup = config[resourceGroupConfigKey]

Expand Down Expand Up @@ -205,7 +217,7 @@ func (b *VolumeSnapshotter) CreateSnapshot(volumeID, volumeAZ string, tags map[s
return "", errors.WithStack(err)
}

fullDiskName := getComputeResourceName(b.subscription, b.disksResourceGroup, disksResource, volumeID)
fullDiskName := getComputeResourceName(b.disksSubscription, b.disksResourceGroup, disksResource, volumeID)
// snapshot names must be <= 80 characters long
var snapshotName string
suffix := "-" + uuid.NewV4().String()
Expand Down Expand Up @@ -242,7 +254,7 @@ func (b *VolumeSnapshotter) CreateSnapshot(volumeID, volumeAZ string, tags map[s
return "", errors.WithStack(err)
}

return getComputeResourceName(b.subscription, b.snapsResourceGroup, snapshotsResource, snapshotName), nil
return getComputeResourceName(b.snapsSubscription, b.snapsResourceGroup, snapshotsResource, snapshotName), nil
}

func getSnapshotTags(veleroTags map[string]string, diskTags map[string]*string) map[string]*string {
Expand Down Expand Up @@ -374,7 +386,7 @@ func (b *VolumeSnapshotter) SetVolumeID(unstructuredPV runtime.Unstructured, vol
}

pv.Spec.AzureDisk.DiskName = volumeID
pv.Spec.AzureDisk.DataDiskURI = getComputeResourceName(b.subscription, b.disksResourceGroup, disksResource, volumeID)
pv.Spec.AzureDisk.DataDiskURI = getComputeResourceName(b.disksSubscription, b.disksResourceGroup, disksResource, volumeID)

res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pv)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion pkg/cloudprovider/azure/volume_snapshotter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func TestGetVolumeID(t *testing.T) {
func TestSetVolumeID(t *testing.T) {
b := &VolumeSnapshotter{
disksResourceGroup: "rg",
subscription: "sub",
disksSubscription: "sub",
}

pv := &unstructured.Unstructured{
Expand Down
1 change: 1 addition & 0 deletions site/docs/master/api-types/backupstoragelocation.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ The configurable parameters are as follows:
| --- | --- | --- | --- |
| `resourceGroup` | string | Required Field | Name of the resource group containing the storage account for this backup storage location. |
| `storageAccount` | string | Required Field | Name of the storage account for this backup storage location. |
| `subscriptionId` | string | Optional Field | ID of the subscription for this backup storage location. |

#### GCP

Expand Down
1 change: 1 addition & 0 deletions site/docs/master/api-types/volumesnapshotlocation.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ The configurable parameters are as follows:
| --- | --- | --- | --- |
| `apiTimeout` | metav1.Duration | 2m0s | How long to wait for an Azure API request to complete before timeout. |
| `resourceGroup` | string | Optional | The name of the resource group where volume snapshots should be stored, if different from the cluster's resource group. |
| `subscriptionId` | string | Optional | The ID of the subscription where volume snapshots should be stored, if different from the cluster's subscription. Requires `resourceGroup`to be set.

#### GCP

Expand Down
36 changes: 33 additions & 3 deletions site/docs/master/azure-config.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,29 @@ of the Velero repository is under active development and is not guaranteed to be

1. Move the `velero` binary from the Velero directory to somewhere in your PATH.

## (Optional) Change to the Azure subscription you want to create your backups in

By default, Velero will store backups in the same Subscription as your VMs and disks and will
not allow you to restore backups to a Resource Group in a different Subscription. To enable backups/restore
across Subscriptions you will need to specify the Subscription ID to backup to.

Use `az` to switch to the Subscription the backups should be created in.

First, find the Subscription ID by name.

```bash
AZURE_BACKUP_SUBSCRIPTION_NAME=<NAME_OF_TARGET_SUBSCRIPTION>
AZURE_BACKUP_SUBSCRIPTION_ID=$(az account list --query="[?name=='$AZURE_BACKUP_SUBSCRIPTION_NAME'].id | [0]" -o tsv)
```

Second, change the Subscription.

```bash
az account set -s $AZURE_BACKUP_SUBSCRIPTION_ID
```

Execute the next step – creating an storage account and blob container – using the active Subscription.

## Create Azure storage account and blob container

Velero requires a storage account and blob container in which to store backups.
Expand Down Expand Up @@ -82,6 +105,9 @@ az storage container create -n $BLOB_CONTAINER --public-access off --account-nam

## Get resource group for persistent volume snapshots

_(Optional) If you decided to backup to a different Subscription, make sure you change back to the Subscription
of your cluster's resources before continuing._

1. Set the name of the Resource Group that contains your Kubernetes cluster's virtual machines/disks.

**WARNING**: If you're using [AKS][22], `AZURE_RESOURCE_GROUP` must be set to the name of the auto-generated resource group that is created
Expand Down Expand Up @@ -115,9 +141,13 @@ To integrate Velero with Azure, you must create a Velero-specific [service princ
If you'll be using Velero to backup multiple clusters with multiple blob containers, it may be desirable to create a unique username per cluster rather than the default `velero`.

Create service principal and let the CLI generate a password for you. Make sure to capture the password.

_(Optional) If you are using a different Subscription for backups and cluster resources, make sure to specify both subscriptions
in the `az` command using `--scopes`._

```bash
AZURE_CLIENT_SECRET=`az ad sp create-for-rbac --name "velero" --role "Contributor" --query 'password' -o tsv`
AZURE_CLIENT_SECRET=`az ad sp create-for-rbac --name "velero" --role "Contributor" --query 'password' -o tsv \
[--scopes /subscriptions/$AZURE_BACKUP_SUBSCRIPTION_ID /subscriptions/$AZURE_SUBSCRIPTION_ID]`
```

After creating the service principal, obtain the client id.
Expand Down Expand Up @@ -147,8 +177,8 @@ velero install \
--provider azure \
--bucket $BLOB_CONTAINER \
--secret-file ./credentials-velero \
--backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID \
--snapshot-location-config apiTimeout=<YOUR_TIMEOUT>
--backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] \
--snapshot-location-config apiTimeout=<YOUR_TIMEOUT>[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID]
```

Additionally, you can specify `--use-restic` to enable restic support, and `--wait` to wait for the deployment to be ready.
Expand Down
2 changes: 1 addition & 1 deletion site/docs/master/locations.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ velero backup create full-cluster-backup

1. If you're using Azure's AKS, you may want to store your volume snapshots outside of the "infrastructure" resource group that is automatically created when you create your AKS cluster. This is possible using a `VolumeSnapshotLocation`, by specifying a `resourceGroup` under the `config` section of the snapshot location. See the [Azure volume snapshot location documentation][3] for details.

1. If you're using Azure, you may want to store your Velero backups across multiple storage accounts and/or resource groups. This is possible using a `BackupStorageLocation`, by specifying a `storageAccount` and/or `resourceGroup`, respectively, under the `config` section of the backup location. See the [Azure backup storage location documentation][4] for details.
1. If you're using Azure, you may want to store your Velero backups across multiple storage accounts and/or resource groups/subscriptions. This is possible using a `BackupStorageLocation`, by specifying a `storageAccount`, `resourceGroup` and/or `subscriptionId`, respectively, under the `config` section of the backup location. See the [Azure backup storage location documentation][4] for details.



Expand Down