From 044b8c38fec71930ced7df4e2b27430efbdc8ddd Mon Sep 17 00:00:00 2001 From: Somtochi Onyekwere Date: Thu, 26 May 2022 11:08:57 +0100 Subject: [PATCH] Add Support for SAS keys in Azure Blob Signed-off-by: Somtochi Onyekwere --- docs/spec/v1beta2/buckets.md | 34 ++++++++++++ go.mod | 1 + go.sum | 2 + pkg/azure/blob.go | 48 ++++++++++++++++ pkg/azure/blob_integration_test.go | 61 +++++++++++++++++++++ pkg/azure/blob_test.go | 88 ++++++++++++++++++++++++++++++ 6 files changed, 234 insertions(+) diff --git a/docs/spec/v1beta2/buckets.md b/docs/spec/v1beta2/buckets.md index ed421141f..26c107d39 100644 --- a/docs/spec/v1beta2/buckets.md +++ b/docs/spec/v1beta2/buckets.md @@ -295,6 +295,7 @@ sets of `.data` fields: - `clientId` for authenticating using a Managed Identity. - `accountKey` for authenticating using a [Shared Key](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/storage/azblob#SharedKeyCredential). +- `sasKey` for authenticating using a [SAS Token](https://docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview) For any Managed Identity and/or Azure Active Directory authentication method, the base URL can be configured using `.data.authorityHost`. If not supplied, @@ -504,6 +505,39 @@ spec: endpoint: https://testfluxsas.blob.core.windows.net ``` +##### Azure Blob SAS Token example + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: azure-sas-token + namespace: default +spec: + interval: 5m0s + provider: azure + bucketName: + endpoint: https://.blob.core.windows.net + secretRef: + name: azure-key +--- +apiVersion: v1 +kind: Secret +metadata: + name: azure-key + namespace: default +type: Opaque +data: + sasKey: +``` + +The query values from the `sasKey` data field in the Secrets gets merged with the `spec.endpoint` of the `Bucket`. +If the same key is present in the both of them, the token takes precedence. + +Note that the Azure SAS Token has an expiry date and it should be updated before it expires so that Flux can +continue to access Azure Storage. + #### GCP When a Bucket's `.spec.provider` is set to `gcp`, the source-controller will diff --git a/go.mod b/go.mod index eecf3366c..e90e6f75c 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/fluxcd/pkg/gitutil v0.1.0 github.com/fluxcd/pkg/helmtestserver v0.7.4 github.com/fluxcd/pkg/lockedfile v0.1.0 + github.com/fluxcd/pkg/masktoken v0.0.1 github.com/fluxcd/pkg/runtime v0.16.2 github.com/fluxcd/pkg/ssh v0.5.0 github.com/fluxcd/pkg/testserver v0.2.0 diff --git a/go.sum b/go.sum index 2c1ef8183..98b239916 100644 --- a/go.sum +++ b/go.sum @@ -281,6 +281,8 @@ github.com/fluxcd/pkg/helmtestserver v0.7.4 h1:/Xj2+XLz7wr38MI3uPYvVAsZB9wQOq6rp github.com/fluxcd/pkg/helmtestserver v0.7.4/go.mod h1:aL5V4o8wUOMqeHMfjbVHS057E3ejzHMRVMqEbsK9FUQ= github.com/fluxcd/pkg/lockedfile v0.1.0 h1:YsYFAkd6wawMCcD74ikadAKXA4s2sukdxrn7w8RB5eo= github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfHbdvIy9VUgIjm8= +github.com/fluxcd/pkg/masktoken v0.0.1 h1:egWR/ibTzf4L3PxE8TauKO1srD1Ye/aalgQRQuKKRdU= +github.com/fluxcd/pkg/masktoken v0.0.1/go.mod h1:sQmMtX4s5RwdGlByJazzNasWFFgBdmtNcgeZcGBI72Y= github.com/fluxcd/pkg/runtime v0.16.2 h1:CexfMmJK+r12sHTvKWyAax0pcPomjd6VnaHXcxjUrRY= github.com/fluxcd/pkg/runtime v0.16.2/go.mod h1:OHSKsrO+T+Ym8WZRS2oidrnauWRARuE2nfm8ewevm7M= github.com/fluxcd/pkg/ssh v0.5.0 h1:jE9F2XvUXC2mgseeXMATvO014fLqdB30/VzlPLKsk20= diff --git a/pkg/azure/blob.go b/pkg/azure/blob.go index 229568779..0380e83fd 100644 --- a/pkg/azure/blob.go +++ b/pkg/azure/blob.go @@ -34,6 +34,7 @@ import ( corev1 "k8s.io/api/core/v1" ctrl "sigs.k8s.io/controller-runtime" + "github.com/fluxcd/pkg/masktoken" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" ) @@ -52,6 +53,7 @@ const ( clientCertificateSendChainField = "clientCertificateSendChain" authorityHostField = "authorityHost" accountKeyField = "accountKey" + sasKeyField = "sasKey" ) // BlobClient is a minimal Azure Blob client for fetching objects. @@ -104,6 +106,14 @@ func NewClient(obj *sourcev1.Bucket, secret *corev1.Secret) (c *BlobClient, err c.ServiceClient, err = azblob.NewServiceClientWithSharedKey(obj.Spec.Endpoint, cred, &azblob.ClientOptions{}) return } + + var fullPath string + if fullPath, err = sasTokenFromSecret(obj.Spec.Endpoint, secret); err != nil { + return + } + + c.ServiceClient, err = azblob.NewServiceClientWithNoCredential(fullPath, &azblob.ClientOptions{}) + return } // Compose token chain based on environment. @@ -148,6 +158,9 @@ func ValidateSecret(secret *corev1.Secret) error { if _, hasAccountKey := secret.Data[accountKeyField]; hasAccountKey { valid = true } + if _, hasSasKey := secret.Data[sasKeyField]; hasSasKey { + valid = true + } if _, hasAuthorityHost := secret.Data[authorityHostField]; hasAuthorityHost { valid = true } @@ -343,6 +356,41 @@ func sharedCredentialFromSecret(endpoint string, secret *corev1.Secret) (*azblob return nil, nil } +// sasTokenFromSecret retrieves the SAS Token from the `sasKey`. It returns an empty string if the Secret +// does not contain a valid set of credentials. +func sasTokenFromSecret(ep string, secret *corev1.Secret) (string, error) { + if sasKey, hasSASKey := secret.Data[sasKeyField]; hasSASKey { + queryString := strings.TrimPrefix(string(sasKey), "?") + values, err := url.ParseQuery(queryString) + if err != nil { + maskedErrorString, maskErr := masktoken.MaskTokenFromString(err.Error(), string(sasKey)) + if maskErr != nil { + return "", fmt.Errorf("error redacting token from error message: %s", maskErr) + } + return "", fmt.Errorf("unable to parse SAS token: %s", maskedErrorString) + } + + epURL, err := url.Parse(ep) + if err != nil { + return "", fmt.Errorf("unable to parse endpoint URL: %s", err) + } + + //merge the query values in the endpoint with the token + epValues := epURL.Query() + for key, val := range epValues { + if !values.Has(key) { + for _, str := range val { + values.Add(key, str) + } + } + } + + epURL.RawQuery = values.Encode() + return epURL.String(), nil + } + return "", nil +} + // chainCredentialWithSecret tries to create a set of tokens, and returns an // azidentity.ChainedTokenCredential if at least one of the following tokens was // successfully created: diff --git a/pkg/azure/blob_integration_test.go b/pkg/azure/blob_integration_test.go index 20b28c99a..a00a90331 100644 --- a/pkg/azure/blob_integration_test.go +++ b/pkg/azure/blob_integration_test.go @@ -163,6 +163,67 @@ func TestBlobClient_FGetObject(t *testing.T) { g.Expect(f).To(Equal([]byte(testFileData))) } +func TestBlobClientSASKey_FGetObject(t *testing.T) { + g := NewWithT(t) + + tempDir := t.TempDir() + + // create a client with the shared key + client, err := NewClient(testBucket.DeepCopy(), testSecret.DeepCopy()) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(client).ToNot(BeNil()) + + g.Expect(client.CanGetAccountSASToken()).To(BeTrue()) + + // Generate test container name. + testContainer := generateString(testContainerGenerateName) + + // Create test container. + ctx, timeout := context.WithTimeout(context.Background(), testTimeout) + defer timeout() + g.Expect(createContainer(ctx, client, testContainer)).To(Succeed()) + t.Cleanup(func() { + g.Expect(deleteContainer(context.Background(), client, testContainer)).To(Succeed()) + }) + + // Create test blob. + ctx, timeout = context.WithTimeout(context.Background(), testTimeout) + defer timeout() + g.Expect(createBlob(ctx, client, testContainer, testFile, testFileData)) + + localPath := filepath.Join(tempDir, testFile) + + // use the shared key client to create a SAS key for the account + sasKey, err := client.GetSASToken(azblob.AccountSASResourceTypes{Object: true, Container: true}, + azblob.AccountSASPermissions{List: true, Read: true}, + azblob.AccountSASServices{Blob: true}, + time.Now(), + time.Now().Add(48*time.Hour)) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(sasKey).ToNot(BeEmpty()) + + // the sdk returns the full SAS url e.g test.blob.core.windows.net/? + sasKey = strings.TrimPrefix(sasKey, testBucket.Spec.Endpoint+"/") + testSASKeySecret := corev1.Secret{ + Data: map[string][]byte{ + sasKeyField: []byte(sasKey), + }, + } + + sasKeyClient, err := NewClient(testBucket.DeepCopy(), testSASKeySecret.DeepCopy()) + g.Expect(err).ToNot(HaveOccurred()) + + // Test if blob exists using sasKey. + ctx, timeout = context.WithTimeout(context.Background(), testTimeout) + defer timeout() + _, err = sasKeyClient.FGetObject(ctx, testContainer, testFile, localPath) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(localPath).To(BeARegularFile()) + f, _ := os.ReadFile(localPath) + g.Expect(f).To(Equal([]byte(testFileData))) +} + func TestBlobClient_FGetObject_NotFoundErr(t *testing.T) { g := NewWithT(t) diff --git a/pkg/azure/blob_test.go b/pkg/azure/blob_test.go index 7d8397590..36f5b5b56 100644 --- a/pkg/azure/blob_test.go +++ b/pkg/azure/blob_test.go @@ -25,6 +25,7 @@ import ( "errors" "fmt" "math/big" + "net/url" "testing" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -68,6 +69,14 @@ func TestValidateSecret(t *testing.T) { }, }, }, + { + name: "valid SAS Key Secret", + secret: &corev1.Secret{ + Data: map[string][]byte{ + sasKeyField: []byte("?spr=