From 73ac8187660cfd2d0407adf8cd10808c2b1fd4c7 Mon Sep 17 00:00:00 2001 From: Rafael Fonseca Date: Sat, 22 Oct 2022 01:25:09 +0200 Subject: [PATCH] Use azidentity instead of ADAL ADAL is being deprecated, so use azidentity with an adapter so the new authentication can work with v1 SDK clients. Note that for certificate authentication, autorest uses the `AZURE_CERTIFICATE_` prefix for environment variables whereas azidentity uses `AZURE_CLIENT_CERTIFICATE_`. We make sure the latter is set to the value of the former so as not to break upgrades. Signed-off-by: Rafael Fonseca --- azure/scope/clients.go | 11 ++++-- azure/scope/identity.go | 65 ++++++++++++++++++++------------ go.mod | 11 +++++- go.sum | 19 +++++++++- test/e2e/aks_autoscaler.go | 3 +- test/e2e/aks_public_ip_prefix.go | 3 +- test/e2e/aks_upgrade.go | 3 +- test/e2e/aks_versions.go | 6 ++- test/e2e/azure_clusterproxy.go | 3 +- test/e2e/azure_logcollector.go | 5 ++- test/e2e/azure_privatecluster.go | 7 ++-- test/e2e/azure_vmextensions.go | 3 +- test/e2e/common.go | 3 +- test/e2e/helpers.go | 3 +- util/azure/azure.go | 64 +++++++++++++++++++++++++++++++ 15 files changed, 164 insertions(+), 45 deletions(-) diff --git a/azure/scope/clients.go b/azure/scope/clients.go index 8ec91b96e56..0c2b424e51a 100644 --- a/azure/scope/clients.go +++ b/azure/scope/clients.go @@ -27,6 +27,8 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/azure/auth" + + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" ) // AzureClients contains all the Azure clients used by the scopes. @@ -94,9 +96,12 @@ func (c *AzureClients) setCredentials(subscriptionID, environmentName string) er c.Values[auth.TenantID] = strings.TrimSuffix(c.Values[auth.TenantID], "\n") if c.Authorizer == nil { - c.Authorizer, err = c.GetAuthorizer() + c.Authorizer, err = azureutil.GetAuthorizer(settings) + if err != nil { + return err + } } - return err + return nil } func (c *AzureClients) setCredentialsWithProvider(ctx context.Context, subscriptionID, environmentName string, credentialsProvider CredentialsProvider) error { @@ -129,7 +134,7 @@ func (c *AzureClients) setCredentialsWithProvider(ctx context.Context, subscript } c.Values[auth.ClientSecret] = strings.TrimSuffix(clientSecret, "\n") - c.Authorizer, err = credentialsProvider.GetAuthorizer(ctx, c.ResourceManagerEndpoint, c.Environment.ActiveDirectoryEndpoint) + c.Authorizer, err = credentialsProvider.GetAuthorizer(ctx, c.ResourceManagerEndpoint, c.Environment.ActiveDirectoryEndpoint, c.Environment.TokenAudience) return err } diff --git a/azure/scope/identity.go b/azure/scope/identity.go index 1693070be31..2baa87380f9 100644 --- a/azure/scope/identity.go +++ b/azure/scope/identity.go @@ -20,11 +20,15 @@ import ( "context" "fmt" "reflect" + "strings" aadpodid "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity" aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/go-autorest/autorest" - "github.com/Azure/go-autorest/autorest/adal" + "github.com/jongio/azidext/go/azidext" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -43,7 +47,7 @@ const azureSecretKey = "clientSecret" // CredentialsProvider defines the behavior for azure identity based credential providers. type CredentialsProvider interface { - GetAuthorizer(ctx context.Context, resourceManagerEndpoint, activeDirectoryEndpoint string) (autorest.Authorizer, error) + GetAuthorizer(ctx context.Context, resourceManagerEndpoint, activeDirectoryEndpoint, tokenAudience string) (autorest.Authorizer, error) GetClientID() string GetClientSecret(ctx context.Context) (string, error) GetTenantID() string @@ -98,8 +102,8 @@ func NewAzureClusterCredentialsProvider(ctx context.Context, kubeClient client.C } // GetAuthorizer returns an Azure authorizer based on the provided azure identity. It delegates to AzureCredentialsProvider with AzureCluster metadata. -func (p *AzureClusterCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint, activeDirectoryEndpoint string) (autorest.Authorizer, error) { - return p.AzureCredentialsProvider.GetAuthorizer(ctx, resourceManagerEndpoint, activeDirectoryEndpoint, p.AzureCluster.ObjectMeta) +func (p *AzureClusterCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint, activeDirectoryEndpoint, tokenAudience string) (autorest.Authorizer, error) { + return p.AzureCredentialsProvider.GetAuthorizer(ctx, resourceManagerEndpoint, activeDirectoryEndpoint, tokenAudience, p.AzureCluster.ObjectMeta) } // NewManagedControlPlaneCredentialsProvider creates a new ManagedControlPlaneCredentialsProvider from the supplied inputs. @@ -130,50 +134,61 @@ func NewManagedControlPlaneCredentialsProvider(ctx context.Context, kubeClient c } // GetAuthorizer returns an Azure authorizer based on the provided azure identity. It delegates to AzureCredentialsProvider with AzureManagedControlPlane metadata. -func (p *ManagedControlPlaneCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint, activeDirectoryEndpoint string) (autorest.Authorizer, error) { - return p.AzureCredentialsProvider.GetAuthorizer(ctx, resourceManagerEndpoint, activeDirectoryEndpoint, p.AzureManagedControlPlane.ObjectMeta) +func (p *ManagedControlPlaneCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint, activeDirectoryEndpoint, tokenAudience string) (autorest.Authorizer, error) { + return p.AzureCredentialsProvider.GetAuthorizer(ctx, resourceManagerEndpoint, activeDirectoryEndpoint, tokenAudience, p.AzureManagedControlPlane.ObjectMeta) } // GetAuthorizer returns an Azure authorizer based on the provided azure identity and cluster metadata. -func (p *AzureCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint, activeDirectoryEndpoint string, clusterMeta metav1.ObjectMeta) (autorest.Authorizer, error) { - var spt *adal.ServicePrincipalToken +func (p *AzureCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint, activeDirectoryEndpoint, tokenAudience string, clusterMeta metav1.ObjectMeta) (autorest.Authorizer, error) { + var authErr error + var cred azcore.TokenCredential switch p.Identity.Spec.Type { case infrav1.ServicePrincipal, infrav1.ServicePrincipalCertificate, infrav1.UserAssignedMSI: if err := createAzureIdentityWithBindings(ctx, p.Identity, resourceManagerEndpoint, activeDirectoryEndpoint, clusterMeta, p.Client); err != nil { return nil, err } - msiEndpoint, err := adal.GetMSIVMEndpoint() - if err != nil { - return nil, errors.Errorf("failed to get MSI endpoint: %v", err) - } - - spt, err = adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, resourceManagerEndpoint, p.Identity.Spec.ClientID) - if err != nil { - return nil, errors.Errorf("failed to get token from service principal identity: %v", err) + options := azidentity.ManagedIdentityCredentialOptions{ + ID: azidentity.ClientID(p.Identity.Spec.ClientID), } + cred, authErr = azidentity.NewManagedIdentityCredential(&options) case infrav1.ManualServicePrincipal: - oauthConfig, err := adal.NewOAuthConfig(activeDirectoryEndpoint, p.GetTenantID()) - if err != nil { - return nil, err - } - clientSecret, err := p.GetClientSecret(ctx) if err != nil { return nil, errors.Wrap(err, "failed to get client secret") } - spt, err = adal.NewServicePrincipalToken(*oauthConfig, p.Identity.Spec.ClientID, clientSecret, resourceManagerEndpoint) - if err != nil { - return nil, errors.Errorf("failed to get token from service principal identity: %v", err) + options := azidentity.ClientSecretCredentialOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: cloud.Configuration{ + ActiveDirectoryAuthorityHost: activeDirectoryEndpoint, + Services: map[cloud.ServiceName]cloud.ServiceConfiguration{ + cloud.ResourceManager: { + Audience: tokenAudience, + Endpoint: resourceManagerEndpoint, + }, + }, + }, + }, } + cred, authErr = azidentity.NewClientSecretCredential(p.GetTenantID(), p.Identity.Spec.ClientID, clientSecret, &options) default: return nil, errors.Errorf("identity type %s not supported", p.Identity.Spec.Type) } - return autorest.NewBearerAuthorizer(spt), nil + if authErr != nil { + return nil, errors.Errorf("failed to get token from service principal identity: %v", authErr) + } + // We must use TokenAudience for StackCloud, otherwise we get an + // AADSTS500011 error from the API + scope := tokenAudience + if !strings.HasSuffix(scope, "/.default") { + scope += "/.default" + } + authorizer := azidext.NewTokenCredentialAdapter(cred, []string{scope}) + return authorizer, nil } // GetClientID returns the Client ID associated with the AzureCredentialsProvider's Identity. diff --git a/go.mod b/go.mod index e669b681def..b50d681423e 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,9 @@ go 1.19 require ( github.com/Azure/aad-pod-identity v1.8.9 github.com/Azure/azure-sdk-for-go v67.1.0+incompatible + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/Azure/go-autorest/autorest v0.11.28 - github.com/Azure/go-autorest/autorest/adal v0.9.21 github.com/Azure/go-autorest/autorest/azure/auth v0.5.10 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/Azure/go-autorest/tracing v0.6.0 @@ -19,6 +20,7 @@ require ( github.com/google/uuid v1.3.0 github.com/hashicorp/go-retryablehttp v0.7.0 github.com/hashicorp/golang-lru v0.5.4 + github.com/jongio/azidext/go/azidext v0.4.0 github.com/onsi/ginkgo/v2 v2.5.1 github.com/onsi/gomega v1.24.1 github.com/pkg/errors v0.9.1 @@ -52,13 +54,16 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/mocks v0.4.2 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/BurntSushi/toml v1.0.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -104,7 +109,7 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/gofrs/uuid v4.2.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.2.0 // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/btree v1.0.1 // indirect @@ -128,6 +133,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.13.6 // indirect github.com/kr/fs v0.1.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/lib/pq v1.10.4 // indirect @@ -156,6 +162,7 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect diff --git a/go.sum b/go.sum index bea4a773d95..ba379a8a4cc 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,12 @@ github.com/Azure/aad-pod-identity v1.8.9 h1:cUSgRUBA6X8RFiLQWOgej2FnIMKV+RhLV09I github.com/Azure/aad-pod-identity v1.8.9/go.mod h1:ddDVh8kyCug/HnWo6E9f8ycR/ganxRJpg72VI0DEQ/E= github.com/Azure/azure-sdk-for-go v67.1.0+incompatible h1:oziYcaopbnIKfM69DL05wXdypiqfrUKdxUKrKpynJTw= github.com/Azure/azure-sdk-for-go v67.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -73,6 +79,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= +github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -183,6 +191,7 @@ github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQ github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/distribution/distribution/v3 v3.0.0-20211118083504-a29a3c99a684 h1:DBZ2sN7CK6dgvHVpQsQj4sRMCbWTmd17l+5SUCjnQSY= +github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= github.com/docker/cli v20.10.11+incompatible h1:tXU1ezXcruZQRrMP8RN2z9N91h+6egZTS1gsPsKantc= github.com/docker/cli v20.10.11+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= @@ -290,8 +299,9 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -444,7 +454,10 @@ github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jongio/azidext/go/azidext v0.4.0 h1:TOYyVFMeWGgXNhURSgrEtUCu7JAAKgsy+5C4+AEfYlw= +github.com/jongio/azidext/go/azidext v0.4.0/go.mod h1:VrlpGde5B+pPbTUxnThE5UIQQkcebdr3jrC2MmlMVSI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -479,6 +492,8 @@ github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= @@ -591,6 +606,8 @@ github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaF github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/test/e2e/aks_autoscaler.go b/test/e2e/aks_autoscaler.go index 3d14cc24342..78b79c65421 100644 --- a/test/e2e/aks_autoscaler.go +++ b/test/e2e/aks_autoscaler.go @@ -29,6 +29,7 @@ import ( . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/types" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -46,7 +47,7 @@ func AKSAutoscaleSpec(ctx context.Context, inputGetter func() AKSAutoscaleSpecIn settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - auth, err := settings.GetAuthorizer() + auth, err := azureutil.GetAuthorizer(settings) Expect(err).NotTo(HaveOccurred()) agentpoolClient := containerservice.NewAgentPoolsClient(subscriptionID) agentpoolClient.Authorizer = auth diff --git a/test/e2e/aks_public_ip_prefix.go b/test/e2e/aks_public_ip_prefix.go index 25c3296222d..abe22dea642 100644 --- a/test/e2e/aks_public_ip_prefix.go +++ b/test/e2e/aks_public_ip_prefix.go @@ -32,6 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/util/conditions" @@ -50,7 +51,7 @@ func AKSPublicIPPrefixSpec(ctx context.Context, inputGetter func() AKSPublicIPPr settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - auth, err := settings.GetAuthorizer() + auth, err := azureutil.GetAuthorizer(settings) Expect(err).NotTo(HaveOccurred()) mgmtClient := bootstrapClusterProxy.GetClient() diff --git a/test/e2e/aks_upgrade.go b/test/e2e/aks_upgrade.go index c890e267153..1679516f6be 100644 --- a/test/e2e/aks_upgrade.go +++ b/test/e2e/aks_upgrade.go @@ -28,6 +28,7 @@ import ( . "github.com/onsi/gomega" "github.com/pkg/errors" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" @@ -48,7 +49,7 @@ func AKSUpgradeSpec(ctx context.Context, inputGetter func() AKSUpgradeSpecInput) settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - auth, err := settings.GetAuthorizer() + auth, err := azureutil.GetAuthorizer(settings) Expect(err).NotTo(HaveOccurred()) managedClustersClient := containerservice.NewManagedClustersClient(subscriptionID) diff --git a/test/e2e/aks_versions.go b/test/e2e/aks_versions.go index 336c3e302d1..933e43b0c07 100644 --- a/test/e2e/aks_versions.go +++ b/test/e2e/aks_versions.go @@ -31,6 +31,8 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/controller-runtime/pkg/client" + + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" ) // GetAKSKubernetesVersion gets the kubernetes version for AKS clusters as specified by the environment variable defined by versionVar. @@ -77,7 +79,7 @@ func GetWorkingAKSKubernetesVersion(ctx context.Context, subscriptionID, locatio if err != nil { return "", errors.Wrap(err, "failed to get settings from environment") } - authorizer, err := settings.GetAuthorizer() + authorizer, err := azureutil.GetAuthorizer(settings) if err != nil { return "", errors.Wrap(err, "failed to create an Authorizer") } @@ -147,7 +149,7 @@ func getLatestStableAKSKubernetesVersionOffset(ctx context.Context, subscription if err != nil { return "", errors.Wrap(err, "failed to get settings from environment") } - authorizer, err := settings.GetAuthorizer() + authorizer, err := azureutil.GetAuthorizer(settings) if err != nil { return "", errors.Wrap(err, "failed to create an Authorizer") } diff --git a/test/e2e/azure_clusterproxy.go b/test/e2e/azure_clusterproxy.go index b7a365b53ac..430070d62e3 100644 --- a/test/e2e/azure_clusterproxy.go +++ b/test/e2e/azure_clusterproxy.go @@ -45,6 +45,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" ) @@ -212,7 +213,7 @@ func (acp *AzureClusterProxy) collectActivityLogs(ctx context.Context, namespace settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - authorizer, err := settings.GetAuthorizer() + authorizer, err := azureutil.GetAuthorizer(settings) Expect(err).NotTo(HaveOccurred()) activityLogsClient := insights.NewActivityLogsClient(subscriptionID) activityLogsClient.Authorizer = authorizer diff --git a/test/e2e/azure_logcollector.go b/test/e2e/azure_logcollector.go index 449a950a5ab..b14219fed53 100644 --- a/test/e2e/azure_logcollector.go +++ b/test/e2e/azure_logcollector.go @@ -36,6 +36,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" @@ -408,7 +409,7 @@ func collectVMBootLog(ctx context.Context, am *infrav1.AzureMachine, outputPath } vmClient := compute.NewVirtualMachinesClient(settings.GetSubscriptionID()) - vmClient.Authorizer, err = settings.GetAuthorizer() + vmClient.Authorizer, err = azureutil.GetAuthorizer(settings) if err != nil { return errors.Wrap(err, "failed to get authorizer") } @@ -440,7 +441,7 @@ func collectVMSSBootLog(ctx context.Context, providerID string, outputPath strin } vmssClient := compute.NewVirtualMachineScaleSetVMsClient(settings.GetSubscriptionID()) - vmssClient.Authorizer, err = settings.GetAuthorizer() + vmssClient.Authorizer, err = azureutil.GetAuthorizer(settings) if err != nil { return errors.Wrap(err, "failed to get authorizer") } diff --git a/test/e2e/azure_privatecluster.go b/test/e2e/azure_privatecluster.go index 0cc410d0d25..3afff8476f2 100644 --- a/test/e2e/azure_privatecluster.go +++ b/test/e2e/azure_privatecluster.go @@ -41,6 +41,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/pointer" "sigs.k8s.io/cluster-api-provider-azure/azure" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework" @@ -183,7 +184,7 @@ func AzurePrivateClusterSpec(ctx context.Context, inputGetter func() AzurePrivat Expect(err).To(BeNil()) azureBastionClient := network.NewBastionHostsClient(settings.GetSubscriptionID()) - azureBastionClient.Authorizer, err = settings.GetAuthorizer() + azureBastionClient.Authorizer, err = azureutil.GetAuthorizer(settings) Expect(err).To(BeNil()) groupName := os.Getenv(AzureResourceGroup) @@ -223,7 +224,7 @@ func SetupExistingVNet(ctx context.Context, vnetCidr string, cpSubnetCidrs, node settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - authorizer, err := settings.GetAuthorizer() + authorizer, err := azureutil.GetAuthorizer(settings) Expect(err).NotTo(HaveOccurred()) groupClient := resources.NewGroupsClient(subscriptionID) groupClient.Authorizer = authorizer @@ -456,7 +457,7 @@ func getClientIDforMSI(resourceID string) string { settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - authorizer, err := settings.GetAuthorizer() + authorizer, err := azureutil.GetAuthorizer(settings) Expect(err).NotTo(HaveOccurred()) msiClient := msi.NewUserAssignedIdentitiesClient(subscriptionID) diff --git a/test/e2e/azure_vmextensions.go b/test/e2e/azure_vmextensions.go index 9ecc5e57abd..f1293bbd469 100644 --- a/test/e2e/azure_vmextensions.go +++ b/test/e2e/azure_vmextensions.go @@ -32,6 +32,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/controller-runtime/pkg/client" @@ -75,7 +76,7 @@ func AzureVMExtensionsSpec(ctx context.Context, inputGetter func() AzureVMExtens settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - auth, err := settings.GetAuthorizer() + auth, err := azureutil.GetAuthorizer(settings) Expect(err).NotTo(HaveOccurred()) if len(machineList.Items) > 0 { diff --git a/test/e2e/common.go b/test/e2e/common.go index 1ced5dc03ca..0307f9b1ab7 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -38,6 +38,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/cluster-api-provider-azure/azure" e2e_namespace "sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/namespace" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" kubeadmv1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" @@ -198,7 +199,7 @@ func ExpectResourceGroupToBe404(ctx context.Context) { settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - authorizer, err := settings.GetAuthorizer() + authorizer, err := azureutil.GetAuthorizer(settings) Expect(err).NotTo(HaveOccurred()) groupsClient := resources.NewGroupsClient(subscriptionID) groupsClient.Authorizer = authorizer diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index c5864157672..30e3f0021f4 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -60,6 +60,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" @@ -725,7 +726,7 @@ func newImagesClient() compute.VirtualMachineImagesClient { settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - authorizer, err := settings.GetAuthorizer() + authorizer, err := azureutil.GetAuthorizer(settings) Expect(err).NotTo(HaveOccurred()) imagesClient := compute.NewVirtualMachineImagesClient(subscriptionID) imagesClient.Authorizer = authorizer diff --git a/util/azure/azure.go b/util/azure/azure.go index a1f1bb1e8c0..24ddd67bd80 100644 --- a/util/azure/azure.go +++ b/util/azure/azure.go @@ -17,7 +17,16 @@ limitations under the License. package azure import ( + "os" "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/go-autorest/autorest/azure/auth" + "github.com/jongio/azidext/go/azidext" ) // AzureSystemNodeLabelPrefix is a standard node label prefix for Azure features, e.g., kubernetes.azure.com/scalesetpriority. @@ -27,3 +36,58 @@ const AzureSystemNodeLabelPrefix = "kubernetes.azure.com" func IsAzureSystemNodeLabelKey(labelKey string) bool { return strings.HasPrefix(labelKey, AzureSystemNodeLabelPrefix) } + +func getCloudConfig(environment azure.Environment) cloud.Configuration { + var config cloud.Configuration + switch environment.Name { + case "AzureStackCloud": + config = cloud.Configuration{ + ActiveDirectoryAuthorityHost: environment.ActiveDirectoryEndpoint, + Services: map[cloud.ServiceName]cloud.ServiceConfiguration{ + cloud.ResourceManager: { + Audience: environment.TokenAudience, + Endpoint: environment.ResourceManagerEndpoint, + }, + }, + } + case "AzureChinaCloud": + config = cloud.AzureChina + case "AzureUSGovernmentCloud": + config = cloud.AzureGovernment + default: + config = cloud.AzurePublic + } + return config +} + +// GetAuthorizer returns an autorest.Authorizer-compatible object from MSAL +func GetAuthorizer(settings auth.EnvironmentSettings) (autorest.Authorizer, error) { + // azidentity uses different envvars for certificate authentication: + // azidentity: AZURE_CLIENT_CERTIFICATE_{PATH,PASSWORD} + // autorest: AZURE_CERTIFICATE_{PATH,PASSWORD} + // Let's set them according to the envvars used by autorest, in case they are present + _, azidSet := os.LookupEnv("AZURE_CLIENT_CERTIFICATE_PATH") + path, autorestSet := os.LookupEnv("AZURE_CERTIFICATE_PATH") + if !azidSet && autorestSet { + os.Setenv("AZURE_CLIENT_CERTIFICATE_PATH", path) + os.Setenv("AZURE_CLIENT_CERTIFICATE_PASSWORD", os.Getenv("AZURE_CERTIFICATE_PASSWORD")) + } + + options := azidentity.DefaultAzureCredentialOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: getCloudConfig(settings.Environment), + }, + } + cred, err := azidentity.NewDefaultAzureCredential(&options) + if err != nil { + return nil, err + } + + // We must use TokenAudience for StackCloud, otherwise we get an + // AADSTS500011 error from the API + scope := settings.Environment.TokenAudience + if !strings.HasSuffix(scope, "/.default") { + scope += "/.default" + } + return azidext.NewTokenCredentialAdapter(cred, []string{scope}), nil +}