Add support of Azure authentication for ASO

Tiltfile

@@ -184,6 +184,8 @@ def observability():
 
     k8s_resource(workload = "capz-controller-manager", labels = ["cluster-api"])
     k8s_resource(workload = "capz-nmi", labels = ["cluster-api"])
+    k8s_resource(workload = "azureserviceoperator-controller-manager", labels = ["cluster-api"])
+
 
 # Build CAPZ and add feature gates
 def capz():
@@ -253,7 +255,6 @@ def create_identity_secret():
 
     os.putenv("AZURE_CLIENT_SECRET_B64", base64_encode(os.environ.get("AZURE_CLIENT_SECRET")))
     local("cat templates/azure-cluster-identity/secret.yaml | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -", quiet = True, echo_off = True)
-    os.unsetenv("AZURE_CLIENT_SECRET_B64")
 
 def create_crs():
     # create config maps

api/v1beta1/azureclusteridentity_types.go

@@ -22,6 +22,11 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 )
 
+const (
+	// ASOSecretLabel represents the name of the associated AzureClusterIdentity for an ASO secret.
+	ASOSecretLabel = "azureclusteridentity.infrastructure.cluster.x-k8s.io/name"
+)
+
 // AllowedNamespaces defines the namespaces the clusters are allowed to use the identity from
 // NamespaceList takes precedence over the Selector.
 type AllowedNamespaces struct {

azure/scope/identity.go

@@ -31,8 +31,10 @@ import (
 	"github.com/jongio/azidext/go/azidext"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-azure/util/identity"
@@ -54,8 +56,9 @@ type CredentialsProvider interface {
 
 // AzureCredentialsProvider represents a credential provider with azure cluster identity.
 type AzureCredentialsProvider struct {
-	Client   client.Client
-	Identity *infrav1.AzureClusterIdentity
+	Client    client.Client
+	Identity  *infrav1.AzureClusterIdentity
+	ASOSecret string
 }
 
 // AzureClusterCredentialsProvider wraps AzureCredentialsProvider with AzureCluster.
@@ -102,6 +105,10 @@ func NewAzureClusterCredentialsProvider(ctx context.Context, kubeClient client.C
 
 // GetAuthorizer returns an Azure authorizer based on the provided azure identity. It delegates to AzureCredentialsProvider with AzureCluster metadata.
 func (p *AzureClusterCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint, activeDirectoryEndpoint, tokenAudience string) (autorest.Authorizer, error) {
+	if err := reconcileASOSecret(ctx, p.AzureCredentialsProvider.Identity, p.Client, p.AzureCluster.Spec.SubscriptionID); err != nil {
+		return nil, errors.Errorf("failed to reconcile ASO secret related to AzureClusterIdentity %q/%q: %v", p.Identity.Namespace, p.Identity.Name, err)
+	}
+	p.ASOSecret = identity.GetASOSecretName(p.AzureCluster.Spec.SubscriptionID, p.AzureCredentialsProvider.Identity.Name)
 	return p.AzureCredentialsProvider.GetAuthorizer(ctx, resourceManagerEndpoint, activeDirectoryEndpoint, tokenAudience, p.AzureCluster.ObjectMeta)
 }
 
@@ -134,6 +141,11 @@ func NewManagedControlPlaneCredentialsProvider(ctx context.Context, kubeClient c
 
 // GetAuthorizer returns an Azure authorizer based on the provided azure identity. It delegates to AzureCredentialsProvider with AzureManagedControlPlane metadata.
 func (p *ManagedControlPlaneCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint, activeDirectoryEndpoint, tokenAudience string) (autorest.Authorizer, error) {
+	if err := reconcileASOSecret(ctx, p.AzureCredentialsProvider.Identity, p.Client, p.AzureManagedControlPlane.Spec.SubscriptionID); err != nil {
+		return nil, errors.Errorf("failed to reconcile ASO secret related to AzureClusterIdentity %q/%q: %v", p.Identity.Namespace, p.Identity.Name, err)
+	}
+	p.ASOSecret = identity.GetASOSecretName(p.AzureManagedControlPlane.Spec.SubscriptionID, p.AzureCredentialsProvider.Identity.Name)
+
 	return p.AzureCredentialsProvider.GetAuthorizer(ctx, resourceManagerEndpoint, activeDirectoryEndpoint, tokenAudience, p.AzureManagedControlPlane.ObjectMeta)
 }
 
@@ -361,3 +373,112 @@ func IsClusterNamespaceAllowed(ctx context.Context, k8sClient client.Client, all
 
 	return false
 }
+
+func reconcileASOSecret(ctx context.Context, azureClusterIdentity *infrav1.AzureClusterIdentity, kubeClient client.Client, subscriptionID string) error {
+	asoSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      identity.GetASOSecretName(subscriptionID, azureClusterIdentity.Name),
+			Namespace: azureClusterIdentity.Namespace,
+			Labels: map[string]string{
+				infrav1.ASOSecretLabel: azureClusterIdentity.Name,
+			},
+		},
+		Data: map[string][]byte{
+			"AZURE_SUBSCRIPTION_ID": []byte(subscriptionID),
+			"AZURE_TENANT_ID":       []byte(azureClusterIdentity.Spec.TenantID),
+			"AZURE_CLIENT_ID":       []byte(azureClusterIdentity.Spec.ClientID)},
+	}
+
+	apiVersion, kind := infrav1.GroupVersion.WithKind(azureClusterIdentity.Kind).ToAPIVersionAndKind()
+	owner := metav1.OwnerReference{
+		APIVersion: apiVersion,
+		Kind:       kind,
+		Name:       azureClusterIdentity.GetName(),
+		UID:        azureClusterIdentity.GetUID(),
+	}
+
+	// Fetch identity secret, if it exists
+	key := types.NamespacedName{
+		Namespace: azureClusterIdentity.Spec.ClientSecret.Namespace,
+		Name:      azureClusterIdentity.Spec.ClientSecret.Name,
+	}
+	identitySecret := &corev1.Secret{}
+	err := kubeClient.Get(ctx, key, identitySecret)
+	if err != nil && !apierrors.IsNotFound(err) {
+		return errors.Errorf("failed to fetch identity secret %s as defined in AzureClusterIdentity %q/%q: %v", identitySecret.Name, azureClusterIdentity.Namespace, azureClusterIdentity.Name, err)
+	}
+
+	// TODO add support for workload identity
+	switch azureClusterIdentity.Spec.Type {
+	case infrav1.ServicePrincipal, infrav1.ManualServicePrincipal:
+		asoSecret.Data["AZURE_CLIENT_SECRET"] = identitySecret.Data["clientSecret"]
+	case infrav1.ServicePrincipalCertificate:
+		asoSecret.Data["AZURE_CLIENT_CERTIFICATE"] = identitySecret.Data["certificate"]
+		asoSecret.Data["AZURE_CLIENT_CERTIFICATE_PASSWORD"] = identitySecret.Data["password"]
+	case infrav1.UserAssignedMSI:
+		asoSecret.Data["AZURE_IDENTITY_RESOURCE_ID"] = []byte(azureClusterIdentity.Spec.ResourceID)
+	}
+
+	// Fetch previous secret, if it exists
+	key = types.NamespacedName{
+		Namespace: asoSecret.Namespace,
+		Name:      asoSecret.Name,
+	}
+	oldSecret := &corev1.Secret{}
+	err = kubeClient.Get(ctx, key, oldSecret)
+	if err != nil && !apierrors.IsNotFound(err) {
+		return errors.Wrap(err, "failed to fetch existing ASO secret")
+	}
+
+	// Create if it wasn't found
+	if apierrors.IsNotFound(err) {
+		if err := kubeClient.Create(ctx, asoSecret); err != nil && !apierrors.IsAlreadyExists(err) {
+			return errors.Wrap(err, "failed to create ASO secret")
+		}
+		return nil
+	}
+
+	// Otherwise, check ownership and data freshness. Update as necessary
+	hasOwner := false
+	for _, ownerRef := range oldSecret.OwnerReferences {
+		if referSameObject(ownerRef, owner) {
+			hasOwner = true
+			break
+		}
+	}
+
+	hasData := equality.Semantic.DeepEqual(oldSecret.Data, asoSecret.Data)
+	if hasData && hasOwner {
+		// no update required
+		return nil
+	}
+
+	if !hasOwner {
+		oldSecret.OwnerReferences = append(oldSecret.OwnerReferences, owner)
+	}
+
+	if !hasData {
+		oldSecret.Data = asoSecret.Data
+	}
+
+	if err := kubeClient.Update(ctx, oldSecret); err != nil {
+		return errors.Wrap(err, "failed to update ASO secret")
+	}
+
+	return nil
+}
+
+// referSameObject returns true if a and b point to the same object.
+func referSameObject(a, b metav1.OwnerReference) bool {
+	aGV, err := schema.ParseGroupVersion(a.APIVersion)
+	if err != nil {
+		return false
+	}
+
+	bGV, err := schema.ParseGroupVersion(b.APIVersion)
+	if err != nil {
+		return false
+	}
+
+	return aGV.Group == bGV.Group && a.Kind == b.Kind && a.Name == b.Name
+}

azure/services/asogroups/spec.go

@@ -55,6 +55,9 @@ func (s *GroupSpec) Parameters(ctx context.Context, object genruntime.MetaObject
 	return &asoresourcesv1.ResourceGroup{
 		ObjectMeta: metav1.ObjectMeta{
 			OwnerReferences: []metav1.OwnerReference{s.Owner},
+			// Annotations:     map[string]string{
+			//   "serviceoperator.azure.com/credential-from": "my-resource-secret",
+			// },
 		},
 		Spec: asoresourcesv1.ResourceGroup_Spec{
 			Location: pointer.String(s.Location),

config/default/kustomization.yaml

@@ -1,2 +1,4 @@
 resources:
   - ../capz
+components:
+- ../aso

config/rbac/role.yaml

@@ -99,6 +99,14 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - infrastructure.cluster.x-k8s.io
+  resources:
+  - azureclusteridentities
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - infrastructure.cluster.x-k8s.io
   resources:

controllers/azureclusteridentity_controller.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"time"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/client-go/tools/record"
+	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-azure/util/reconciler"
+	"sigs.k8s.io/cluster-api-provider-azure/util/tele"
+	"sigs.k8s.io/cluster-api/util/predicates"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+// AzureClusterIdentityReconciler reconciles ASO identity secrets objects.
+type AzureClusterIdentityReconciler struct {
+	client.Client
+	Recorder         record.EventRecorder
+	ReconcileTimeout time.Duration
+	WatchFilterValue string
+}
+
+// SetupWithManager initializes this controller with a manager.
+func (acir *AzureClusterIdentityReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
+	_, log, done := tele.StartSpanWithLogger(ctx,
+		"controllers.AzureClusterIdentityReconciler.SetupWithManager",
+		tele.KVP("controller", "AzureClusterIdentity"),
+	)
+	defer done()
+
+	_, err := ctrl.NewControllerManagedBy(mgr).
+		WithOptions(options).
+		For(&infrav1.AzureClusterIdentity{}).
+		WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(log, acir.WatchFilterValue)).
+		WithEventFilter(predicates.ResourceIsNotExternallyManaged(log)).
+		Named("AzureClusterIdentity").
+		Build(acir)
+	if err != nil {
+		return errors.Wrap(err, "error creating controller")
+	}
+
+	return nil
+}
+
+// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=azureclusteridentities,verbs=get;list;watch
+// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete
+
+// Reconcile reconciles the AzureClusterIdentity objects.
+func (acir *AzureClusterIdentityReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) {
+	ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultedLoopTimeout(acir.ReconcileTimeout))
+	defer cancel()
+
+	ctx, log, done := tele.StartSpanWithLogger(ctx, "controllers.AzureClusterIdentityReconciler.Reconcile",
+		tele.KVP("namespace", req.Namespace),
+		tele.KVP("name", req.Name),
+		tele.KVP("kind", "AzureClusterIdentity"),
+	)
+	defer done()
+
+	log = log.WithValues("namespace", req.Namespace, "AzureClusterIdentity", req.Name)
+
+	// Fetch the AzureClusterIdentity instance
+	azureClusterIdentity := &infrav1.AzureClusterIdentity{}
+	err := acir.Get(ctx, req.NamespacedName, azureClusterIdentity)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			log.Info("object was not found")
+			return reconcile.Result{}, nil
+		}
+		return reconcile.Result{}, err
+	}
+
+	// Handle deleted azureClusterIdentity
+	if !azureClusterIdentity.DeletionTimestamp.IsZero() {
+		// Cleanup ASO related secrets
+		opt1 := client.InNamespace(azureClusterIdentity.Namespace)
+		opt2 := client.MatchingLabels(map[string]string{
+			infrav1.ASOSecretLabel: azureClusterIdentity.Name,
+		})
+		var asoSecrets corev1.SecretList
+		if err := acir.List(ctx, &asoSecrets, opt1, opt2); err != nil {
+			if apierrors.IsNotFound(err) {
+				log.Info("ASO related secrets not found")
+			} else {
+				return ctrl.Result{}, errors.Wrap(err, "failed to list ASO secrets")
+			}
+		}
+
+		for _, secret := range asoSecrets.Items {
+			secret := secret
+			if err := acir.Client.Delete(ctx, &secret); err != nil {
+				acir.Recorder.Eventf(azureClusterIdentity, corev1.EventTypeWarning, "Error reconciling AzureClusterIdentity", err.Error())
+				log.Error(err, "failed to delete ASO secrets")
+				return ctrl.Result{}, err
+			}
+		}
+	}
+	return ctrl.Result{}, nil
+}

main.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
@@ -69,6 +70,7 @@ func init() {
 	_ = clusterv1.AddToScheme(scheme)
 	_ = expv1.AddToScheme(scheme)
 	_ = kubeadmv1.AddToScheme(scheme)
+	_ = asoresourcesv1.AddToScheme(scheme)
 	// +kubebuilder:scaffold:scheme
 
 	// Add aadpodidentity v1 to the scheme.
@@ -376,6 +378,16 @@ func registerControllers(ctx context.Context, mgr manager.Manager) {
 		os.Exit(1)
 	}
 
+	if err := (&controllers.AzureClusterIdentityReconciler{
+		Client:           mgr.GetClient(),
+		Recorder:         mgr.GetEventRecorderFor("azureclusteridentity-reconciler"),
+		ReconcileTimeout: reconcileTimeout,
+		WatchFilterValue: watchFilterValue,
+	}).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: azureClusterConcurrency}); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "AzureClusterIdentity")
+		os.Exit(1)
+	}
+
 	// just use CAPI MachinePool feature flag rather than create a new one
 	setupLog.V(1).Info(fmt.Sprintf("%+v\n", feature.Gates))
 	if feature.Gates.Enabled(capifeature.MachinePool) {

util/identity/defaults.go

@@ -22,3 +22,8 @@ import "fmt"
 func GetAzureIdentityName(clusterName, clusterNamespace, identityName string) string {
 	return fmt.Sprintf("%s-%s-%s", clusterName, clusterNamespace, identityName)
 }
+
+// GetASOSecretName formats the name of the ASO Secret created by the capz controller.
+func GetASOSecretName(subscriptionID, clusterIdentitySecret string) string {
+	return fmt.Sprintf("%s-%s", clusterIdentitySecret, subscriptionID)
+}