Merge pull request kubernetes-sigs#4772 from nojnhuh/v2-kubeconfig

reconcile kubeconfig for AzureASOManagedControlPlane

exp/controllers/azureasomanagedcontrolplane_controller.go

@@ -22,6 +22,9 @@ import (
 	"fmt"
 
 	asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001"
+	"github.com/Azure/azure-service-operator/v2/pkg/genruntime"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	infracontroller "sigs.k8s.io/cluster-api-provider-azure/controllers"
 	infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha1"
@@ -32,6 +35,7 @@ import (
 	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/cluster-api/util/predicates"
+	"sigs.k8s.io/cluster-api/util/secret"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -68,6 +72,7 @@ func (r *AzureASOManagedControlPlaneReconciler) SetupWithManager(ctx context.Con
 				infracontroller.ClusterPauseChangeAndInfrastructureReady(log),
 			),
 		).
+		Owns(&corev1.Secret{}).
 		Build(r)
 	if err != nil {
 		return err
@@ -210,9 +215,58 @@ func (r *AzureASOManagedControlPlaneReconciler) reconcileNormal(ctx context.Cont
 		asoManagedControlPlane.Status.Version = "v" + *managedCluster.Status.CurrentKubernetesVersion
 	}
 
+	err = r.reconcileKubeconfig(ctx, asoManagedControlPlane, cluster, managedCluster)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to reconcile kubeconfig: %w", err)
+	}
+
 	return ctrl.Result{}, nil
 }
 
+func (r *AzureASOManagedControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, asoManagedControlPlane *infrav1exp.AzureASOManagedControlPlane, cluster *clusterv1.Cluster, managedCluster *asocontainerservicev1.ManagedCluster) error {
+	ctx, _, done := tele.StartSpanWithLogger(ctx,
+		"controllers.AzureASOManagedControlPlaneReconciler.reconcileKubeconfig",
+	)
+	defer done()
+
+	var secretRef *genruntime.SecretDestination
+	if managedCluster.Spec.OperatorSpec != nil &&
+		managedCluster.Spec.OperatorSpec.Secrets != nil {
+		secretRef = managedCluster.Spec.OperatorSpec.Secrets.UserCredentials
+		if managedCluster.Spec.OperatorSpec.Secrets.AdminCredentials != nil {
+			secretRef = managedCluster.Spec.OperatorSpec.Secrets.AdminCredentials
+		}
+	}
+	if secretRef == nil {
+		return reconcile.TerminalError(fmt.Errorf("ManagedCluster must define at least one of spec.operatorSpec.secrets.{userCredentials,adminCredentials}"))
+	}
+	asoKubeconfig := &corev1.Secret{}
+	err := r.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: secretRef.Name}, asoKubeconfig)
+	if err != nil {
+		return fmt.Errorf("failed to fetch secret created by ASO: %w", err)
+	}
+
+	expectedSecret := &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.Identifier(),
+			Kind:       "Secret",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secret.Name(cluster.Name, secret.Kubeconfig),
+			Namespace: cluster.Namespace,
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(asoManagedControlPlane, infrav1exp.GroupVersion.WithKind(infrav1exp.AzureASOManagedControlPlaneKind)),
+			},
+			Labels: map[string]string{clusterv1.ClusterNameLabel: cluster.Name},
+		},
+		Data: map[string][]byte{
+			secret.KubeconfigDataName: asoKubeconfig.Data[secretRef.Key],
+		},
+	}
+
+	return r.Patch(ctx, expectedSecret, client.Apply, client.FieldOwner("capz-manager"), client.ForceOwnership)
+}
+
 //nolint:unparam // these parameters will be used soon enough
 func (r *AzureASOManagedControlPlaneReconciler) reconcilePaused(ctx context.Context, asoManagedControlPlane *infrav1exp.AzureASOManagedControlPlane, cluster *clusterv1.Cluster) (ctrl.Result, error) {
 	//nolint:all // ctx will be used soon

exp/controllers/azureasomanagedcontrolplane_controller_test.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001"
+	"github.com/Azure/azure-service-operator/v2/pkg/genruntime"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -33,6 +34,7 @@ import (
 	"k8s.io/utils/ptr"
 	infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/util/secret"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -46,6 +48,7 @@ func TestAzureASOManagedControlPlaneReconcile(t *testing.T) {
 		infrav1exp.AddToScheme,
 		clusterv1.AddToScheme,
 		asocontainerservicev1.AddToScheme,
+		corev1.AddToScheme,
 	)
 	NewGomegaWithT(t).Expect(sb.AddToScheme(s)).To(Succeed())
 	fakeClientBuilder := func() *fakeclient.ClientBuilder {
@@ -218,11 +221,30 @@ func TestAzureASOManagedControlPlaneReconcile(t *testing.T) {
 				},
 			},
 		}
+		kubeconfig := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      secret.Name(cluster.Name, secret.Kubeconfig),
+				Namespace: cluster.Namespace,
+			},
+			Data: map[string][]byte{
+				"some other key": []byte("some data"),
+			},
+		}
 		managedCluster := &asocontainerservicev1.ManagedCluster{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "mc",
 				Namespace: cluster.Namespace,
 			},
+			Spec: asocontainerservicev1.ManagedCluster_Spec{
+				OperatorSpec: &asocontainerservicev1.ManagedClusterOperatorSpec{
+					Secrets: &asocontainerservicev1.ManagedClusterOperatorSecrets{
+						UserCredentials: &genruntime.SecretDestination{
+							Name: secret.Name(cluster.Name, secret.Kubeconfig),
+							Key:  "some other key",
+						},
+					},
+				},
+			},
 			Status: asocontainerservicev1.ManagedCluster_STATUS{
 				Fqdn:                     ptr.To("endpoint"),
 				CurrentKubernetesVersion: ptr.To("Current"),
@@ -258,10 +280,19 @@ func TestAzureASOManagedControlPlaneReconcile(t *testing.T) {
 			},
 		}
 		c := fakeClientBuilder().
-			WithObjects(cluster, asoManagedControlPlane, managedCluster).
+			WithObjects(cluster, asoManagedControlPlane, managedCluster, kubeconfig).
 			Build()
+		kubeConfigPatched := false
 		r := &AzureASOManagedControlPlaneReconciler{
-			Client: c,
+			Client: &FakeClient{
+				Client: c,
+				patchFunc: func(_ context.Context, obj client.Object, _ client.Patch, _ ...client.PatchOption) error {
+					kubeconfig := obj.(*corev1.Secret)
+					g.Expect(kubeconfig.Data[secret.KubeconfigDataName]).NotTo(BeEmpty())
+					kubeConfigPatched = true
+					return nil
+				},
+			},
 			newResourceReconciler: func(_ *infrav1exp.AzureASOManagedControlPlane, _ []*unstructured.Unstructured) resourceReconciler {
 				return &fakeResourceReconciler{
 					reconcileFunc: func(ctx context.Context, o client.Object) error {
@@ -277,6 +308,7 @@ func TestAzureASOManagedControlPlaneReconcile(t *testing.T) {
 		g.Expect(c.Get(ctx, client.ObjectKeyFromObject(asoManagedControlPlane), asoManagedControlPlane)).To(Succeed())
 		g.Expect(asoManagedControlPlane.Status.ControlPlaneEndpoint.Host).To(Equal("endpoint"))
 		g.Expect(asoManagedControlPlane.Status.Version).To(Equal("vCurrent"))
+		g.Expect(kubeConfigPatched).To(BeTrue())
 	})
 
 	t.Run("successfully reconciles pause", func(t *testing.T) {