diff --git a/azure/scope/identity.go b/azure/scope/identity.go index 1ee6e2e8327..773b84c1d1a 100644 --- a/azure/scope/identity.go +++ b/azure/scope/identity.go @@ -42,7 +42,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const azureSecretKey = "clientSecret" +// AzureSecretKey is the value for they client secret key. +const AzureSecretKey = "clientSecret" // CredentialsProvider defines the behavior for azure identity based credential providers. type CredentialsProvider interface { @@ -220,7 +221,7 @@ func (p *AzureCredentialsProvider) GetClientSecret(ctx context.Context) (string, if err := p.Client.Get(ctx, key, secret); err != nil { return "", errors.Wrap(err, "Unable to fetch ClientSecret") } - return string(secret.Data[azureSecretKey]), nil + return string(secret.Data[AzureSecretKey]), nil } return "", nil } diff --git a/azure/services/aso/aso.go b/azure/services/aso/aso.go index c96549cfabc..3ee4f210153 100644 --- a/azure/services/aso/aso.go +++ b/azure/services/aso/aso.go @@ -29,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/util/aso" "sigs.k8s.io/cluster-api-provider-azure/util/tele" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" @@ -46,6 +47,9 @@ const ( createOrUpdateFutureType = "ASOCreateOrUpdate" deleteFutureType = "ASODelete" + + // SecretNameAnnotation is the annotation key for ASO's credentials to use. + SecretNameAnnotation = "serviceoperator.azure.com/credential-from" ) // Service is an implementation of the Reconciler interface. It handles creation @@ -181,6 +185,10 @@ func (s *Service) CreateOrUpdateResource(ctx context.Context, spec azure.ASOReso annotations[ReconcilePolicyAnnotation] = ReconcilePolicyManage } + // Set the secret name annotation in order to leverage the ASO resource credential scope as defined in + // https://azure.github.io/azure-service-operator/guide/authentication/credential-scope/#resource-scope. + annotations[SecretNameAnnotation] = aso.GetASOSecretName(s.clusterName) + if len(labels) == 0 { labels = nil } diff --git a/azure/services/aso/aso_test.go b/azure/services/aso/aso_test.go index df0495c816c..dae7d314346 100644 --- a/azure/services/aso/aso_test.go +++ b/azure/services/aso/aso_test.go @@ -149,6 +149,7 @@ func TestCreateOrUpdateResource(t *testing.T) { })) g.Expect(created.Annotations).To(Equal(map[string]string{ ReconcilePolicyAnnotation: ReconcilePolicySkip, + SecretNameAnnotation: "cluster-aso-secret", })) g.Expect(created.Spec).To(Equal(asoresourcesv1.ResourceGroup_Spec{ Location: ptr.To("location"), @@ -425,6 +426,7 @@ func TestCreateOrUpdateResource(t *testing.T) { g.Expect(c.Get(ctx, types.NamespacedName{Name: "name", Namespace: "namespace"}, updated)).To(Succeed()) g.Expect(updated.Annotations).To(Equal(map[string]string{ ReconcilePolicyAnnotation: ReconcilePolicyManage, + SecretNameAnnotation: "cluster-aso-secret", })) }) @@ -482,6 +484,7 @@ func TestCreateOrUpdateResource(t *testing.T) { g.Expect(c.Get(ctx, types.NamespacedName{Name: "name", Namespace: "namespace"}, updated)).To(Succeed()) g.Expect(updated.Annotations).To(Equal(map[string]string{ ReconcilePolicyAnnotation: ReconcilePolicyManage, + SecretNameAnnotation: "cluster-aso-secret", })) }) @@ -603,6 +606,7 @@ func TestCreateOrUpdateResource(t *testing.T) { }, Annotations: map[string]string{ ReconcilePolicyAnnotation: ReconcilePolicyManage, + SecretNameAnnotation: "cluster-aso-secret", }, }, Spec: asoresourcesv1.ResourceGroup_Spec{ diff --git a/controllers/asosecret_controller.go b/controllers/asosecret_controller.go new file mode 100644 index 00000000000..a3a2c145d1c --- /dev/null +++ b/controllers/asosecret_controller.go @@ -0,0 +1,324 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "time" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure/scope" + "sigs.k8s.io/cluster-api-provider-azure/util/aso" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/annotations" + "sigs.k8s.io/cluster-api/util/predicates" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// ASOSecretReconciler reconciles ASO secrets associated with AzureCluster objects. +type ASOSecretReconciler struct { + client.Client + Recorder record.EventRecorder + ReconcileTimeout time.Duration + WatchFilterValue string +} + +// SetupWithManager initializes this controller with a manager. +func (asos *ASOSecretReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + _, log, done := tele.StartSpanWithLogger(ctx, + "controllers.ASOSecretReconciler.SetupWithManager", + tele.KVP("controller", "ASOSecret"), + ) + defer done() + + c, err := ctrl.NewControllerManagedBy(mgr). + WithOptions(options). + For(&infrav1.AzureCluster{}). + WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(log, asos.WatchFilterValue)). + WithEventFilter(predicates.ResourceIsNotExternallyManaged(log)). + Named("ASOSecret"). + Owns(&corev1.Secret{}). + Build(asos) + if err != nil { + return errors.Wrap(err, "error creating controller") + } + + // Add a watch on infrav1.AzureManagedControlPlane. + if err = c.Watch( + &source.Kind{Type: &infrav1.AzureManagedControlPlane{}}, + &handler.EnqueueRequestForObject{}, + predicates.ResourceNotPausedAndHasFilterLabel(log, asos.WatchFilterValue), + ); err != nil { + return errors.Wrap(err, "failed adding a watch for ready AzureManagedControlPlanes") + } + + // Add a watch on ASO secrets owned by an AzureManagedControlPlane + if err = c.Watch( + &source.Kind{Type: &corev1.Secret{}}, + &handler.EnqueueRequestForOwner{ + OwnerType: &infrav1.AzureManagedControlPlane{}, + IsController: true, + }, + ); err != nil { + return errors.Wrap(err, "failed adding a watch for secrets") + } + + // Add a watch on clusterv1.Cluster object for unpause notifications. + if err = c.Watch( + &source.Kind{Type: &clusterv1.Cluster{}}, + handler.EnqueueRequestsFromMapFunc(util.ClusterToInfrastructureMapFunc(ctx, infrav1.GroupVersion.WithKind("AzureCluster"), mgr.GetClient(), &infrav1.AzureCluster{})), + predicates.ClusterUnpaused(log), + predicates.ResourceNotPausedAndHasFilterLabel(log, asos.WatchFilterValue), + ); err != nil { + return errors.Wrap(err, "failed adding a watch for ready clusters") + } + + return nil +} + +// Reconcile reconciles the ASO secrets associated with AzureCluster objects. +func (asos *ASOSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { + ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultedLoopTimeout(asos.ReconcileTimeout)) + defer cancel() + + ctx, log, done := tele.StartSpanWithLogger(ctx, "controllers.ASOSecret.Reconcile", + tele.KVP("namespace", req.Namespace), + tele.KVP("name", req.Name), + tele.KVP("kind", "AzureCluster"), + ) + defer done() + + log = log.WithValues("namespace", req.Namespace, "AzureCluster", req.Name) + + // asoSecretOwner is the resource that created the identity. This could be either an AzureCluster or AzureManagedControlPlane (if AKS is enabled). + // check for AzureCluster first and if it is not found, check for AzureManagedControlPlane. + var asoSecretOwner client.Object + + azureCluster := &infrav1.AzureCluster{} + checkForManagedControlPlane := false + // Fetch the AzureCluster or AzureManagedControlPlane instance + asoSecretOwner = azureCluster + err := asos.Get(ctx, req.NamespacedName, azureCluster) + if err != nil { + if apierrors.IsNotFound(err) { + checkForManagedControlPlane = true + } else { + return reconcile.Result{}, err + } + } else { + log = log.WithValues("AzureCluster", req.Name) + } + + if checkForManagedControlPlane { + // Fetch the AzureManagedControlPlane instance instead + azureManagedControlPlane := &infrav1.AzureManagedControlPlane{} + asoSecretOwner = azureManagedControlPlane + err = asos.Get(ctx, req.NamespacedName, azureManagedControlPlane) + if err != nil { + if apierrors.IsNotFound(err) { + asos.Recorder.Eventf(azureCluster, corev1.EventTypeNormal, "AzureClusterObjectNotFound", + fmt.Sprintf("AzureCluster object %s/%s not found", req.Namespace, req.Name)) + asos.Recorder.Eventf(azureManagedControlPlane, corev1.EventTypeNormal, "AzureManagedControlPlaneObjectNotFound", + fmt.Sprintf("AzureManagedControlPlane object %s/%s not found", req.Namespace, req.Name)) + log.Info("object was not found") + return reconcile.Result{}, nil + } else { + return reconcile.Result{}, err + } + } else { + log = log.WithValues("AzureManagedControlPlane", req.Name) + } + } + + var clusterIdentity *corev1.ObjectReference + var cluster *clusterv1.Cluster + var azureClient scope.AzureClients + + switch ownerType := asoSecretOwner.(type) { + case *infrav1.AzureCluster: + clusterIdentity = ownerType.Spec.IdentityRef + + // Fetch the Cluster. + cluster, err = util.GetOwnerCluster(ctx, asos.Client, ownerType.ObjectMeta) + if err != nil { + return reconcile.Result{}, err + } + + // Create the scope. + clusterScope, err := scope.NewClusterScope(ctx, scope.ClusterScopeParams{ + Client: asos.Client, + Cluster: cluster, + AzureCluster: ownerType, + }) + if err != nil { + return reconcile.Result{}, errors.Wrap(err, "failed to create scope") + } + + azureClient = clusterScope.AzureClients + + case *infrav1.AzureManagedControlPlane: + clusterIdentity = ownerType.Spec.IdentityRef + + // Fetch the Cluster. + cluster, err = util.GetOwnerCluster(ctx, asos.Client, ownerType.ObjectMeta) + if err != nil { + return reconcile.Result{}, err + } + + // Create the scope. + clusterScope, err := scope.NewManagedControlPlaneScope(ctx, scope.ManagedControlPlaneScopeParams{ + Client: asos.Client, + Cluster: cluster, + ControlPlane: ownerType, + }) + if err != nil { + return reconcile.Result{}, errors.Wrap(err, "failed to create scope") + } + + azureClient = clusterScope.AzureClients + } + + if cluster == nil { + log.Info("Cluster Controller has not yet set OwnerRef") + asos.Recorder.Eventf(asoSecretOwner, corev1.EventTypeNormal, "OwnerRefNotFound", + fmt.Sprintf("Cluster Controller has not yet set OwnerRef for object %s/%s", req.Namespace, req.Name)) + return reconcile.Result{}, nil + } + + log = log.WithValues("cluster", cluster.Name) + + // Return early if the ASO Secret Owner(AzureCluster or AzureManagedControlPlane) or Cluster is paused. + if annotations.IsPaused(cluster, asoSecretOwner) { + log.Info(fmt.Sprintf("%s or linked Cluster is marked as paused. Won't reconcile", asoSecretOwner.GetObjectKind())) + asos.Recorder.Eventf(asoSecretOwner, corev1.EventTypeNormal, "ClusterPaused", + fmt.Sprintf("%s or linked Cluster is marked as paused. Won't reconcile", asoSecretOwner.GetObjectKind().GroupVersionKind().Kind)) + return ctrl.Result{}, nil + } + + // Construct the ASO secret for this Cluster + newASOSecret, err := asos.createSecretFromClusterIdentity(ctx, clusterIdentity, cluster, azureClient) + if err != nil { + return reconcile.Result{}, err + } + + gvk := asoSecretOwner.GetObjectKind().GroupVersionKind() + owner := metav1.OwnerReference{ + APIVersion: gvk.GroupVersion().String(), + Kind: gvk.Kind, + Name: asoSecretOwner.GetName(), + UID: asoSecretOwner.GetUID(), + Controller: ptr.To(true), + } + + newASOSecret.OwnerReferences = []metav1.OwnerReference{owner} + + if err := reconcileAzureSecret(ctx, asos.Client, owner, newASOSecret, cluster.GetName()); err != nil { + asos.Recorder.Eventf(cluster, corev1.EventTypeWarning, "Error reconciling ASO secret", err.Error()) + return ctrl.Result{}, errors.Wrap(err, "failed to reconcile ASO secret") + } + + return ctrl.Result{}, nil +} + +func (asos *ASOSecretReconciler) createSecretFromClusterIdentity(ctx context.Context, clusterIdentity *corev1.ObjectReference, cluster *clusterv1.Cluster, azureClient scope.AzureClients) (*corev1.Secret, error) { + newASOSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: aso.GetASOSecretName(cluster.GetName()), + Namespace: cluster.GetNamespace(), + Labels: map[string]string{ + cluster.GetName(): string(infrav1.ResourceLifecycleOwned), + }, + }, + Data: map[string][]byte{ + "AZURE_SUBSCRIPTION_ID": []byte(azureClient.SubscriptionID()), + }, + } + + if clusterIdentity != nil { + // if the namespace isn't specified then assume it's in the same namespace as the Cluster's one + namespace := clusterIdentity.Namespace + if namespace == "" { + namespace = cluster.GetNamespace() + } + identity := &infrav1.AzureClusterIdentity{} + key := client.ObjectKey{ + Name: clusterIdentity.Name, + Namespace: namespace, + } + if err := asos.Get(ctx, key, identity); err != nil { + return nil, errors.Wrap(err, "failed to retrieve AzureClusterIdentity") + } + + newASOSecret.Data["AZURE_TENANT_ID"] = []byte(identity.Spec.TenantID) + newASOSecret.Data["AZURE_CLIENT_ID"] = []byte(identity.Spec.ClientID) + + // Fetch identity secret, if it exists + key = types.NamespacedName{ + Namespace: identity.Spec.ClientSecret.Namespace, + Name: identity.Spec.ClientSecret.Name, + } + identitySecret := &corev1.Secret{} + err := asos.Get(ctx, key, identitySecret) + if err != nil && !apierrors.IsNotFound(err) { + return nil, errors.Wrap(err, "failed to fetch AzureClusterIdentity secret") + } + + switch identity.Spec.Type { + case infrav1.ServicePrincipal, infrav1.ManualServicePrincipal: + newASOSecret.Data["AZURE_CLIENT_SECRET"] = identitySecret.Data[scope.AzureSecretKey] + case infrav1.ServicePrincipalCertificate: + newASOSecret.Data["AZURE_CLIENT_CERTIFICATE"] = identitySecret.Data["certificate"] + newASOSecret.Data["AZURE_CLIENT_CERTIFICATE_PASSWORD"] = identitySecret.Data["password"] + } + } else { + newASOSecret.Data["AZURE_TENANT_ID"] = []byte(azureClient.TenantID()) + newASOSecret.Data["AZURE_CLIENT_ID"] = []byte(azureClient.ClientID()) + + // Populate ASO data in the following order: + // 1. Client credentials + // 2. Client certificate + if _, e := azureClient.GetClientCredentials(); e == nil { + newASOSecret.Data["AZURE_CLIENT_SECRET"] = []byte(azureClient.ClientSecret()) + } else if clientCert, e := azureClient.GetClientCertificate(); e == nil { + cert, err := getCertificateFromFile(clientCert.CertificatePath) + if err != nil { + return nil, errors.Wrap(err, "failed to read client certificate") + } + newASOSecret.Data["AZURE_CLIENT_CERTIFICATE"] = cert + newASOSecret.Data["AZURE_CLIENT_CERTIFICATE_PASSWORD"] = []byte(clientCert.CertificatePassword) + } else { + return nil, errors.Wrap(e, "failed to configure an authentication method for ASO secret") + } + } + return newASOSecret, nil +} diff --git a/controllers/asosecret_controller_test.go b/controllers/asosecret_controller_test.go new file mode 100644 index 00000000000..06eaf004ab7 --- /dev/null +++ b/controllers/asosecret_controller_test.go @@ -0,0 +1,367 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "os" + "testing" + + aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1" + "github.com/Azure/go-autorest/autorest/azure/auth" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestASOSecretReconcile(t *testing.T) { + os.Setenv(auth.ClientID, "fooClient") + os.Setenv(auth.ClientSecret, "fooSecret") + os.Setenv(auth.TenantID, "fooTenant") + os.Setenv(auth.SubscriptionID, "fooSubscription") + + scheme := runtime.NewScheme() + _ = clusterv1.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + _ = clientgoscheme.AddToScheme(scheme) + _ = aadpodv1.AddToScheme(scheme) + + defaultCluster := getASOCluster() + defaultAzureCluster := getASOAzureCluster() + defaultAzureManagedControlPlane := getASOAzureManagedControlPlane() + defaultASOSecret := getASOSecret(defaultAzureCluster) + + cases := map[string]struct { + clusterName string + objects []runtime.Object + err string + event string + asoSecret *corev1.Secret + }{ + "should reconcile normally for AzureCluster without IdentityRef configured": { + clusterName: defaultAzureCluster.Name, + objects: []runtime.Object{ + getASOCluster(func(c *clusterv1.Cluster) { + c.Spec.InfrastructureRef.Name = defaultAzureCluster.Name + c.Spec.InfrastructureRef.Kind = defaultAzureCluster.Kind + }), + defaultAzureCluster, + }, + asoSecret: getASOSecret(defaultAzureCluster, func(s *corev1.Secret) { + s.Data = map[string][]byte{ + "AZURE_SUBSCRIPTION_ID": []byte("123"), + "AZURE_TENANT_ID": []byte("fooTenant"), + "AZURE_CLIENT_ID": []byte("fooClient"), + "AZURE_CLIENT_SECRET": []byte("fooSecret"), + } + }), + }, + "should reconcile normally for AzureManagedControlPlane without IdentityRef configured": { + clusterName: defaultAzureManagedControlPlane.Name, + objects: []runtime.Object{ + getASOCluster(func(c *clusterv1.Cluster) { + c.Spec.InfrastructureRef.Name = defaultAzureManagedControlPlane.Name + c.Spec.InfrastructureRef.Kind = defaultAzureManagedControlPlane.Kind + }), + defaultAzureManagedControlPlane, + }, + asoSecret: getASOSecret(defaultAzureManagedControlPlane, func(s *corev1.Secret) { + s.Data = map[string][]byte{ + "AZURE_SUBSCRIPTION_ID": []byte("fooSubscription"), + "AZURE_TENANT_ID": []byte("fooTenant"), + "AZURE_CLIENT_ID": []byte("fooClient"), + "AZURE_CLIENT_SECRET": []byte("fooSecret"), + } + }), + }, + "should not fail if the azure cluster is not found": { + clusterName: defaultAzureCluster.Name, + objects: []runtime.Object{ + getASOCluster(func(c *clusterv1.Cluster) { + c.Spec.InfrastructureRef.Name = defaultAzureCluster.Name + c.Spec.InfrastructureRef.Kind = defaultAzureCluster.Kind + }), + }, + }, + "should reconcile normally for AzureCluster with IdentityRef configured": { + clusterName: defaultAzureCluster.Name, + objects: []runtime.Object{ + getASOAzureCluster(func(c *infrav1.AzureCluster) { + c.Spec.IdentityRef = &corev1.ObjectReference{ + Name: "my-azure-cluster-identity", + Namespace: "default", + } + }), + getASOAzureClusterIdentity(), + getASOAzureClusterIdentitySecret(), + defaultCluster, + }, + asoSecret: getASOSecret(defaultAzureCluster, func(s *corev1.Secret) { + s.Data = map[string][]byte{ + "AZURE_SUBSCRIPTION_ID": []byte("123"), + "AZURE_TENANT_ID": []byte("fooTenant"), + "AZURE_CLIENT_ID": []byte("fooClient"), + "AZURE_CLIENT_SECRET": []byte("fooSecret"), + } + }), + }, + "should reconcile normally for AzureManagedControlPlane with IdentityRef configured": { + clusterName: defaultAzureManagedControlPlane.Name, + objects: []runtime.Object{ + getASOAzureManagedControlPlane(func(c *infrav1.AzureManagedControlPlane) { + c.Spec.IdentityRef = &corev1.ObjectReference{ + Name: "my-azure-cluster-identity", + Namespace: "default", + } + }), + getASOAzureClusterIdentity(), + getASOAzureClusterIdentitySecret(), + defaultCluster, + }, + asoSecret: getASOSecret(defaultAzureManagedControlPlane, func(s *corev1.Secret) { + s.Data = map[string][]byte{ + "AZURE_SUBSCRIPTION_ID": []byte("fooSubscription"), + "AZURE_TENANT_ID": []byte("fooTenant"), + "AZURE_CLIENT_ID": []byte("fooClient"), + "AZURE_CLIENT_SECRET": []byte("fooSecret"), + } + }), + }, + "should return if cluster does not exist": { + clusterName: defaultAzureCluster.Name, + objects: []runtime.Object{ + defaultAzureCluster, + }, + err: "failed to get Cluster/my-cluster: clusters.cluster.x-k8s.io \"my-cluster\" not found", + }, + "should return if cluster is paused": { + clusterName: defaultAzureCluster.Name, + objects: []runtime.Object{ + getASOCluster(func(c *clusterv1.Cluster) { + c.Spec.Paused = true + }), + defaultAzureCluster, + }, + event: "AzureCluster or linked Cluster is marked as paused. Won't reconcile", + }, + "should return if azureCluster is not yet available": { + clusterName: defaultAzureCluster.Name, + objects: []runtime.Object{ + defaultCluster, + }, + event: "AzureClusterObjectNotFound AzureCluster object default/my-azure-cluster not found", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + g := NewWithT(t) + clientBuilder := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(tc.objects...).Build() + + reconciler := &ASOSecretReconciler{ + Client: clientBuilder, + Recorder: record.NewFakeRecorder(128), + } + + _, err := reconciler.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: "default", + Name: tc.clusterName, + }, + }) + + existingASOSecret := &corev1.Secret{} + asoSecretErr := clientBuilder.Get(context.Background(), types.NamespacedName{ + Namespace: defaultASOSecret.Namespace, + Name: defaultASOSecret.Name, + }, existingASOSecret) + + if tc.asoSecret != nil { + g.Expect(asoSecretErr).NotTo(HaveOccurred()) + g.Expect(tc.asoSecret.Data).To(BeEquivalentTo(existingASOSecret.Data)) + } else { + g.Expect(asoSecretErr).To(HaveOccurred()) + } + + if tc.event != "" { + g.Expect(reconciler.Recorder.(*record.FakeRecorder).Events).To(Receive(ContainSubstring(tc.event))) + } + if tc.err != "" { + g.Expect(err).To(MatchError(tc.err)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} + +func getASOCluster(changes ...func(*clusterv1.Cluster)) *clusterv1.Cluster { + input := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: "default", + }, + Spec: clusterv1.ClusterSpec{ + InfrastructureRef: &corev1.ObjectReference{ + APIVersion: infrav1.GroupVersion.String(), + }, + }, + Status: clusterv1.ClusterStatus{ + InfrastructureReady: true, + }, + } + + for _, change := range changes { + change(input) + } + + return input +} + +func getASOAzureCluster(changes ...func(*infrav1.AzureCluster)) *infrav1.AzureCluster { + input := &infrav1.AzureCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-azure-cluster", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: "my-cluster", + }, + }, + }, + Spec: infrav1.AzureClusterSpec{ + AzureClusterClassSpec: infrav1.AzureClusterClassSpec{ + SubscriptionID: "123", + }, + }, + } + for _, change := range changes { + change(input) + } + + return input +} + +func getASOAzureManagedControlPlane(changes ...func(*infrav1.AzureManagedControlPlane)) *infrav1.AzureManagedControlPlane { + input := &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-azure-managed-control-plane", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + { + Name: "my-cluster", + Kind: "Cluster", + APIVersion: clusterv1.GroupVersion.String(), + }, + }, + }, + Spec: infrav1.AzureManagedControlPlaneSpec{}, + Status: infrav1.AzureManagedControlPlaneStatus{ + Ready: true, + Initialized: true, + }, + } + for _, change := range changes { + change(input) + } + + return input +} + +func getASOAzureClusterIdentity(changes ...func(identity *infrav1.AzureClusterIdentity)) *infrav1.AzureClusterIdentity { + input := &infrav1.AzureClusterIdentity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-azure-cluster-identity", + Namespace: "default", + }, + Spec: infrav1.AzureClusterIdentitySpec{ + Type: infrav1.IdentityType("ServicePrincipal"), + ClientID: "fooClient", + ClientSecret: corev1.SecretReference{ + Name: "fooSecret", + Namespace: "default", + }, + TenantID: "fooTenant", + }, + } + + for _, change := range changes { + change(input) + } + + return input +} + +func getASOAzureClusterIdentitySecret(changes ...func(secret *corev1.Secret)) *corev1.Secret { + input := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fooSecret", + Namespace: "default", + }, + Data: map[string][]byte{ + "clientSecret": []byte("fooSecret"), + }, + } + + for _, change := range changes { + change(input) + } + + return input +} + +func getASOSecret(cluster client.Object, changes ...func(secret *corev1.Secret)) *corev1.Secret { + input := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster-aso-secret", + Namespace: "default", + Labels: map[string]string{ + "my-cluster": "owned", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: cluster.GetObjectKind().GroupVersionKind().GroupVersion().String(), + Kind: cluster.GetObjectKind().GroupVersionKind().Kind, + Name: cluster.GetName(), + UID: cluster.GetUID(), + Controller: ptr.To(true), + }, + }, + }, + Data: map[string][]byte{ + "AZURE_SUBSCRIPTION_ID": []byte("fooSubscription"), + }, + } + + for _, change := range changes { + change(input) + } + + return input +} diff --git a/controllers/helpers.go b/controllers/helpers.go index 05fd93695e5..f2eb54c2909 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -22,6 +22,8 @@ import ( "encoding/hex" "encoding/json" "fmt" + "os" + "strings" "github.com/go-logr/logr" "github.com/pkg/errors" @@ -475,13 +477,13 @@ func reconcileAzureSecret(ctx context.Context, kubeclient client.Client, owner m old := &corev1.Secret{} err := kubeclient.Get(ctx, key, old) if err != nil && !apierrors.IsNotFound(err) { - return errors.Wrap(err, "failed to fetch existing azure json") + return errors.Wrap(err, "failed to fetch existing secret") } // Create if it wasn't found if apierrors.IsNotFound(err) { if err := kubeclient.Create(ctx, newSecret); err != nil && !apierrors.IsAlreadyExists(err) { - return errors.Wrap(err, "failed to create cluster azure json") + return errors.Wrap(err, "failed to create secret") } return nil } @@ -489,7 +491,7 @@ func reconcileAzureSecret(ctx context.Context, kubeclient client.Client, owner m tag, exists := old.Labels[clusterName] if !exists || tag != string(infrav1.ResourceLifecycleOwned) { - log.V(2).Info("returning early from json reconcile, user provided secret already exists") + log.V(2).Info("returning early from secret reconcile, user provided secret already exists") return nil } @@ -505,7 +507,7 @@ func reconcileAzureSecret(ctx context.Context, kubeclient client.Client, owner m hasData := equality.Semantic.DeepEqual(old.Data, newSecret.Data) if hasData && hasOwner { // no update required - log.V(2).Info("returning early from json reconcile, no update needed") + log.V(2).Info("returning early from secret reconcile, no update needed") return nil } @@ -517,12 +519,12 @@ func reconcileAzureSecret(ctx context.Context, kubeclient client.Client, owner m old.Data = newSecret.Data } - log.V(2).Info("updating azure json") + log.V(2).Info("updating azure secret") if err := kubeclient.Update(ctx, old); err != nil { - return errors.Wrap(err, "failed to update cluster azure json when diff was required") + return errors.Wrap(err, "failed to update secret when diff was required") } - log.V(2).Info("done updating azure json") + log.V(2).Info("done updating secret") return nil } @@ -1056,3 +1058,12 @@ func ClusterUpdatePauseChange(logger logr.Logger) predicate.Funcs { GenericFunc: func(e event.GenericEvent) bool { return false }, } } + +func getCertificateFromFile(certificateFilePath string) ([]byte, error) { + certificateFilePathTrimmed := strings.TrimSpace(certificateFilePath) + if certificateFilePathTrimmed == "" { + return nil, fmt.Errorf("certificate path is empty") + } + + return os.ReadFile(certificateFilePathTrimmed) +} diff --git a/main.go b/main.go index 4cd66fbc26c..4491460bb98 100644 --- a/main.go +++ b/main.go @@ -376,6 +376,16 @@ func registerControllers(ctx context.Context, mgr manager.Manager) { os.Exit(1) } + if err := (&controllers.ASOSecretReconciler{ + Client: mgr.GetClient(), + Recorder: mgr.GetEventRecorderFor("asosecret-reconciler"), + ReconcileTimeout: reconcileTimeout, + WatchFilterValue: watchFilterValue, + }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: azureClusterConcurrency}); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ASOSecret") + os.Exit(1) + } + // just use CAPI MachinePool feature flag rather than create a new one setupLog.V(1).Info(fmt.Sprintf("%+v\n", feature.Gates)) if feature.Gates.Enabled(capifeature.MachinePool) { diff --git a/util/aso/defaults.go b/util/aso/defaults.go new file mode 100644 index 00000000000..0139a1c59ba --- /dev/null +++ b/util/aso/defaults.go @@ -0,0 +1,24 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aso + +import "fmt" + +// GetASOSecretName formats the name of the ASO Secret created by the capz controller. +func GetASOSecretName(clusterOwner string) string { + return fmt.Sprintf("%s-aso-secret", clusterOwner) +}