From ba6b705f51438f067fe179738d564ee1d49f6034 Mon Sep 17 00:00:00 2001 From: LochanRn Date: Tue, 19 Sep 2023 02:00:31 +0530 Subject: [PATCH] Disable local accounts for aks aad clusters --- api/v1beta1/azuremanagedcontrolplane_types.go | 4 + .../azuremanagedcontrolplane_webhook.go | 19 + .../azuremanagedcontrolplane_webhook_test.go | 74 +++ api/v1beta1/zz_generated.deepcopy.go | 5 + azure/scope/managedcontrolplane.go | 52 ++- azure/scope/managedcontrolplane_test.go | 428 ++++++++++++++++++ azure/services/managedclusters/client.go | 18 + .../managedclusters/managedclusters.go | 98 +++- .../managedclusters/managedclusters_test.go | 56 ++- .../mock_managedclusters/client_mock.go | 15 + .../managedclusters_mock.go | 90 +++- azure/services/managedclusters/spec.go | 14 + azure/services/token/client.go | 81 ++++ ...er.x-k8s.io_azuremanagedcontrolplanes.yaml | 5 + .../azuremanagedcontrolplane_reconciler.go | 30 +- 15 files changed, 939 insertions(+), 50 deletions(-) create mode 100644 azure/services/token/client.go diff --git a/api/v1beta1/azuremanagedcontrolplane_types.go b/api/v1beta1/azuremanagedcontrolplane_types.go index e417a56b9fc1..a4d236cdb4e3 100644 --- a/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/api/v1beta1/azuremanagedcontrolplane_types.go @@ -213,6 +213,10 @@ type AzureManagedControlPlaneSpec struct { // OIDCIssuerProfile is the OIDC issuer profile of the Managed Cluster. // +optional OIDCIssuerProfile *OIDCIssuerProfile `json:"oidcIssuerProfile,omitempty"` + + // DisableLocalAccounts - If set to true, getting static credential will be disabled for this cluster. Expected to only be used for AAD clusters. + // +optional + DisableLocalAccounts *bool `json:"disableLocalAccounts,omitempty"` } // HTTPProxyConfig is the HTTP proxy configuration for the cluster. diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook.go b/api/v1beta1/azuremanagedcontrolplane_webhook.go index df5f6b264567..f23542acabb6 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -240,6 +240,16 @@ func (mw *azureManagedControlPlaneWebhook) ValidateUpdate(ctx context.Context, o } } + if old.Spec.DisableLocalAccounts == nil && + m.Spec.DisableLocalAccounts != nil && + m.Spec.AADProfile == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "DisableLocalAccounts"), + m.Spec.DisableLocalAccounts, + "DisableLocalAccounts can be set only for AAD enabled clusters")) + } + // Consider removing this once moves out of preview // Updating outboundType after cluster creation (PREVIEW) // https://learn.microsoft.com/en-us/azure/aks/egress-outboundtype#updating-outboundtype-after-cluster-creation-preview @@ -290,6 +300,7 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { m.validateAutoScalerProfile, m.validateIdentity, m.validateNetworkPluginMode, + m.validateDisableLocalAccounts, } var errs []error @@ -302,6 +313,14 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { return kerrors.NewAggregate(errs) } +// validateVersion disabling local accounts for AAD based clusters. +func (m *AzureManagedControlPlane) validateDisableLocalAccounts(_ client.Client) error { + if m.Spec.DisableLocalAccounts != nil && m.Spec.AADProfile == nil { + return errors.New("DisableLocalAccounts should be set only for AAD enabled clusters") + } + return nil +} + // validateVersion validates the Kubernetes version. func (m *AzureManagedControlPlane) validateVersion(_ client.Client) error { if !kubeSemver.MatchString(m.Spec.Version) { diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index 438bcbf0266c..ebb78b1489fa 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -848,6 +848,30 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { }, wantErr: false, }, + { + name: "DisableLocalAccounts cannot be set for non AAD clusters", + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.21.2", + DisableLocalAccounts: ptr.To[bool](true), + }, + }, + wantErr: true, + }, + { + name: "DisableLocalAccounts can be set for AAD clusters", + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.21.2", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To[bool](true), + }, + }, + wantErr: false, + }, } client := mockClient{ReturnError: false} for _, tc := range tests { @@ -1609,6 +1633,56 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: false, }, + { + name: "DisableLocalAccounts can be set only for AAD enabled clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + DisableLocalAccounts: ptr.To[bool](true), + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + wantErr: false, + }, + { + name: "DisableLocalAccounts cannot be set only for non AAD clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + DisableLocalAccounts: ptr.To[bool](true), + }, + }, + wantErr: true, + }, } client := mockClient{ReturnError: false} for _, tc := range tests { diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index d9270d475bae..f27663d321b3 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1247,6 +1247,11 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = new(OIDCIssuerProfile) (*in).DeepCopyInto(*out) } + if in.DisableLocalAccounts != nil { + in, out := &in.DisableLocalAccounts, &out.DisableLocalAccounts + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneSpec. diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 9be7e235e45f..84bd509283d6 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -116,10 +116,11 @@ func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlane // ManagedControlPlaneScope defines the basic context for an actuator to operate upon. type ManagedControlPlaneScope struct { - Client client.Client - patchHelper *patch.Helper - kubeConfigData []byte - cache *ManagedControlPlaneCache + Client client.Client + patchHelper *patch.Helper + adminKubeConfigData []byte + userKubeConfigData []byte + cache *ManagedControlPlaneCache AzureClients Cluster *clusterv1.Cluster @@ -462,6 +463,24 @@ func (s *ManagedControlPlaneScope) ManagedClusterAnnotations() map[string]string return s.ControlPlane.Annotations } +// IsLocalAcountsDisabled checks if local accounts are disabled for aad enabled managed clusters. +func (s *ManagedControlPlaneScope) IsLocalAcountsDisabled() bool { + if s.IsAadEnabled() && + s.ControlPlane.Spec.DisableLocalAccounts != nil && + *s.ControlPlane.Spec.DisableLocalAccounts { + return true + } + return false +} + +// IsAadEnabled checks if azure active directory is enabled for managed clusters. +func (s *ManagedControlPlaneScope) IsAadEnabled() bool { + if s.ControlPlane.Spec.AADProfile != nil && s.ControlPlane.Spec.AADProfile.Managed { + return true + } + return false +} + // ManagedClusterSpec returns the managed cluster spec. func (s *ManagedControlPlaneScope) ManagedClusterSpec() azure.ResourceSpecGetter { managedClusterSpec := managedclusters.ManagedClusterSpec{ @@ -515,6 +534,9 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec() azure.ResourceSpecGetter EnableAzureRBAC: s.ControlPlane.Spec.AADProfile.Managed, AdminGroupObjectIDs: s.ControlPlane.Spec.AADProfile.AdminGroupObjectIDs, } + if s.ControlPlane.Spec.DisableLocalAccounts != nil { + managedClusterSpec.DisableLocalAccounts = s.ControlPlane.Spec.DisableLocalAccounts + } } if s.ControlPlane.Spec.AddonProfiles != nil { @@ -642,14 +664,24 @@ func (s *ManagedControlPlaneScope) MakeEmptyKubeConfigSecret() corev1.Secret { } } -// GetKubeConfigData returns a []byte that contains kubeconfig. -func (s *ManagedControlPlaneScope) GetKubeConfigData() []byte { - return s.kubeConfigData +// GetAdminKubeConfigData returns admin kubeconfig. +func (s *ManagedControlPlaneScope) GetAdminKubeConfigData() []byte { + return s.adminKubeConfigData +} + +// SetAdminKubeConfigData sets adminKubeconfig data. +func (s *ManagedControlPlaneScope) SetAdminKubeConfigData(kubeConfigData []byte) { + s.adminKubeConfigData = kubeConfigData +} + +// GetUserKubeConfigData returns user kubeconfig, required when using AAD with AKS cluster. +func (s *ManagedControlPlaneScope) GetUserKubeConfigData() []byte { + return s.userKubeConfigData } -// SetKubeConfigData sets kubeconfig data. -func (s *ManagedControlPlaneScope) SetKubeConfigData(kubeConfigData []byte) { - s.kubeConfigData = kubeConfigData +// SetUserKubeConfigData sets userKubeconfig data. +func (s *ManagedControlPlaneScope) SetUserKubeConfigData(kubeConfigData []byte) { + s.userKubeConfigData = kubeConfigData } // SetKubeletIdentity sets the ID of the user-assigned identity for kubelet if not already set. diff --git a/azure/scope/managedcontrolplane_test.go b/azure/scope/managedcontrolplane_test.go index 9b477d48fa57..bc74e0c856da 100644 --- a/azure/scope/managedcontrolplane_test.go +++ b/azure/scope/managedcontrolplane_test.go @@ -610,3 +610,431 @@ func TestManagedControlPlaneScope_IsVnetManagedCache(t *testing.T) { }) } } + +func TestManagedControlPlaneScope_AADProfile(t *testing.T) { + scheme := runtime.NewScheme() + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected *managedclusters.AADProfile + }{ + { + Name: "Without AADProfile", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "With AADProfile", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: &managedclusters.AADProfile{ + Managed: true, + EnableAzureRBAC: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + managedClusterGetter := s.ManagedClusterSpec() + managedCluster, ok := managedClusterGetter.(*managedclusters.ManagedClusterSpec) + g.Expect(ok).To(BeTrue()) + g.Expect(managedCluster.AADProfile).To(Equal(c.Expected)) + }) + } +} + +func TestManagedControlPlaneScope_DisableLocalAccounts(t *testing.T) { + scheme := runtime.NewScheme() + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected *bool + }{ + { + Name: "Without DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "Without AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + DisableLocalAccounts: ptr.To[bool](true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "With AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To[bool](true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: ptr.To[bool](true), + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + managedClusterGetter := s.ManagedClusterSpec() + managedCluster, ok := managedClusterGetter.(*managedclusters.ManagedClusterSpec) + g.Expect(ok).To(BeTrue()) + g.Expect(managedCluster.DisableLocalAccounts).To(Equal(c.Expected)) + }) + } +} + +func TestIsAaDEnabled(t *testing.T) { + scheme := runtime.NewScheme() + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected bool + }{ + { + Name: "AAD is not enabled", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To[bool](true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + aadEnabled := s.IsAadEnabled() + g.Expect(aadEnabled).To(Equal(c.Expected)) + }) + } +} + +func TestIsLocalAcountsDisabled(t *testing.T) { + scheme := runtime.NewScheme() + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected bool + }{ + { + Name: "DisbaleLocalAccount is not enabled", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "With AAdProfile and Without DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "With AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To[bool](true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + localAccountsDisabled := s.IsLocalAcountsDisabled() + g.Expect(localAccountsDisabled).To(Equal(c.Expected)) + }) + } +} diff --git a/azure/services/managedclusters/client.go b/azure/services/managedclusters/client.go index e64975c63cd7..9f7757331c42 100644 --- a/azure/services/managedclusters/client.go +++ b/azure/services/managedclusters/client.go @@ -31,6 +31,7 @@ import ( // CredentialGetter is a helper interface for getting managed cluster credentials. type CredentialGetter interface { GetCredentials(context.Context, string, string) ([]byte, error) + GetUserCredentials(context.Context, string, string) ([]byte, error) } // azureClient contains the Azure go-sdk Client. @@ -84,6 +85,23 @@ func (ac *azureClient) GetCredentials(ctx context.Context, resourceGroupName, na return credentialList.Kubeconfigs[0].Value, nil } +// GetUserCredentials fetches the user kubeconfig for a managed cluster. +func (ac *azureClient) GetUserCredentials(ctx context.Context, resourceGroupName, name string) ([]byte, error) { + ctx, _, done := tele.StartSpanWithLogger(ctx, "managedclusters.azureClient.GetCredentials") + defer done() + + credentialList, err := ac.managedclusters.ListClusterUserCredentials(ctx, resourceGroupName, name, nil) + if err != nil { + return nil, err + } + + if len(credentialList.Kubeconfigs) == 0 { + return nil, errors.New("no user kubeconfigs available for the managed cluster") + } + + return credentialList.Kubeconfigs[0].Value, nil +} + // CreateOrUpdateAsync creates or updates a managed cluster. // It sends a PUT request to Azure and if accepted without error, the func will return a Poller which can be used to track the ongoing // progress of the operation. diff --git a/azure/services/managedclusters/managedclusters.go b/azure/services/managedclusters/managedclusters.go index 3b3fd23cd365..7f6b66ad3e2e 100644 --- a/azure/services/managedclusters/managedclusters.go +++ b/azure/services/managedclusters/managedclusters.go @@ -22,18 +22,22 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/clientcmd" "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/async" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/token" "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" "sigs.k8s.io/cluster-api-provider-azure/util/tele" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) -const serviceName = "managedcluster" - -const kubeletIdentityKey = "kubeletidentity" +const ( + serviceName = "managedcluster" + kubeletIdentityKey = "kubeletidentity" + aadResourceID = "6dae42f8-4368-4678-94ff-3960e28e3630" +) // ManagedClusterScope defines the scope interface for a managed cluster. type ManagedClusterScope interface { @@ -43,8 +47,12 @@ type ManagedClusterScope interface { SetControlPlaneEndpoint(clusterv1.APIEndpoint) SetKubeletIdentity(string) MakeEmptyKubeConfigSecret() corev1.Secret - GetKubeConfigData() []byte - SetKubeConfigData([]byte) + GetAdminKubeConfigData() []byte + SetAdminKubeConfigData([]byte) + GetUserKubeConfigData() []byte + SetUserKubeConfigData([]byte) + IsAadEnabled() bool + IsLocalAcountsDisabled() bool SetOIDCIssuerProfileStatus(*infrav1.OIDCIssuerProfileStatus) } @@ -102,11 +110,13 @@ func (s *Service) Reconcile(ctx context.Context) error { // Update kubeconfig data // Always fetch credentials in case of rotation - kubeConfigData, err := s.GetCredentials(ctx, managedClusterSpec.ResourceGroupName(), managedClusterSpec.ResourceName()) + adminKubeConfigData, userKubeConfigData, err := s.ReconcileKubeconfig(ctx, managedClusterSpec) if err != nil { - return errors.Wrap(err, "failed to get credentials for managed cluster") + return errors.Wrap(err, "error while reconciling adminKubeConfigData") } - s.Scope.SetKubeConfigData(kubeConfigData) + + s.Scope.SetAdminKubeConfigData(adminKubeConfigData) + s.Scope.SetUserKubeConfigData(userKubeConfigData) // This field gets populated by AKS when not set by the user. Persist AKS's value so for future diffs, // the "before" reflects the correct value. @@ -147,3 +157,75 @@ func (s *Service) Delete(ctx context.Context) error { func (s *Service) IsManaged(ctx context.Context) (bool, error) { return true, nil } + +// ReconcileKubeconfig will reconcile admin kubeconfig and user kubeconfig. +/* + Returns the admin kubeconfig and user kubeconfig + If aad is enabled a user kubeconfig will also get generated and stored in the secret -kubeconfig-user + If we disable local accounts for aad clusters we do not have access to admin kubeconfig, hence we need to create + the admin kubeconfig by authenticating with the user credentials and retrieving the token for kubeconfig. + The token is used to create the admin kubeconfig. + The user needs to ensure to provide service principle with admin aad privileges. +*/ +func (s *Service) ReconcileKubeconfig(ctx context.Context, managedClusterSpec azure.ResourceSpecGetter) (userKubeConfigData []byte, adminKubeConfigData []byte, err error) { + if s.Scope.IsAadEnabled() { + if userKubeConfigData, err = s.GetUserKubeConfigData(ctx, managedClusterSpec); err != nil { + return nil, nil, errors.Wrap(err, "error while trying to get user kubeconfig") + } + } + + if s.Scope.IsLocalAcountsDisabled() { + userKubeconfigWithToken, err := s.GetUserKubeConfigWithToken(userKubeConfigData, ctx, managedClusterSpec) + if err != nil { + return nil, nil, errors.Wrap(err, "error while trying to get user kubeconfig with token") + } + return userKubeconfigWithToken, userKubeConfigData, nil + } + + adminKubeConfigData, err = s.GetCredentials(ctx, managedClusterSpec.ResourceGroupName(), managedClusterSpec.ResourceName()) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to get credentials for managed cluster") + } + return adminKubeConfigData, userKubeConfigData, nil +} + +// GetUserKubeConfigData gets user kubeconfig when aad is enabled for the aad clusters. +func (s *Service) GetUserKubeConfigData(ctx context.Context, managedClusterSpec azure.ResourceSpecGetter) ([]byte, error) { + kubeConfigData, err := s.GetUserCredentials(ctx, managedClusterSpec.ResourceGroupName(), managedClusterSpec.ResourceName()) + if err != nil { + return nil, errors.Wrap(err, "failed to get credentials for managed cluster") + } + return kubeConfigData, nil +} + +// GetUserKubeConfigWithToken returns the kubeconfig with user token, for capz to create the target cluster. +func (s *Service) GetUserKubeConfigWithToken(userKubeConfigData []byte, ctx context.Context, managedClusterSpec azure.ResourceSpecGetter) ([]byte, error) { + tokenClient, err := token.NewClient(s.Scope) + if err != nil { + return nil, errors.Wrap(err, "error while getting aad token client") + } + + token, err := tokenClient.GetAzureActiveDirectoryToken(ctx, aadResourceID) + if err != nil { + return nil, errors.Wrap(err, "error while getting aad token for user kubeconfig") + } + + return s.CreateUserKubeconfigWithToken(token, userKubeConfigData) +} + +// CreateUserKubeconfigWithToken gets the kubeconfigdata for authenticating with target cluster. +func (s *Service) CreateUserKubeconfigWithToken(token string, userKubeConfigData []byte) ([]byte, error) { + config, err := clientcmd.Load(userKubeConfigData) + if err != nil { + return nil, errors.Wrap(err, "error while trying to unmarshal new user kubeconfig with token") + } + for _, auth := range config.AuthInfos { + auth.Token = token + auth.Exec = nil + } + kubeconfig, err := clientcmd.Write(*config) + if err != nil { + return nil, errors.Wrap(err, "error while trying to marshal new user kubeconfig with token") + } + return kubeconfig, nil +} diff --git a/azure/services/managedclusters/managedclusters_test.go b/azure/services/managedclusters/managedclusters_test.go index ef0a592b6e4e..fa2cfff18147 100644 --- a/azure/services/managedclusters/managedclusters_test.go +++ b/azure/services/managedclusters/managedclusters_test.go @@ -33,6 +33,14 @@ import ( ) var fakeManagedClusterSpec = &ManagedClusterSpec{Name: "my-managedcluster", ResourceGroup: "my-rg"} +var fakeManagedClusterSpecWithAAD = &ManagedClusterSpec{ + Name: "my-managedcluster", + ResourceGroup: "my-rg", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"000000-000000-000000-000000"}, + }, +} func TestReconcile(t *testing.T) { testcases := []struct { @@ -60,6 +68,7 @@ func TestReconcile(t *testing.T) { name: "create managed cluster succeeds", expectedError: "", expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + var userKubeConfigData []byte s.ManagedClusterSpec().Return(fakeManagedClusterSpec) r.CreateOrUpdateResource(gomockinternal.AContext(), fakeManagedClusterSpec, serviceName).Return(armcontainerservice.ManagedCluster{ Properties: &armcontainerservice.ManagedClusterProperties{ @@ -80,8 +89,49 @@ func TestReconcile(t *testing.T) { Host: "my-managedcluster-fqdn", Port: 443, }) + s.IsAadEnabled().Return(false) + s.IsLocalAcountsDisabled().Return(false) + m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials"), nil) + s.SetAdminKubeConfigData([]byte("credentials")) + s.SetUserKubeConfigData(userKubeConfigData) + s.SetKubeletIdentity("kubelet-id") + s.SetOIDCIssuerProfileStatus(nil) + s.SetOIDCIssuerProfileStatus(&infrav1.OIDCIssuerProfileStatus{ + IssuerURL: ptr.To("oidc issuer url"), + }) + s.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, nil) + }, + }, + { + name: "create managed cluster succeeds with user kubeconfig", + expectedError: "", + expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.ManagedClusterSpec().Return(fakeManagedClusterSpecWithAAD) + r.CreateOrUpdateResource(gomockinternal.AContext(), fakeManagedClusterSpecWithAAD, serviceName).Return(armcontainerservice.ManagedCluster{ + Properties: &armcontainerservice.ManagedClusterProperties{ + Fqdn: ptr.To("my-managedcluster-fqdn"), + ProvisioningState: ptr.To("Succeeded"), + IdentityProfile: map[string]*armcontainerservice.UserAssignedIdentity{ + kubeletIdentityKey: { + ResourceID: ptr.To("kubelet-id"), + }, + }, + OidcIssuerProfile: &armcontainerservice.ManagedClusterOIDCIssuerProfile{ + Enabled: ptr.To(true), + IssuerURL: ptr.To("oidc issuer url"), + }, + }, + }, nil) + s.SetControlPlaneEndpoint(clusterv1.APIEndpoint{ + Host: "my-managedcluster-fqdn", + Port: 443, + }) + s.IsAadEnabled().Return(true) + s.IsLocalAcountsDisabled().Return(false) m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials"), nil) - s.SetKubeConfigData([]byte("credentials")) + m.GetUserCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials-user"), nil) + s.SetAdminKubeConfigData([]byte("credentials")) + s.SetUserKubeConfigData([]byte("credentials-user")) s.SetKubeletIdentity("kubelet-id") s.SetOIDCIssuerProfileStatus(nil) s.SetOIDCIssuerProfileStatus(&infrav1.OIDCIssuerProfileStatus{ @@ -92,7 +142,7 @@ func TestReconcile(t *testing.T) { }, { name: "fail to get managed cluster credentials", - expectedError: "failed to get credentials for managed cluster: internal server error", + expectedError: "error while reconciling adminKubeConfigData: failed to get credentials for managed cluster: internal server error", expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { s.ManagedClusterSpec().Return(fakeManagedClusterSpec) r.CreateOrUpdateResource(gomockinternal.AContext(), fakeManagedClusterSpec, serviceName).Return(armcontainerservice.ManagedCluster{ @@ -105,6 +155,8 @@ func TestReconcile(t *testing.T) { Host: "my-managedcluster-fqdn", Port: 443, }) + s.IsAadEnabled().Return(false) + s.IsLocalAcountsDisabled().Return(false) m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte(""), errors.New("internal server error")) }, }, diff --git a/azure/services/managedclusters/mock_managedclusters/client_mock.go b/azure/services/managedclusters/mock_managedclusters/client_mock.go index 8104a3c47071..b9159c4494eb 100644 --- a/azure/services/managedclusters/mock_managedclusters/client_mock.go +++ b/azure/services/managedclusters/mock_managedclusters/client_mock.go @@ -64,3 +64,18 @@ func (mr *MockCredentialGetterMockRecorder) GetCredentials(arg0, arg1, arg2 inte mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCredentials", reflect.TypeOf((*MockCredentialGetter)(nil).GetCredentials), arg0, arg1, arg2) } + +// GetUserCredentials mocks base method. +func (m *MockCredentialGetter) GetUserCredentials(arg0 context.Context, arg1, arg2 string) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUserCredentials", arg0, arg1, arg2) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetUserCredentials indicates an expected call of GetUserCredentials. +func (mr *MockCredentialGetterMockRecorder) GetUserCredentials(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserCredentials", reflect.TypeOf((*MockCredentialGetter)(nil).GetUserCredentials), arg0, arg1, arg2) +} diff --git a/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go b/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go index e158d4732a1b..be92a31f4693 100644 --- a/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go +++ b/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go @@ -137,18 +137,18 @@ func (mr *MockManagedClusterScopeMockRecorder) DeleteLongRunningOperationState(a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLongRunningOperationState", reflect.TypeOf((*MockManagedClusterScope)(nil).DeleteLongRunningOperationState), arg0, arg1, arg2) } -// GetKubeConfigData mocks base method. -func (m *MockManagedClusterScope) GetKubeConfigData() []byte { +// GetAdminKubeConfigData mocks base method. +func (m *MockManagedClusterScope) GetAdminKubeConfigData() []byte { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetKubeConfigData") + ret := m.ctrl.Call(m, "GetAdminKubeConfigData") ret0, _ := ret[0].([]byte) return ret0 } -// GetKubeConfigData indicates an expected call of GetKubeConfigData. -func (mr *MockManagedClusterScopeMockRecorder) GetKubeConfigData() *gomock.Call { +// GetAdminKubeConfigData indicates an expected call of GetAdminKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) GetAdminKubeConfigData() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).GetKubeConfigData)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAdminKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).GetAdminKubeConfigData)) } // GetLongRunningOperationState mocks base method. @@ -165,6 +165,20 @@ func (mr *MockManagedClusterScopeMockRecorder) GetLongRunningOperationState(arg0 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLongRunningOperationState", reflect.TypeOf((*MockManagedClusterScope)(nil).GetLongRunningOperationState), arg0, arg1, arg2) } +// GetUserKubeConfigData mocks base method. +func (m *MockManagedClusterScope) GetUserKubeConfigData() []byte { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUserKubeConfigData") + ret0, _ := ret[0].([]byte) + return ret0 +} + +// GetUserKubeConfigData indicates an expected call of GetUserKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) GetUserKubeConfigData() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).GetUserKubeConfigData)) +} + // HashKey mocks base method. func (m *MockManagedClusterScope) HashKey() string { m.ctrl.T.Helper() @@ -179,6 +193,34 @@ func (mr *MockManagedClusterScopeMockRecorder) HashKey() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HashKey", reflect.TypeOf((*MockManagedClusterScope)(nil).HashKey)) } +// IsAadEnabled mocks base method. +func (m *MockManagedClusterScope) IsAadEnabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsAadEnabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsAadEnabled indicates an expected call of IsAadEnabled. +func (mr *MockManagedClusterScopeMockRecorder) IsAadEnabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAadEnabled", reflect.TypeOf((*MockManagedClusterScope)(nil).IsAadEnabled)) +} + +// IsLocalAcountsDisabled mocks base method. +func (m *MockManagedClusterScope) IsLocalAcountsDisabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsLocalAcountsDisabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsLocalAcountsDisabled indicates an expected call of IsLocalAcountsDisabled. +func (mr *MockManagedClusterScopeMockRecorder) IsLocalAcountsDisabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsLocalAcountsDisabled", reflect.TypeOf((*MockManagedClusterScope)(nil).IsLocalAcountsDisabled)) +} + // MakeEmptyKubeConfigSecret mocks base method. func (m *MockManagedClusterScope) MakeEmptyKubeConfigSecret() v1.Secret { m.ctrl.T.Helper() @@ -207,28 +249,28 @@ func (mr *MockManagedClusterScopeMockRecorder) ManagedClusterSpec() *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ManagedClusterSpec", reflect.TypeOf((*MockManagedClusterScope)(nil).ManagedClusterSpec)) } -// SetControlPlaneEndpoint mocks base method. -func (m *MockManagedClusterScope) SetControlPlaneEndpoint(arg0 v1beta10.APIEndpoint) { +// SetAdminKubeConfigData mocks base method. +func (m *MockManagedClusterScope) SetAdminKubeConfigData(arg0 []byte) { m.ctrl.T.Helper() - m.ctrl.Call(m, "SetControlPlaneEndpoint", arg0) + m.ctrl.Call(m, "SetAdminKubeConfigData", arg0) } -// SetControlPlaneEndpoint indicates an expected call of SetControlPlaneEndpoint. -func (mr *MockManagedClusterScopeMockRecorder) SetControlPlaneEndpoint(arg0 interface{}) *gomock.Call { +// SetAdminKubeConfigData indicates an expected call of SetAdminKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) SetAdminKubeConfigData(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetControlPlaneEndpoint", reflect.TypeOf((*MockManagedClusterScope)(nil).SetControlPlaneEndpoint), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetAdminKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).SetAdminKubeConfigData), arg0) } -// SetKubeConfigData mocks base method. -func (m *MockManagedClusterScope) SetKubeConfigData(arg0 []byte) { +// SetControlPlaneEndpoint mocks base method. +func (m *MockManagedClusterScope) SetControlPlaneEndpoint(arg0 v1beta10.APIEndpoint) { m.ctrl.T.Helper() - m.ctrl.Call(m, "SetKubeConfigData", arg0) + m.ctrl.Call(m, "SetControlPlaneEndpoint", arg0) } -// SetKubeConfigData indicates an expected call of SetKubeConfigData. -func (mr *MockManagedClusterScopeMockRecorder) SetKubeConfigData(arg0 interface{}) *gomock.Call { +// SetControlPlaneEndpoint indicates an expected call of SetControlPlaneEndpoint. +func (mr *MockManagedClusterScopeMockRecorder) SetControlPlaneEndpoint(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).SetKubeConfigData), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetControlPlaneEndpoint", reflect.TypeOf((*MockManagedClusterScope)(nil).SetControlPlaneEndpoint), arg0) } // SetKubeletIdentity mocks base method. @@ -267,6 +309,18 @@ func (mr *MockManagedClusterScopeMockRecorder) SetOIDCIssuerProfileStatus(arg0 i return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetOIDCIssuerProfileStatus", reflect.TypeOf((*MockManagedClusterScope)(nil).SetOIDCIssuerProfileStatus), arg0) } +// SetUserKubeConfigData mocks base method. +func (m *MockManagedClusterScope) SetUserKubeConfigData(arg0 []byte) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetUserKubeConfigData", arg0) +} + +// SetUserKubeConfigData indicates an expected call of SetUserKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) SetUserKubeConfigData(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUserKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).SetUserKubeConfigData), arg0) +} + // SubscriptionID mocks base method. func (m *MockManagedClusterScope) SubscriptionID() string { m.ctrl.T.Helper() diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index b75f0335a0d8..2919be38bdb8 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -122,6 +122,9 @@ type ManagedClusterSpec struct { // OIDCIssuerProfile is the OIDC issuer profile of the Managed Cluster. OIDCIssuerProfile *OIDCIssuerProfile + + // DisableLocalAccounts - If set to true, getting static credential will be disabled for this cluster. Expected to only be used for AAD clusters. + DisableLocalAccounts *bool } // HTTPProxyConfig is the HTTP proxy configuration for the cluster. @@ -387,6 +390,9 @@ func (s *ManagedClusterSpec) Parameters(ctx context.Context, existing interface{ EnableAzureRBAC: &s.AADProfile.EnableAzureRBAC, AdminGroupObjectIDs: azure.PtrSlice[string](&s.AADProfile.AdminGroupObjectIDs), } + if s.DisableLocalAccounts != nil { + managedCluster.Properties.DisableLocalAccounts = s.DisableLocalAccounts + } } for i := range s.AddonProfiles { @@ -737,6 +743,14 @@ func computeDiffOfNormalizedClusters(managedCluster armcontainerservice.ManagedC } } + if managedCluster.Properties.DisableLocalAccounts != nil { + clusterNormalized.Properties.DisableLocalAccounts = managedCluster.Properties.DisableLocalAccounts + } + + if existingMC.Properties.DisableLocalAccounts != nil { + existingMCClusterNormalized.Properties.DisableLocalAccounts = existingMC.Properties.DisableLocalAccounts + } + diff := cmp.Diff(clusterNormalized, existingMCClusterNormalized) return diff } diff --git a/azure/services/token/client.go b/azure/services/token/client.go new file mode 100644 index 000000000000..9b9d3f3787dd --- /dev/null +++ b/azure/services/token/client.go @@ -0,0 +1,81 @@ +package token + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + azureautorest "github.com/Azure/go-autorest/autorest/azure" + "github.com/pkg/errors" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" +) + +const ( + defaultEnvironmentName = "AzurePublicCloud" +) + +// AzureClient to get azure active directory token. +type AzureClient struct { + aadToken *azidentity.ClientSecretCredential +} + +// NewClient creates a new azure active directory token client from an authorizer. +func NewClient(auth azure.Authorizer) (*AzureClient, error) { + aadToken, err := newAzureActiveDirectoryTokenClient(auth.TenantID(), + auth.ClientID(), + auth.ClientSecret(), + auth.CloudEnvironment()) + if err != nil { + return nil, err + } + return &AzureClient{ + aadToken: aadToken, + }, nil +} + +// newAzureActiveDirectoryTokenClient creates a new aad token client from an authorizer. +func newAzureActiveDirectoryTokenClient(tenantID, clientID, clientSecret, envName string) (*azidentity.ClientSecretCredential, error) { + cliOpts, err := getAzureClientOptions(envName) + if err != nil { + return nil, errors.Wrap(err, "error while getting client options") + } + clientOptions := &azidentity.ClientSecretCredentialOptions{ + ClientOptions: cliOpts, + } + cred, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, clientOptions) + if err != nil { + return nil, errors.Wrap(err, "error while getting az client secret credentials") + } + return cred, nil +} + +func getAzureClientOptions(environment string) (azcore.ClientOptions, error) { + if environment == "" { + environment = defaultEnvironmentName + } + env, err := azureautorest.EnvironmentFromName(environment) + if err != nil { + return azcore.ClientOptions{}, errors.Wrap(err, "error while getting azure env") + } + c := cloud.Configuration{ + ActiveDirectoryAuthorityHost: env.ActiveDirectoryEndpoint, + } + return azcore.ClientOptions{ + Cloud: c, + }, nil +} + +// GetAzureActiveDirectoryToken gets the token for authentication with azure active directory. +func (ac *AzureClient) GetAzureActiveDirectoryToken(ctx context.Context, resourceID string) (string, error) { + ctx, _, done := tele.StartSpanWithLogger(ctx, "aadToken.GetToken") + defer done() + + spnAccessToken, err := ac.aadToken.GetToken(ctx, policy.TokenRequestOptions{Scopes: []string{resourceID + "/.default"}}) + if err != nil { + return "", errors.Wrap(err, "failed to get token") + } + return spnAccessToken.Token, nil +} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index 4d1bac26596f..a68f483a8ac4 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -239,6 +239,11 @@ spec: - host - port type: object + disableLocalAccounts: + description: DisableLocalAccounts - If set to true, getting static + credential will be disabled for this cluster. Expected to only be + used for AAD clusters. + type: boolean dnsServiceIP: description: DNSServiceIP is an IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range diff --git a/controllers/azuremanagedcontrolplane_reconciler.go b/controllers/azuremanagedcontrolplane_reconciler.go index 0314a8fbedb8..c9f523191b7a 100644 --- a/controllers/azuremanagedcontrolplane_reconciler.go +++ b/controllers/azuremanagedcontrolplane_reconciler.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "fmt" "github.com/pkg/errors" "sigs.k8s.io/cluster-api-provider-azure/azure" @@ -138,20 +139,25 @@ func (r *azureManagedControlPlaneService) reconcileKubeconfig(ctx context.Contex ctx, _, done := tele.StartSpanWithLogger(ctx, "controllers.azureManagedControlPlaneService.reconcileKubeconfig") defer done() - kubeConfigData := r.scope.GetKubeConfigData() - if kubeConfigData == nil { - return nil - } - kubeConfigSecret := r.scope.MakeEmptyKubeConfigSecret() + kubeConfigs := [][]byte{r.scope.GetAdminKubeConfigData(), r.scope.GetUserKubeConfigData()} - // Always update credentials in case of rotation - if _, err := controllerutil.CreateOrUpdate(ctx, r.kubeclient, &kubeConfigSecret, func() error { - kubeConfigSecret.Data = map[string][]byte{ - secret.KubeconfigDataName: kubeConfigData, + for i, kubeConfigData := range kubeConfigs { + if len(kubeConfigData) == 0 { + continue + } + kubeConfigSecret := r.scope.MakeEmptyKubeConfigSecret() + if i == 1 { + // 2nd kubeconfig is the user kubeconfig + kubeConfigSecret.Name = fmt.Sprintf("%s-user", kubeConfigSecret.Name) + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.kubeclient, &kubeConfigSecret, func() error { + kubeConfigSecret.Data = map[string][]byte{ + secret.KubeconfigDataName: kubeConfigData, + } + return nil + }); err != nil { + return errors.Wrap(err, "failed to kubeconfig secret for cluster") } - return nil - }); err != nil { - return errors.Wrap(err, "failed to kubeconfig secret for cluster") } return nil