From 9c4991046ae050de70c4643a0133fbcd209fbec9 Mon Sep 17 00:00:00 2001 From: willie-yao Date: Mon, 4 Dec 2023 22:48:19 +0000 Subject: [PATCH] Add support for AKS Marketplace extensions --- Makefile | 2 +- .../azuremanagedcontrolplane_default.go | 11 + api/v1beta1/azuremanagedcontrolplane_types.go | 46 + .../azuremanagedcontrolplane_webhook.go | 128 +++ .../azuremanagedcontrolplane_webhook_test.go | 182 ++++ ...zuremanagedcontrolplanetemplate_webhook.go | 6 + ...anagedcontrolplanetemplate_webhook_test.go | 62 ++ api/v1beta1/consts.go | 2 + api/v1beta1/types.go | 69 ++ api/v1beta1/types_class.go | 4 + api/v1beta1/zz_generated.deepcopy.go | 89 ++ azure/scope/managedcontrolplane.go | 34 + azure/scope/managedcontrolplane_test.go | 106 +++ azure/services/aksextensions/aksextensions.go | 50 + azure/services/aksextensions/spec.go | 110 +++ azure/services/aksextensions/spec_test.go | 155 ++++ config/aso/crds.yaml | 874 ++++++++++++++++++ ...er.x-k8s.io_azuremanagedcontrolplanes.yaml | 104 +++ ....io_azuremanagedcontrolplanetemplates.yaml | 109 +++ config/rbac/role.yaml | 20 + .../azuremanagedcontrolplane_controller.go | 2 + .../azuremanagedcontrolplane_reconciler.go | 2 + docs/book/src/topics/managedcluster.md | 26 + go.mod | 1 + go.sum | 2 + main.go | 2 + test/e2e/aks_marketplace.go | 179 ++++ test/e2e/azure_test.go | 9 + 28 files changed, 2385 insertions(+), 1 deletion(-) create mode 100644 azure/services/aksextensions/aksextensions.go create mode 100644 azure/services/aksextensions/spec.go create mode 100644 azure/services/aksextensions/spec_test.go create mode 100644 test/e2e/aks_marketplace.go diff --git a/Makefile b/Makefile index fb82b131340..78d206fe2cb 100644 --- a/Makefile +++ b/Makefile @@ -158,7 +158,7 @@ WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac ASO_CRDS_PATH := $(MANIFEST_ROOT)/aso/crds.yaml ASO_VERSION := v2.5.0 -ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com virtualnetworks.network.azure.com virtualnetworkssubnets.network.azure.com privateendpoints.network.azure.com fleetsmembers.containerservice.azure.com +ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com virtualnetworks.network.azure.com virtualnetworkssubnets.network.azure.com privateendpoints.network.azure.com fleetsmembers.containerservice.azure.com extensions.kubernetesconfiguration.azure.com # Allow overriding the imagePullPolicy PULL_POLICY ?= Always diff --git a/api/v1beta1/azuremanagedcontrolplane_default.go b/api/v1beta1/azuremanagedcontrolplane_default.go index 5f26197c08a..772a44ba62e 100644 --- a/api/v1beta1/azuremanagedcontrolplane_default.go +++ b/api/v1beta1/azuremanagedcontrolplane_default.go @@ -207,3 +207,14 @@ func (m *AzureManagedControlPlane) setDefaultDNSPrefix() { m.Spec.DNSPrefix = ptr.To(m.Name) } } + +func (m *AzureManagedControlPlane) setDefaultAKSExtensions() { + for _, extension := range m.Spec.Extensions { + if extension.Plan.Name == "" { + extension.Plan.Name = fmt.Sprintf("%s-%s", m.Name, extension.Plan.Product) + } + if extension.AutoUpgradeMinorVersion == nil { + extension.AutoUpgradeMinorVersion = ptr.To(true) + } + } +} diff --git a/api/v1beta1/azuremanagedcontrolplane_types.go b/api/v1beta1/azuremanagedcontrolplane_types.go index 5e274d40505..f6557cb58e5 100644 --- a/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/api/v1beta1/azuremanagedcontrolplane_types.go @@ -428,6 +428,52 @@ type OIDCIssuerProfile struct { Enabled *bool `json:"enabled,omitempty"` } +// AKSExtension represents the configuration for an AKS cluster extension. +// See also [AKS doc]. +// +// [AKS doc]: https://learn.microsoft.com/en-us/azure/aks/cluster-extensions +type AKSExtension struct { + // Name is the name of the extension. + Name string `json:"name"` + + // AKSAssignedIdentityType is the type of the AKS assigned identity. + // +optional + AKSAssignedIdentityType AKSAssignedIdentity `json:"aksAssignedIdentityType,omitempty"` + + // AutoUpgradeMinorVersion is a flag to note if this extension participates in auto upgrade of minor version, or not. + // +kubebuilder:default=true + // +optional + AutoUpgradeMinorVersion *bool `json:"autoUpgradeMinorVersion,omitempty"` + + // ConfigurationSettings are the name-value pairs for configuring this extension. + // +optional + ConfigurationSettings map[string]string `json:"configurationSettings,omitempty"` + + // ExtensionType is the type of the Extension of which this resource is an instance. + // It must be one of the Extension Types registered with Microsoft.KubernetesConfiguration by the Extension publisher. + ExtensionType *string `json:"extensionType"` + + // Plan is the plan of the extension. + Plan *ExtensionPlan `json:"plan"` + + // ReleaseTrain is the release train this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) + // This is only used if autoUpgradeMinorVersion is ‘true’. + // +optional + ReleaseTrain *string `json:"releaseTrain,omitempty"` + + // Scope is the scope at which this extension is enabled. + // +optional + Scope *ExtensionScope `json:"scope,omitempty"` + + // Version is the version of the extension. + // +optional + Version *string `json:"version,omitempty"` + + // Identity is the identity type of the Extension resource in an AKS cluster. + // +optional + Identity ExtensionIdentity `json:"identity,omitempty"` +} + // +kubebuilder:object:root=true // +kubebuilder:resource:path=azuremanagedcontrolplanes,scope=Namespaced,categories=cluster-api,shortName=amcp // +kubebuilder:storageversion diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook.go b/api/v1beta1/azuremanagedcontrolplane_webhook.go index 4c9da179082..2ac28c69973 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -95,6 +95,7 @@ func (mw *azureManagedControlPlaneWebhook) Default(ctx context.Context, obj runt m.setDefaultSubnet() m.setDefaultOIDCIssuerProfile() m.setDefaultDNSPrefix() + m.setDefaultAKSExtensions() return nil } @@ -265,6 +266,10 @@ func (mw *azureManagedControlPlaneWebhook) ValidateUpdate(ctx context.Context, o allErrs = append(allErrs, errs...) } + if errs := validateAKSExtensionsUpdate(old.Spec.Extensions, m.Spec.Extensions); len(errs) > 0 { + allErrs = append(allErrs, errs...) + } + if len(allErrs) == 0 { return nil, m.Validate(mw.Client) } @@ -314,6 +319,8 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { allErrs = append(allErrs, validateAutoScalerProfile(m.Spec.AutoScalerProfile, field.NewPath("spec").Child("AutoScalerProfile"))...) + allErrs = append(allErrs, validateAKSExtensions(m.Spec.Extensions, field.NewPath("spec").Child("AKSExtensions"))...) + return allErrs.ToAggregate() } @@ -711,6 +718,89 @@ func (m *AzureManagedControlPlane) validateFleetsMember(old *AzureManagedControl return allErrs } +// validateAKSExtensionsUpdate validates update to AKS extensions. +func validateAKSExtensionsUpdate(old []AKSExtension, current []AKSExtension) field.ErrorList { + var allErrs field.ErrorList + + oldAKSExtensionsMap := make(map[string]AKSExtension, len(old)) + oldAKSExtensionsIndex := make(map[string]int, len(old)) + for i, extension := range old { + oldAKSExtensionsMap[extension.Name] = extension + oldAKSExtensionsIndex[extension.Name] = i + } + for i, extension := range current { + oldExtension, ok := oldAKSExtensionsMap[extension.Name] + if !ok { + continue + } + if extension.Name != oldExtension.Name { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Extensions", fmt.Sprintf("[%d]", i), "Name"), + extension.Name, + "field is immutable", + ), + ) + } + if (oldExtension.ExtensionType != nil && extension.ExtensionType != nil) && *extension.ExtensionType != *oldExtension.ExtensionType { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Extensions", fmt.Sprintf("[%d]", i), "ExtensionType"), + extension.ExtensionType, + "field is immutable", + ), + ) + } + if (extension.Plan != nil && oldExtension.Plan != nil) && *extension.Plan != *oldExtension.Plan { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Extensions", fmt.Sprintf("[%d]", i), "Plan"), + extension.Plan, + "field is immutable", + ), + ) + } + if extension.Scope != oldExtension.Scope { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Extensions", fmt.Sprintf("[%d]", i), "Scope"), + extension.Scope, + "field is immutable", + ), + ) + } + if (extension.ReleaseTrain != nil && oldExtension.ReleaseTrain != nil) && *extension.ReleaseTrain != *oldExtension.ReleaseTrain { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Extensions", fmt.Sprintf("[%d]", i), "ReleaseTrain"), + extension.ReleaseTrain, + "field is immutable", + ), + ) + } + if (extension.Version != nil && oldExtension.Version != nil) && *extension.Version != *oldExtension.Version { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Extensions", fmt.Sprintf("[%d]", i), "Version"), + extension.Version, + "field is immutable", + ), + ) + } + if extension.Identity != oldExtension.Identity { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Extensions", fmt.Sprintf("[%d]", i), "Identity"), + extension.Identity, + "field is immutable", + ), + ) + } + } + + return allErrs +} + func validateName(name string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList if lName := strings.ToLower(name); strings.Contains(lName, "microsoft") || @@ -722,6 +812,44 @@ func validateName(name string, fldPath *field.Path) field.ErrorList { return allErrs } +// validateAKSExtensions validates the AKS extensions. +func validateAKSExtensions(extensions []AKSExtension, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + for _, extension := range extensions { + if extension.Version != nil && (extension.AutoUpgradeMinorVersion == nil || (extension.AutoUpgradeMinorVersion != nil && *extension.AutoUpgradeMinorVersion)) { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("Version"), "Version must not be given if AutoUpgradeMinorVersion is true (or not provided, as it is true by default)")) + } + if extension.Plan.Product == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("Plan", "Product"), "Product must be provided")) + } + if extension.Plan.Publisher == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("Plan", "Publisher"), "Publisher must be provided")) + } + if extension.AutoUpgradeMinorVersion == ptr.To(false) && extension.ReleaseTrain != nil { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("ReleaseTrain"), "ReleaseTrain must not be given if AutoUpgradeMinorVersion is false")) + } + if extension.Scope != nil { + if extension.Scope.ScopeType == ExtensionScopeCluster { + if extension.Scope.ReleaseNamespace == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("Scope", "ReleaseNamespace"), "ReleaseNamespace must be provided if Scope is Cluster")) + } + if extension.Scope.TargetNamespace != "" { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("Scope", "TargetNamespace"), "TargetNamespace can only be given if Scope is Namespace")) + } + } else if extension.Scope.ScopeType == ExtensionScopeNamespace { + if extension.Scope.TargetNamespace == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("Scope", "TargetNamespace"), "TargetNamespace must be provided if Scope is Namespace")) + } + if extension.Scope.ReleaseNamespace != "" { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("Scope", "ReleaseNamespace"), "ReleaseNamespace can only be given if Scope is Cluster")) + } + } + } + } + + return allErrs +} + // validateAutoScalerProfile validates an AutoScalerProfile. func validateAutoScalerProfile(autoScalerProfile *AutoScalerProfile, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index bca8dd29efd..467ae0295b6 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -45,6 +45,15 @@ func TestDefaultingWebhook(t *testing.T) { AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ Location: "fooLocation", Version: "1.17.5", + Extensions: []AKSExtension{ + { + Name: "test-extension", + Plan: &ExtensionPlan{ + Product: "test-product", + Publisher: "test-publisher", + }, + }, + }, }, ResourceGroupName: "fooRg", SSHPublicKey: ptr.To(""), @@ -67,6 +76,7 @@ func TestDefaultingWebhook(t *testing.T) { g.Expect(*amcp.Spec.OIDCIssuerProfile.Enabled).To(BeFalse()) g.Expect(amcp.Spec.DNSPrefix).ToNot(BeNil()) g.Expect(*amcp.Spec.DNSPrefix).To(Equal(amcp.Name)) + g.Expect(amcp.Spec.Extensions[0].Plan.Name).To(Equal("fooName-test-product")) t.Logf("Testing amcp defaulting webhook with baseline") netPlug := "kubenet" @@ -1114,6 +1124,77 @@ func TestValidatingWebhook(t *testing.T) { }, expectErr: false, }, + { + name: "Testing valid AKS Extension", + amcp: AzureManagedControlPlane{ + ObjectMeta: getAMCPMetaData(), + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + Extensions: []AKSExtension{ + { + Name: "extension1", + ExtensionType: ptr.To("test-type"), + Plan: &ExtensionPlan{ + Name: "test-plan", + Product: "test-product", + Publisher: "test-publisher", + }, + }, + }, + }, + }, + }, + expectErr: false, + }, + { + name: "Testing invalid AKS Extension: version given when AutoUpgradeMinorVersion is true", + amcp: AzureManagedControlPlane{ + ObjectMeta: getAMCPMetaData(), + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + Extensions: []AKSExtension{ + { + Name: "extension1", + ExtensionType: ptr.To("test-type"), + Version: ptr.To("1.0.0"), + AutoUpgradeMinorVersion: ptr.To(true), + Plan: &ExtensionPlan{ + Name: "test-plan", + Product: "test-product", + Publisher: "test-publisher", + }, + }, + }, + }, + }, + }, + expectErr: true, + }, + { + name: "Testing invalid AKS Extension: missing plan.product and plan.publisher", + amcp: AzureManagedControlPlane{ + ObjectMeta: getAMCPMetaData(), + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + Extensions: []AKSExtension{ + { + Name: "extension1", + ExtensionType: ptr.To("test-type"), + Version: ptr.To("1.0.0"), + AutoUpgradeMinorVersion: ptr.To(true), + Plan: &ExtensionPlan{ + Name: "test-plan", + }, + }, + }, + }, + }, + }, + expectErr: true, + }, } for _, tt := range tests { @@ -2653,6 +2734,107 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: false, }, + { + name: "AzureManagedControlPlane AKSExtensions ConfigurationSettings and AutoUpgradeMinorVersion are mutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + Extensions: []AKSExtension{ + { + Name: "extension1", + AutoUpgradeMinorVersion: ptr.To(false), + ConfigurationSettings: map[string]string{ + "key1": "value1", + }, + Plan: &ExtensionPlan{ + Name: "planName", + Product: "planProduct", + Publisher: "planPublisher", + }, + }, + }, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + Extensions: []AKSExtension{ + { + Name: "extension1", + AutoUpgradeMinorVersion: ptr.To(true), + ConfigurationSettings: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + Plan: &ExtensionPlan{ + Name: "planName", + Product: "planProduct", + Publisher: "planPublisher", + }, + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "AzureManagedControlPlane all other fields are immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + Extensions: []AKSExtension{ + { + Name: "extension1", + AKSAssignedIdentityType: AKSAssignedIdentitySystemAssigned, + ExtensionType: ptr.To("extensionType"), + Plan: &ExtensionPlan{ + Name: "planName", + Product: "planProduct", + Publisher: "planPublisher", + }, + Scope: &ExtensionScope{ + ScopeType: "Cluster", + ReleaseNamespace: "default", + }, + ReleaseTrain: ptr.To("releaseTrain"), + Version: ptr.To("v1.0.0"), + }, + }, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + Extensions: []AKSExtension{ + { + Name: "extension2", + AKSAssignedIdentityType: AKSAssignedIdentityUserAssigned, + ExtensionType: ptr.To("extensionType1"), + Plan: &ExtensionPlan{ + Name: "planName1", + Product: "planProduct1", + Publisher: "planPublisher1", + }, + Scope: &ExtensionScope{ + ScopeType: "Namespace", + ReleaseNamespace: "default", + }, + ReleaseTrain: ptr.To("releaseTrain1"), + Version: ptr.To("v1.1.0"), + }, + }, + }, + }, + }, + wantErr: true, + }, } client := mockClient{ReturnError: false} for _, tc := range tests { diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go index 245d18113e1..c1e1dab795b 100644 --- a/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go @@ -172,6 +172,10 @@ func (mcpw *azureManagedControlPlaneTemplateWebhook) ValidateUpdate(ctx context. allErrs = append(allErrs, errs...) } + if errs := validateAKSExtensionsUpdate(old.Spec.Template.Spec.Extensions, mcp.Spec.Template.Spec.Extensions); len(errs) > 0 { + allErrs = append(allErrs, errs...) + } + if len(allErrs) == 0 { return nil, mcp.validateManagedControlPlaneTemplate(mcpw.Client) } @@ -203,6 +207,8 @@ func (mcp *AzureManagedControlPlaneTemplate) validateManagedControlPlaneTemplate allErrs = append(allErrs, validateAutoScalerProfile(mcp.Spec.Template.Spec.AutoScalerProfile, field.NewPath("spec").Child("template").Child("spec").Child("AutoScalerProfile"))...) + allErrs = append(allErrs, validateAKSExtensions(mcp.Spec.Template.Spec.Extensions, field.NewPath("spec").Child("Extensions"))...) + return allErrs.ToAggregate() } diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go index 326d9c40eeb..daca2a38751 100644 --- a/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go @@ -186,6 +186,68 @@ func TestControlPlaneTemplateUpdateWebhook(t *testing.T) { }), wantErr: true, }, + { + name: "azuremanagedcontrolplanetemplate AKSExtension type and plan are immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.Extensions = []AKSExtension{ + { + Name: "foo", + ExtensionType: ptr.To("foo-type"), + Plan: &ExtensionPlan{ + Name: "foo-name", + Product: "foo-product", + Publisher: "foo-publisher", + }, + }, + } + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.Extensions = []AKSExtension{ + { + Name: "foo", + ExtensionType: ptr.To("bar"), + Plan: &ExtensionPlan{ + Name: "bar-name", + Product: "bar-product", + Publisher: "bar-publisher", + }, + }, + } + }), + wantErr: true, + }, + { + name: "azuremanagedcontrolplanetemplate AKSExtension autoUpgradeMinorVersion is mutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.Extensions = []AKSExtension{ + { + Name: "foo", + ExtensionType: ptr.To("foo"), + AutoUpgradeMinorVersion: ptr.To(true), + Plan: &ExtensionPlan{ + Name: "bar-name", + Product: "bar-product", + Publisher: "bar-publisher", + }, + }, + } + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.Extensions = []AKSExtension{ + { + Name: "foo", + ExtensionType: ptr.To("foo"), + AutoUpgradeMinorVersion: ptr.To(false), + Plan: &ExtensionPlan{ + Name: "bar-name", + Product: "bar-product", + Publisher: "bar-publisher", + }, + }, + } + }), + wantErr: false, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/api/v1beta1/consts.go b/api/v1beta1/consts.go index 75e9d422f8f..dbdb43f99b9 100644 --- a/api/v1beta1/consts.go +++ b/api/v1beta1/consts.go @@ -132,6 +132,8 @@ const ( PrivateEndpointsReadyCondition clusterv1.ConditionType = "PrivateEndpointsReady" // FleetReadyCondition means the Fleet exists and is ready to be used. FleetReadyCondition clusterv1.ConditionType = "FleetReady" + // AKSExtensionsReadyCondition means the AKS Extensions exist and are ready to be used. + AKSExtensionsReadyCondition clusterv1.ConditionType = "AKSExtensionsReady" // CreatingReason means the resource is being created. CreatingReason = "Creating" diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 1314d3bc6cf..1f7e2d1f4b8 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -1095,3 +1095,72 @@ const ( // UniformOrchestrationMode treats VMs as identical instances accessible by the VMSS VM API. UniformOrchestrationMode OrchestrationModeType = "Uniform" ) + +// ExtensionPlan represents the plan for an AKS marketplace extension. +type ExtensionPlan struct { + // Name is the user-defined name of the 3rd Party Artifact that is being procured. + // +optional + Name string `json:"name,omitempty"` + + // Product is the name of the 3rd Party artifact that is being procured. + Product string `json:"product"` + + // PromotionCode is a publisher-provided promotion code as provisioned in Data Market for the said product/artifact. + // +optional + PromotionCode string `json:"promotionCode,omitempty"` + + // Publisher is the name of the publisher of the 3rd Party Artifact that is being bought. + Publisher string `json:"publisher"` + + // Version is the version of the plan. + // +optional + Version string `json:"version,omitempty"` +} + +// ExtensionScope defines the scope of the AKS marketplace extension, if configured. +type ExtensionScope struct { + // ScopeType is the scope of the extension. It can be either Cluster or Namespace, but not both. + ScopeType ExtensionScopeType `json:"scopeType"` + + // ReleaseNamespace is the namespace where the extension Release must be placed, for a Cluster-scoped extension. + // Required for Cluster-scoped extensions. + // +optional + ReleaseNamespace string `json:"releaseNamespace,omitempty"` + + // TargetNamespace is the namespace where the extension will be created for a Namespace-scoped extension. + // Required for Namespace-scoped extensions. + // +optional + TargetNamespace string `json:"targetNamespace,omitempty"` +} + +// ExtensionScopeType defines the scope type of the AKS marketplace extension, if configured. +// +kubebuilder:validation:Enum=Cluster;Namespace +type ExtensionScopeType string + +const ( + // ExtensionScopeCluster ... + ExtensionScopeCluster ExtensionScopeType = "Cluster" + // ExtensionScopeNamespace ... + ExtensionScopeNamespace ExtensionScopeType = "Namespace" +) + +// ExtensionIdentity defines the identity of the AKS marketplace extension, if configured. +// +kubebuilder:validation:Enum=SystemAssigned +type ExtensionIdentity string + +const ( + // ExtensionIdentitySystemAssigned ... + ExtensionIdentitySystemAssigned ExtensionIdentity = "SystemAssigned" +) + +// AKSAssignedIdentity defines the AKS assigned-identity of the aks marketplace extension, if configured. +// +kubebuilder:validation:Enum=SystemAssigned;UserAssigned +type AKSAssignedIdentity string + +const ( + // AKSAssignedIdentitySystemAssigned ... + AKSAssignedIdentitySystemAssigned AKSAssignedIdentity = "SystemAssigned" + + // AKSAssignedIdentityUserAssigned ... + AKSAssignedIdentityUserAssigned AKSAssignedIdentity = "UserAssigned" +) diff --git a/api/v1beta1/types_class.go b/api/v1beta1/types_class.go index 668dbdea3ad..79df3ba5e8d 100644 --- a/api/v1beta1/types_class.go +++ b/api/v1beta1/types_class.go @@ -218,6 +218,10 @@ type AzureManagedControlPlaneClassSpec struct { // [AKS doc]: https://learn.microsoft.com/en-us/azure/templates/microsoft.containerservice/2023-03-15-preview/fleets/members // +optional FleetsMember *FleetsMemberClassSpec `json:"fleetsMember,omitempty"` + + // Extensions is a list of AKS extensions to be installed on the cluster. + // +optional + Extensions []AKSExtension `json:"extensions,omitempty"` } // AzureManagedMachinePoolClassSpec defines the AzureManagedMachinePool properties that may be shared across several Azure managed machinepools. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 656d54f2338..60fd8adcfbe 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -48,6 +48,58 @@ func (in *AADProfile) DeepCopy() *AADProfile { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AKSExtension) DeepCopyInto(out *AKSExtension) { + *out = *in + if in.AutoUpgradeMinorVersion != nil { + in, out := &in.AutoUpgradeMinorVersion, &out.AutoUpgradeMinorVersion + *out = new(bool) + **out = **in + } + if in.ConfigurationSettings != nil { + in, out := &in.ConfigurationSettings, &out.ConfigurationSettings + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ExtensionType != nil { + in, out := &in.ExtensionType, &out.ExtensionType + *out = new(string) + **out = **in + } + if in.Plan != nil { + in, out := &in.Plan, &out.Plan + *out = new(ExtensionPlan) + **out = **in + } + if in.ReleaseTrain != nil { + in, out := &in.ReleaseTrain, &out.ReleaseTrain + *out = new(string) + **out = **in + } + if in.Scope != nil { + in, out := &in.Scope, &out.Scope + *out = new(ExtensionScope) + **out = **in + } + if in.Version != nil { + in, out := &in.Version, &out.Version + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AKSExtension. +func (in *AKSExtension) DeepCopy() *AKSExtension { + if in == nil { + return nil + } + out := new(AKSExtension) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AKSSku) DeepCopyInto(out *AKSSku) { *out = *in @@ -1351,6 +1403,13 @@ func (in *AzureManagedControlPlaneClassSpec) DeepCopyInto(out *AzureManagedContr *out = new(FleetsMemberClassSpec) **out = **in } + if in.Extensions != nil { + in, out := &in.Extensions, &out.Extensions + *out = make([]AKSExtension, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneClassSpec. @@ -2223,6 +2282,36 @@ func (in *ExtendedLocationSpec) DeepCopy() *ExtendedLocationSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExtensionPlan) DeepCopyInto(out *ExtensionPlan) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtensionPlan. +func (in *ExtensionPlan) DeepCopy() *ExtensionPlan { + if in == nil { + return nil + } + out := new(ExtensionPlan) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExtensionScope) DeepCopyInto(out *ExtensionScope) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtensionScope. +func (in *ExtensionScope) DeepCopy() *ExtensionScope { + if in == nil { + return nil + } + out := new(ExtensionScope) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FleetsMember) DeepCopyInto(out *FleetsMember) { *out = *in diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index bbf7bca5c94..2c4f6a6d9e6 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -25,6 +25,7 @@ import ( asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview" asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001" + asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501" asonetworkv1api20201101 "github.com/Azure/azure-service-operator/v2/api/network/v1api20201101" asonetworkv1api20220701 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601" @@ -39,6 +40,7 @@ import ( "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/aksextensions" "sigs.k8s.io/cluster-api-provider-azure/azure/services/fleetsmembers" "sigs.k8s.io/cluster-api-provider-azure/azure/services/groups" "sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters" @@ -912,3 +914,35 @@ func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ASOResourceSpe func (s *ManagedControlPlaneScope) SetOIDCIssuerProfileStatus(oidc *infrav1.OIDCIssuerProfileStatus) { s.ControlPlane.Status.OIDCIssuerProfile = oidc } + +// AKSExtension returns the cluster AKS extensions. +func (s *ManagedControlPlaneScope) AKSExtension() []infrav1.AKSExtension { + return s.ControlPlane.Spec.Extensions +} + +// AKSExtensionSpecs returns the AKS extension specs. +func (s *ManagedControlPlaneScope) AKSExtensionSpecs() []azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension] { + if s.AKSExtension() == nil { + return nil + } + extensionSpecs := make([]azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension], 0, len(s.ControlPlane.Spec.Extensions)) + for _, extension := range s.AKSExtension() { + extensionSpec := &aksextensions.AKSExtensionSpec{ + Name: extension.Name, + Namespace: s.Cluster.Namespace, + AutoUpgradeMinorVersion: extension.AutoUpgradeMinorVersion, + ConfigurationSettings: extension.ConfigurationSettings, + ExtensionType: extension.ExtensionType, + ReleaseTrain: extension.ReleaseTrain, + Version: extension.Version, + Owner: azure.ManagedClusterID(s.SubscriptionID(), s.ResourceGroup(), s.ControlPlane.Name), + Plan: *extension.Plan, + AKSAssignedIdentityType: extension.AKSAssignedIdentityType, + ExtensionIdentity: extension.Identity, + } + + extensionSpecs = append(extensionSpecs, extensionSpec) + } + + return extensionSpecs +} diff --git a/azure/scope/managedcontrolplane_test.go b/azure/scope/managedcontrolplane_test.go index 8ce0106572f..4feb4005125 100644 --- a/azure/scope/managedcontrolplane_test.go +++ b/azure/scope/managedcontrolplane_test.go @@ -22,6 +22,7 @@ import ( "testing" asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001" + asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501" asonetworkv1 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -31,6 +32,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/agentpools" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/aksextensions" "sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters" "sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -1433,3 +1435,107 @@ func TestManagedControlPlaneScope_PrivateEndpointSpecs(t *testing.T) { }) } } + +func TestManagedControlPlaneScope_AKSExtensionSpecs(t *testing.T) { + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected []azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension] + Err string + }{ + { + Name: "returns empty AKS extensions list if no extensions are specified", + Input: ManagedControlPlaneScopeParams{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "dummy-ns", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: "dummy-ns", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{}, + }, + }, + }, + }, + { + Name: "returns list of AKS extensions if extensions are specified", + Input: ManagedControlPlaneScopeParams{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: "dummy-ns", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: "dummy-ns", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + Extensions: []infrav1.AKSExtension{ + { + Name: "my-extension", + AutoUpgradeMinorVersion: ptr.To(true), + ConfigurationSettings: map[string]string{ + "my-key": "my-value", + }, + ExtensionType: ptr.To("my-extension-type"), + ReleaseTrain: ptr.To("my-release-train"), + Version: ptr.To("my-version"), + Plan: &infrav1.ExtensionPlan{ + Name: "my-plan-name", + Product: "my-product", + Publisher: "my-publisher", + }, + AKSAssignedIdentityType: infrav1.AKSAssignedIdentitySystemAssigned, + Identity: infrav1.ExtensionIdentitySystemAssigned, + }, + }, + }, + }, + }, + }, + Expected: []azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension]{ + &aksextensions.AKSExtensionSpec{ + Name: "my-extension", + Namespace: "dummy-ns", + AutoUpgradeMinorVersion: ptr.To(true), + ConfigurationSettings: map[string]string{ + "my-key": "my-value", + }, + ExtensionType: ptr.To("my-extension-type"), + ReleaseTrain: ptr.To("my-release-train"), + Version: ptr.To("my-version"), + Owner: "/subscriptions//resourceGroups//providers/Microsoft.ContainerService/managedClusters/my-cluster", + Plan: infrav1.ExtensionPlan{ + Name: "my-plan-name", + Product: "my-product", + Publisher: "my-publisher", + }, + AKSAssignedIdentityType: infrav1.AKSAssignedIdentitySystemAssigned, + ExtensionIdentity: infrav1.ExtensionIdentitySystemAssigned, + }, + }, + }, + } + + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + s := &ManagedControlPlaneScope{ + ControlPlane: c.Input.ControlPlane, + Cluster: c.Input.Cluster, + } + if got := s.AKSExtensionSpecs(); !reflect.DeepEqual(got, c.Expected) { + t.Errorf("AKSExtensionSpecs() = %s, want %s", specArrayToString(got), specArrayToString(c.Expected)) + } + }) + } +} diff --git a/azure/services/aksextensions/aksextensions.go b/azure/services/aksextensions/aksextensions.go new file mode 100644 index 00000000000..9bff9a082ba --- /dev/null +++ b/azure/services/aksextensions/aksextensions.go @@ -0,0 +1,50 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aksextensions + +import ( + asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/aso" +) + +const serviceName = "extension" + +// AKSExtensionScope defines the scope interface for an AKS extensions service. +type AKSExtensionScope interface { + azure.ClusterScoper + aso.Scope + AKSExtensionSpecs() []azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension] +} + +// Service provides operations on Azure resources. +type Service struct { + Scope AKSExtensionScope + *aso.Service[*asokubernetesconfigurationv1.Extension, AKSExtensionScope] +} + +// New creates a new service. +func New(scope AKSExtensionScope) *Service { + svc := aso.NewService[*asokubernetesconfigurationv1.Extension, AKSExtensionScope](serviceName, scope) + svc.Specs = scope.AKSExtensionSpecs() + svc.ConditionType = infrav1.AKSExtensionsReadyCondition + return &Service{ + Scope: scope, + Service: svc, + } +} diff --git a/azure/services/aksextensions/spec.go b/azure/services/aksextensions/spec.go new file mode 100644 index 00000000000..1a47c97c540 --- /dev/null +++ b/azure/services/aksextensions/spec.go @@ -0,0 +1,110 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aksextensions + +import ( + "context" + + asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501" + "github.com/Azure/azure-service-operator/v2/pkg/genruntime" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" +) + +// AKSExtensionSpec defines the specification for an AKS Extension. +type AKSExtensionSpec struct { + Name string + Namespace string + AKSAssignedIdentityType infrav1.AKSAssignedIdentity + ExtensionIdentity infrav1.ExtensionIdentity + AutoUpgradeMinorVersion *bool + ConfigurationSettings map[string]string + ExtensionType *string + ReleaseTrain *string + Version *string + Owner string + OwnerRef metav1.OwnerReference + Plan infrav1.ExtensionPlan + Scope infrav1.ExtensionScope +} + +// ResourceRef implements azure.ASOResourceSpecGetter. +func (s *AKSExtensionSpec) ResourceRef() *asokubernetesconfigurationv1.Extension { + return &asokubernetesconfigurationv1.Extension{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.Name, + Namespace: s.Namespace, + }, + } +} + +// Parameters implements azure.ASOResourceSpecGetter. +func (s *AKSExtensionSpec) Parameters(ctx context.Context, existingAKSExtension *asokubernetesconfigurationv1.Extension) (parameters *asokubernetesconfigurationv1.Extension, err error) { + aksExtension := &asokubernetesconfigurationv1.Extension{} + if existingAKSExtension != nil { + aksExtension = existingAKSExtension + } + + aksExtension.Spec = asokubernetesconfigurationv1.Extension_Spec{} + aksExtension.Spec.AzureName = s.Name + aksExtension.Spec.AutoUpgradeMinorVersion = s.AutoUpgradeMinorVersion + aksExtension.Spec.ConfigurationSettings = s.ConfigurationSettings + aksExtension.Spec.ExtensionType = s.ExtensionType + aksExtension.Spec.ReleaseTrain = s.ReleaseTrain + aksExtension.Spec.Version = s.Version + aksExtension.Spec.Owner = &genruntime.ArbitraryOwnerReference{ + ARMID: s.Owner, + } + aksExtension.Spec.Plan = &asokubernetesconfigurationv1.Plan{ + Name: ptr.To(s.Plan.Name), + Product: ptr.To(s.Plan.Product), + Publisher: ptr.To(s.Plan.Publisher), + Version: ptr.To(s.Plan.Version), + } + if s.ExtensionIdentity != "" { + aksExtension.Spec.Identity = &asokubernetesconfigurationv1.Identity{ + Type: (*asokubernetesconfigurationv1.Identity_Type)(ptr.To(s.ExtensionIdentity)), + } + } + if s.AKSAssignedIdentityType != "" { + aksExtension.Spec.AksAssignedIdentity = &asokubernetesconfigurationv1.Extension_Properties_AksAssignedIdentity_Spec{ + Type: (*asokubernetesconfigurationv1.Extension_Properties_AksAssignedIdentity_Type_Spec)(ptr.To(s.AKSAssignedIdentityType)), + } + } + if s.Scope.ScopeType == infrav1.ExtensionScopeCluster { + aksExtension.Spec.Scope = &asokubernetesconfigurationv1.Scope{ + Cluster: &asokubernetesconfigurationv1.ScopeCluster{ + ReleaseNamespace: ptr.To(s.Scope.ReleaseNamespace), + }, + } + } else if s.Scope.ScopeType == infrav1.ExtensionScopeNamespace { + aksExtension.Spec.Scope = &asokubernetesconfigurationv1.Scope{ + Namespace: &asokubernetesconfigurationv1.ScopeNamespace{ + TargetNamespace: ptr.To(s.Scope.TargetNamespace), + }, + } + } + + return aksExtension, nil +} + +// WasManaged implements azure.ASOResourceSpecGetter. +func (s *AKSExtensionSpec) WasManaged(resource *asokubernetesconfigurationv1.Extension) bool { + // returns always returns true as CAPZ does not support BYO extension. + return true +} diff --git a/azure/services/aksextensions/spec_test.go b/azure/services/aksextensions/spec_test.go new file mode 100644 index 00000000000..2672d59ae18 --- /dev/null +++ b/azure/services/aksextensions/spec_test.go @@ -0,0 +1,155 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aksextensions + +import ( + "context" + "testing" + + asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501" + "github.com/Azure/azure-service-operator/v2/pkg/genruntime" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" +) + +var ( + fakeAKSExtension = asokubernetesconfigurationv1.Extension{ + Spec: asokubernetesconfigurationv1.Extension_Spec{ + AksAssignedIdentity: &asokubernetesconfigurationv1.Extension_Properties_AksAssignedIdentity_Spec{ + Type: (*asokubernetesconfigurationv1.Extension_Properties_AksAssignedIdentity_Type_Spec)(&fakeAKSExtensionSpec1.AKSAssignedIdentityType), + }, + AzureName: fakeAKSExtensionSpec1.Name, + AutoUpgradeMinorVersion: ptr.To(true), + ConfigurationSettings: map[string]string{ + "fake-key": "fake-value", + }, + ExtensionType: fakeAKSExtensionSpec1.ExtensionType, + Owner: &genruntime.ArbitraryOwnerReference{ + ARMID: fakeAKSExtensionSpec1.Owner, + }, + Plan: &asokubernetesconfigurationv1.Plan{ + Name: ptr.To(fakeAKSExtensionSpec1.Plan.Name), + Product: ptr.To(fakeAKSExtensionSpec1.Plan.Product), + Publisher: ptr.To(fakeAKSExtensionSpec1.Plan.Publisher), + Version: ptr.To(fakeAKSExtensionSpec1.Plan.Version), + }, + ReleaseTrain: fakeAKSExtensionSpec1.ReleaseTrain, + Version: fakeAKSExtensionSpec1.Version, + Identity: &asokubernetesconfigurationv1.Identity{ + Type: (*asokubernetesconfigurationv1.Identity_Type)(&fakeAKSExtensionSpec1.ExtensionIdentity), + }, + }, + } + fakeAKSExtensionSpec1 = AKSExtensionSpec{ + Name: "fake-aks-extension", + Namespace: "fake-namespace", + AKSAssignedIdentityType: "SystemAssigned", + AutoUpgradeMinorVersion: ptr.To(true), + ConfigurationSettings: map[string]string{ + "fake-key": "fake-value", + }, + ExtensionType: ptr.To("fake-extension-type"), + ReleaseTrain: ptr.To("fake-release-train"), + Version: ptr.To("fake-version"), + Owner: "fake-owner", + Plan: infrav1.ExtensionPlan{ + Name: "fake-plan-name", + }, + ExtensionIdentity: "SystemAssigned", + } + fakeAKSExtensionStatus = asokubernetesconfigurationv1.Extension_STATUS{ + Name: ptr.To(fakeAKSExtensionSpec1.Name), + ProvisioningState: ptr.To(asokubernetesconfigurationv1.ProvisioningStateDefinition_STATUS_Succeeded), + } +) + +func getASOAKSExtension(changes ...func(*asokubernetesconfigurationv1.Extension)) *asokubernetesconfigurationv1.Extension { + aksExtension := fakeAKSExtension.DeepCopy() + for _, change := range changes { + change(aksExtension) + } + return aksExtension +} + +func TestAzureAKSExtensionSpec_Parameters(t *testing.T) { + testcases := []struct { + name string + spec *AKSExtensionSpec + existing *asokubernetesconfigurationv1.Extension + expect func(g *WithT, result asokubernetesconfigurationv1.Extension) + expectedError string + }{ + { + name: "Creating a new AKS Extension", + spec: &fakeAKSExtensionSpec1, + existing: nil, + expect: func(g *WithT, result asokubernetesconfigurationv1.Extension) { + g.Expect(result).To(Not(BeNil())) + + // ObjectMeta is populated later in the codeflow + g.Expect(result.ObjectMeta).To(Equal(metav1.ObjectMeta{})) + + // Spec is populated from the spec passed in + g.Expect(result.Spec).To(Equal(getASOAKSExtension().Spec)) + }, + }, + { + name: "user updates to AKS Extension resource and capz should overwrite it", + spec: &fakeAKSExtensionSpec1, + existing: getASOAKSExtension( + // user added AutoUpgradeMinorVersion which should be overwritten by capz + func(aksExtension *asokubernetesconfigurationv1.Extension) { + aksExtension.Spec.AutoUpgradeMinorVersion = ptr.To(false) + }, + // user added Status + func(aksExtension *asokubernetesconfigurationv1.Extension) { + aksExtension.Status = fakeAKSExtensionStatus + }, + ), + expect: func(g *WithT, result asokubernetesconfigurationv1.Extension) { + g.Expect(result).To(Not(BeNil())) + + // ObjectMeta is populated later in the codeflow + g.Expect(result.ObjectMeta).To(Equal(metav1.ObjectMeta{})) + + // Spec is populated from the spec passed in + g.Expect(result.Spec).To(Equal(getASOAKSExtension().Spec)) + + // Status should be carried over + g.Expect(result.Status).To(Equal(fakeAKSExtensionStatus)) + }, + }, + } + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + result, err := tc.spec.Parameters(context.TODO(), tc.existing) + if tc.expectedError != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(tc.expectedError)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + tc.expect(g, *result) + }) + } +} diff --git a/config/aso/crds.yaml b/config/aso/crds.yaml index b10f688d311..c7aad70f566 100644 --- a/config/aso/crds.yaml +++ b/config/aso/crds.yaml @@ -535,6 +535,880 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: azureserviceoperator-system/azureserviceoperator-serving-cert + controller-gen.kubebuilder.io/version: v0.13.0 + labels: + app.kubernetes.io/name: azure-service-operator + app.kubernetes.io/version: v2.5.0 + name: extensions.kubernetesconfiguration.azure.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: azureserviceoperator-webhook-service + namespace: azureserviceoperator-system + path: /convert + port: 443 + conversionReviewVersions: + - v1 + group: kubernetesconfiguration.azure.com + names: + kind: Extension + listKind: ExtensionList + plural: extensions + singular: extension + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].severity + name: Severity + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + type: string + name: v1api20230501 + schema: + openAPIV3Schema: + description: 'Generator information: - Generated from: /kubernetesconfiguration/resource-manager/Microsoft.KubernetesConfiguration/stable/2023-05-01/extensions.json - ARM URI: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{clusterRp}/{clusterResourceName}/{clusterName}/providers/Microsoft.KubernetesConfiguration/extensions/{extensionName}' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + aksAssignedIdentity: + description: 'AksAssignedIdentity: Identity of the Extension resource in an AKS cluster' + properties: + type: + description: 'Type: The identity type.' + enum: + - SystemAssigned + - UserAssigned + type: string + type: object + autoUpgradeMinorVersion: + description: 'AutoUpgradeMinorVersion: Flag to note if this extension participates in auto upgrade of minor version, or not.' + type: boolean + azureName: + description: 'AzureName: The name of the resource in Azure. This is often the same as the name of the resource in Kubernetes but it doesn''t have to be.' + type: string + configurationSettings: + additionalProperties: + type: string + description: 'ConfigurationSettings: Configuration settings, as name-value pairs for configuring this extension.' + type: object + extensionType: + description: 'ExtensionType: Type of the Extension, of which this resource is an instance of. It must be one of the Extension Types registered with Microsoft.KubernetesConfiguration by the Extension publisher.' + type: string + identity: + description: 'Identity: Identity of the Extension resource' + properties: + type: + description: 'Type: The identity type.' + enum: + - SystemAssigned + type: string + type: object + owner: + description: 'Owner: The owner of the resource. The owner controls where the resource goes when it is deployed. The owner also controls the resources lifecycle. When the owner is deleted the resource will also be deleted. This resource is an extension resource, which means that any other Azure resource can be its owner.' + properties: + armId: + description: Ownership across namespaces is not supported. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: This is the name of the Kubernetes resource to reference. + type: string + type: object + plan: + description: 'Plan: The plan information.' + properties: + name: + description: 'Name: A user defined name of the 3rd Party Artifact that is being procured.' + type: string + product: + description: 'Product: The 3rd Party artifact that is being procured. E.g. NewRelic. Product maps to the OfferID specified for the artifact at the time of Data Market onboarding.' + type: string + promotionCode: + description: 'PromotionCode: A publisher provided promotion code as provisioned in Data Market for the said product/artifact.' + type: string + publisher: + description: 'Publisher: The publisher of the 3rd Party Artifact that is being bought. E.g. NewRelic' + type: string + version: + description: 'Version: The version of the desired product/artifact.' + type: string + required: + - name + - product + - publisher + type: object + releaseTrain: + description: 'ReleaseTrain: ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is ''true''.' + type: string + scope: + description: 'Scope: Scope at which the extension is installed.' + properties: + cluster: + description: 'Cluster: Specifies that the scope of the extension is Cluster' + properties: + releaseNamespace: + description: 'ReleaseNamespace: Namespace where the extension Release must be placed, for a Cluster scoped extension. If this namespace does not exist, it will be created' + type: string + type: object + namespace: + description: 'Namespace: Specifies that the scope of the extension is Namespace' + properties: + targetNamespace: + description: 'TargetNamespace: Namespace where the extension will be created for an Namespace scoped extension. If this namespace does not exist, it will be created' + type: string + type: object + type: object + systemData: + description: 'SystemData: Top level metadata https://github.com/Azure/azure-resource-manager-rpc/blob/master/v1.0/common-api-contracts.md#system-metadata-for-all-azure-resources' + properties: + createdAt: + description: 'CreatedAt: The timestamp of resource creation (UTC).' + type: string + createdBy: + description: 'CreatedBy: The identity that created the resource.' + type: string + createdByType: + description: 'CreatedByType: The type of identity that created the resource.' + enum: + - Application + - Key + - ManagedIdentity + - User + type: string + lastModifiedAt: + description: 'LastModifiedAt: The timestamp of resource last modification (UTC)' + type: string + lastModifiedBy: + description: 'LastModifiedBy: The identity that last modified the resource.' + type: string + lastModifiedByType: + description: 'LastModifiedByType: The type of identity that last modified the resource.' + enum: + - Application + - Key + - ManagedIdentity + - User + type: string + type: object + version: + description: 'Version: User-specified version of the extension for this extension to ''pin''. To use ''version'', autoUpgradeMinorVersion must be ''false''.' + type: string + required: + - owner + type: object + status: + description: The Extension object. + properties: + aksAssignedIdentity: + description: 'AksAssignedIdentity: Identity of the Extension resource in an AKS cluster' + properties: + principalId: + description: 'PrincipalId: The principal ID of resource identity.' + type: string + tenantId: + description: 'TenantId: The tenant ID of resource.' + type: string + type: + description: 'Type: The identity type.' + type: string + type: object + autoUpgradeMinorVersion: + description: 'AutoUpgradeMinorVersion: Flag to note if this extension participates in auto upgrade of minor version, or not.' + type: boolean + conditions: + description: 'Conditions: The observed state of the resource' + items: + description: Condition defines an extension to status (an observation) of a resource + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: Message is a human readable message indicating details about the transition. This field may be empty. + type: string + observedGeneration: + description: ObservedGeneration is the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + type: integer + reason: + description: Reason for the condition's last transition. Reasons are upper CamelCase (PascalCase) with no spaces. A reason is always provided, this field will not be empty. + type: string + severity: + description: Severity with which to treat failures of this type of condition. For conditions which have positive polarity (Status == True is their normal/healthy state), this will be omitted when Status == True For conditions which have negative polarity (Status == False is their normal/healthy state), this will be omitted when Status == False. This is omitted in all cases when Status == Unknown + type: string + status: + description: Status of the condition, one of True, False, or Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - lastTransitionTime + - reason + - status + - type + type: object + type: array + configurationSettings: + additionalProperties: + type: string + description: 'ConfigurationSettings: Configuration settings, as name-value pairs for configuring this extension.' + type: object + currentVersion: + description: 'CurrentVersion: Currently installed version of the extension.' + type: string + customLocationSettings: + additionalProperties: + type: string + description: 'CustomLocationSettings: Custom Location settings properties.' + type: object + errorInfo: + description: 'ErrorInfo: Error information from the Agent - e.g. errors during installation.' + properties: + additionalInfo: + description: 'AdditionalInfo: The error additional info.' + items: + description: The resource management error additional info. + properties: + info: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + description: 'Info: The additional info.' + type: object + type: + description: 'Type: The additional info type.' + type: string + type: object + type: array + code: + description: 'Code: The error code.' + type: string + details: + description: 'Details: The error details.' + items: + properties: + additionalInfo: + description: 'AdditionalInfo: The error additional info.' + items: + description: The resource management error additional info. + properties: + info: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + description: 'Info: The additional info.' + type: object + type: + description: 'Type: The additional info type.' + type: string + type: object + type: array + code: + description: 'Code: The error code.' + type: string + message: + description: 'Message: The error message.' + type: string + target: + description: 'Target: The error target.' + type: string + type: object + type: array + message: + description: 'Message: The error message.' + type: string + target: + description: 'Target: The error target.' + type: string + type: object + extensionType: + description: 'ExtensionType: Type of the Extension, of which this resource is an instance of. It must be one of the Extension Types registered with Microsoft.KubernetesConfiguration by the Extension publisher.' + type: string + id: + description: 'Id: Fully qualified resource ID for the resource. Ex - /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}' + type: string + identity: + description: 'Identity: Identity of the Extension resource' + properties: + principalId: + description: 'PrincipalId: The principal ID of resource identity.' + type: string + tenantId: + description: 'TenantId: The tenant ID of resource.' + type: string + type: + description: 'Type: The identity type.' + type: string + type: object + isSystemExtension: + description: 'IsSystemExtension: Flag to note if this extension is a system extension' + type: boolean + name: + description: 'Name: The name of the resource' + type: string + packageUri: + description: 'PackageUri: Uri of the Helm package' + type: string + plan: + description: 'Plan: The plan information.' + properties: + name: + description: 'Name: A user defined name of the 3rd Party Artifact that is being procured.' + type: string + product: + description: 'Product: The 3rd Party artifact that is being procured. E.g. NewRelic. Product maps to the OfferID specified for the artifact at the time of Data Market onboarding.' + type: string + promotionCode: + description: 'PromotionCode: A publisher provided promotion code as provisioned in Data Market for the said product/artifact.' + type: string + publisher: + description: 'Publisher: The publisher of the 3rd Party Artifact that is being bought. E.g. NewRelic' + type: string + version: + description: 'Version: The version of the desired product/artifact.' + type: string + type: object + provisioningState: + description: 'ProvisioningState: Status of installation of this extension.' + type: string + releaseTrain: + description: 'ReleaseTrain: ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is ''true''.' + type: string + scope: + description: 'Scope: Scope at which the extension is installed.' + properties: + cluster: + description: 'Cluster: Specifies that the scope of the extension is Cluster' + properties: + releaseNamespace: + description: 'ReleaseNamespace: Namespace where the extension Release must be placed, for a Cluster scoped extension. If this namespace does not exist, it will be created' + type: string + type: object + namespace: + description: 'Namespace: Specifies that the scope of the extension is Namespace' + properties: + targetNamespace: + description: 'TargetNamespace: Namespace where the extension will be created for an Namespace scoped extension. If this namespace does not exist, it will be created' + type: string + type: object + type: object + statuses: + description: 'Statuses: Status from this extension.' + items: + description: Status from the extension. + properties: + code: + description: 'Code: Status code provided by the Extension' + type: string + displayStatus: + description: 'DisplayStatus: Short description of status of the extension.' + type: string + level: + description: 'Level: Level of the status.' + type: string + message: + description: 'Message: Detailed message of the status from the Extension.' + type: string + time: + description: 'Time: DateLiteral (per ISO8601) noting the time of installation status.' + type: string + type: object + type: array + systemData: + description: 'SystemData: Top level metadata https://github.com/Azure/azure-resource-manager-rpc/blob/master/v1.0/common-api-contracts.md#system-metadata-for-all-azure-resources' + properties: + createdAt: + description: 'CreatedAt: The timestamp of resource creation (UTC).' + type: string + createdBy: + description: 'CreatedBy: The identity that created the resource.' + type: string + createdByType: + description: 'CreatedByType: The type of identity that created the resource.' + type: string + lastModifiedAt: + description: 'LastModifiedAt: The timestamp of resource last modification (UTC)' + type: string + lastModifiedBy: + description: 'LastModifiedBy: The identity that last modified the resource.' + type: string + lastModifiedByType: + description: 'LastModifiedByType: The type of identity that last modified the resource.' + type: string + type: object + type: + description: 'Type: The type of the resource. E.g. "Microsoft.Compute/virtualMachines" or "Microsoft.Storage/storageAccounts"' + type: string + version: + description: 'Version: User-specified version of the extension for this extension to ''pin''. To use ''version'', autoUpgradeMinorVersion must be ''false''.' + type: string + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].severity + name: Severity + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + type: string + name: v1api20230501storage + schema: + openAPIV3Schema: + description: 'Storage version of v1api20230501.Extension Generator information: - Generated from: /kubernetesconfiguration/resource-manager/Microsoft.KubernetesConfiguration/stable/2023-05-01/extensions.json - ARM URI: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{clusterRp}/{clusterResourceName}/{clusterName}/providers/Microsoft.KubernetesConfiguration/extensions/{extensionName}' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Storage version of v1api20230501.Extension_Spec + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + aksAssignedIdentity: + description: Storage version of v1api20230501.Extension_Properties_AksAssignedIdentity_Spec + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + type: + type: string + type: object + autoUpgradeMinorVersion: + type: boolean + azureName: + description: 'AzureName: The name of the resource in Azure. This is often the same as the name of the resource in Kubernetes but it doesn''t have to be.' + type: string + configurationSettings: + additionalProperties: + type: string + type: object + extensionType: + type: string + identity: + description: Storage version of v1api20230501.Identity Identity for the resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + type: + type: string + type: object + originalVersion: + type: string + owner: + description: 'Owner: The owner of the resource. The owner controls where the resource goes when it is deployed. The owner also controls the resources lifecycle. When the owner is deleted the resource will also be deleted. This resource is an extension resource, which means that any other Azure resource can be its owner.' + properties: + armId: + description: Ownership across namespaces is not supported. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: This is the name of the Kubernetes resource to reference. + type: string + type: object + plan: + description: Storage version of v1api20230501.Plan Plan for the resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + name: + type: string + product: + type: string + promotionCode: + type: string + publisher: + type: string + version: + type: string + type: object + releaseTrain: + type: string + scope: + description: Storage version of v1api20230501.Scope Scope of the extension. It can be either Cluster or Namespace; but not both. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + cluster: + description: Storage version of v1api20230501.ScopeCluster Specifies that the scope of the extension is Cluster + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + releaseNamespace: + type: string + type: object + namespace: + description: Storage version of v1api20230501.ScopeNamespace Specifies that the scope of the extension is Namespace + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + targetNamespace: + type: string + type: object + type: object + systemData: + description: Storage version of v1api20230501.SystemData Metadata pertaining to creation and last modification of the resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + createdAt: + type: string + createdBy: + type: string + createdByType: + type: string + lastModifiedAt: + type: string + lastModifiedBy: + type: string + lastModifiedByType: + type: string + type: object + version: + type: string + required: + - owner + type: object + status: + description: Storage version of v1api20230501.Extension_STATUS The Extension object. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + aksAssignedIdentity: + description: Storage version of v1api20230501.Extension_Properties_AksAssignedIdentity_STATUS + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + principalId: + type: string + tenantId: + type: string + type: + type: string + type: object + autoUpgradeMinorVersion: + type: boolean + conditions: + items: + description: Condition defines an extension to status (an observation) of a resource + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: Message is a human readable message indicating details about the transition. This field may be empty. + type: string + observedGeneration: + description: ObservedGeneration is the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + type: integer + reason: + description: Reason for the condition's last transition. Reasons are upper CamelCase (PascalCase) with no spaces. A reason is always provided, this field will not be empty. + type: string + severity: + description: Severity with which to treat failures of this type of condition. For conditions which have positive polarity (Status == True is their normal/healthy state), this will be omitted when Status == True For conditions which have negative polarity (Status == False is their normal/healthy state), this will be omitted when Status == False. This is omitted in all cases when Status == Unknown + type: string + status: + description: Status of the condition, one of True, False, or Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - lastTransitionTime + - reason + - status + - type + type: object + type: array + configurationSettings: + additionalProperties: + type: string + type: object + currentVersion: + type: string + customLocationSettings: + additionalProperties: + type: string + type: object + errorInfo: + description: Storage version of v1api20230501.ErrorDetail_STATUS The error detail. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + additionalInfo: + items: + description: Storage version of v1api20230501.ErrorAdditionalInfo_STATUS The resource management error additional info. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + info: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + type: object + type: + type: string + type: object + type: array + code: + type: string + details: + items: + description: Storage version of v1api20230501.ErrorDetail_STATUS_Unrolled + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + additionalInfo: + items: + description: Storage version of v1api20230501.ErrorAdditionalInfo_STATUS The resource management error additional info. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + info: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + type: object + type: + type: string + type: object + type: array + code: + type: string + message: + type: string + target: + type: string + type: object + type: array + message: + type: string + target: + type: string + type: object + extensionType: + type: string + id: + type: string + identity: + description: Storage version of v1api20230501.Identity_STATUS Identity for the resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + principalId: + type: string + tenantId: + type: string + type: + type: string + type: object + isSystemExtension: + type: boolean + name: + type: string + packageUri: + type: string + plan: + description: Storage version of v1api20230501.Plan_STATUS Plan for the resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + name: + type: string + product: + type: string + promotionCode: + type: string + publisher: + type: string + version: + type: string + type: object + provisioningState: + type: string + releaseTrain: + type: string + scope: + description: Storage version of v1api20230501.Scope_STATUS Scope of the extension. It can be either Cluster or Namespace; but not both. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + cluster: + description: Storage version of v1api20230501.ScopeCluster_STATUS Specifies that the scope of the extension is Cluster + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + releaseNamespace: + type: string + type: object + namespace: + description: Storage version of v1api20230501.ScopeNamespace_STATUS Specifies that the scope of the extension is Namespace + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + targetNamespace: + type: string + type: object + type: object + statuses: + items: + description: Storage version of v1api20230501.ExtensionStatus_STATUS Status from the extension. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + code: + type: string + displayStatus: + type: string + level: + type: string + message: + type: string + time: + type: string + type: object + type: array + systemData: + description: Storage version of v1api20230501.SystemData_STATUS Metadata pertaining to creation and last modification of the resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + createdAt: + type: string + createdBy: + type: string + createdByType: + type: string + lastModifiedAt: + type: string + lastModifiedBy: + type: string + lastModifiedByType: + type: string + type: object + type: + type: string + version: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: azureserviceoperator-system/azureserviceoperator-serving-cert diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index 66a944752b7..5f4dab717ea 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -257,6 +257,110 @@ spec: DNS service. It must be within the Kubernetes service address range specified in serviceCidr. Immutable. type: string + extensions: + description: Extensions is a list of AKS extensions to be installed + on the cluster. + items: + description: "AKSExtension represents the configuration for an AKS + cluster extension. See also [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/en-us/azure/aks/cluster-extensions" + properties: + aksAssignedIdentityType: + description: AKSAssignedIdentityType is the type of the AKS + assigned identity. + enum: + - SystemAssigned + - UserAssigned + type: string + autoUpgradeMinorVersion: + default: true + description: AutoUpgradeMinorVersion is a flag to note if this + extension participates in auto upgrade of minor version, or + not. + type: boolean + configurationSettings: + additionalProperties: + type: string + description: ConfigurationSettings are the name-value pairs + for configuring this extension. + type: object + extensionType: + description: ExtensionType is the type of the Extension of which + this resource is an instance. It must be one of the Extension + Types registered with Microsoft.KubernetesConfiguration by + the Extension publisher. + type: string + identity: + description: Identity is the identity type of the Extension + resource in an AKS cluster. + enum: + - SystemAssigned + type: string + name: + description: Name is the name of the extension. + type: string + plan: + description: Plan is the plan of the extension. + properties: + name: + description: Name is the user-defined name of the 3rd Party + Artifact that is being procured. + type: string + product: + description: Product is the name of the 3rd Party artifact + that is being procured. + type: string + promotionCode: + description: PromotionCode is a publisher-provided promotion + code as provisioned in Data Market for the said product/artifact. + type: string + publisher: + description: Publisher is the name of the publisher of the + 3rd Party Artifact that is being bought. + type: string + version: + description: Version is the version of the plan. + type: string + required: + - product + - publisher + type: object + releaseTrain: + description: ReleaseTrain is the release train this extension + participates in for auto-upgrade (e.g. Stable, Preview, etc.) + This is only used if autoUpgradeMinorVersion is ‘true’. + type: string + scope: + description: Scope is the scope at which this extension is enabled. + properties: + releaseNamespace: + description: ReleaseNamespace is the namespace where the + extension Release must be placed, for a Cluster-scoped + extension. Required for Cluster-scoped extensions. + type: string + scopeType: + description: ScopeType is the scope of the extension. It + can be either Cluster or Namespace, but not both. + enum: + - Cluster + - Namespace + type: string + targetNamespace: + description: TargetNamespace is the namespace where the + extension will be created for a Namespace-scoped extension. + Required for Namespace-scoped extensions. + type: string + required: + - scopeType + type: object + version: + description: Version is the version of the extension. + type: string + required: + - extensionType + - name + - plan + type: object + type: array fleetsMember: description: "FleetsMember is the spec for the fleet this cluster is a member of. See also [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/en-us/azure/templates/microsoft.containerservice/2023-03-15-preview/fleets/members" diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml index 758e659ce85..a87c1b47ed8 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml @@ -260,6 +260,115 @@ spec: Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. Immutable. type: string + extensions: + description: Extensions is a list of AKS extensions to be + installed on the cluster. + items: + description: "AKSExtension represents the configuration + for an AKS cluster extension. See also [AKS doc]. \n [AKS + doc]: https://learn.microsoft.com/en-us/azure/aks/cluster-extensions" + properties: + aksAssignedIdentityType: + description: AKSAssignedIdentityType is the type of + the AKS assigned identity. + enum: + - SystemAssigned + - UserAssigned + type: string + autoUpgradeMinorVersion: + default: true + description: AutoUpgradeMinorVersion is a flag to note + if this extension participates in auto upgrade of + minor version, or not. + type: boolean + configurationSettings: + additionalProperties: + type: string + description: ConfigurationSettings are the name-value + pairs for configuring this extension. + type: object + extensionType: + description: ExtensionType is the type of the Extension + of which this resource is an instance. It must be + one of the Extension Types registered with Microsoft.KubernetesConfiguration + by the Extension publisher. + type: string + identity: + description: Identity is the identity type of the Extension + resource in an AKS cluster. + enum: + - SystemAssigned + type: string + name: + description: Name is the name of the extension. + type: string + plan: + description: Plan is the plan of the extension. + properties: + name: + description: Name is the user-defined name of the + 3rd Party Artifact that is being procured. + type: string + product: + description: Product is the name of the 3rd Party + artifact that is being procured. + type: string + promotionCode: + description: PromotionCode is a publisher-provided + promotion code as provisioned in Data Market for + the said product/artifact. + type: string + publisher: + description: Publisher is the name of the publisher + of the 3rd Party Artifact that is being bought. + type: string + version: + description: Version is the version of the plan. + type: string + required: + - product + - publisher + type: object + releaseTrain: + description: ReleaseTrain is the release train this + extension participates in for auto-upgrade (e.g. Stable, + Preview, etc.) This is only used if autoUpgradeMinorVersion + is ‘true’. + type: string + scope: + description: Scope is the scope at which this extension + is enabled. + properties: + releaseNamespace: + description: ReleaseNamespace is the namespace where + the extension Release must be placed, for a Cluster-scoped + extension. Required for Cluster-scoped extensions. + type: string + scopeType: + description: ScopeType is the scope of the extension. + It can be either Cluster or Namespace, but not + both. + enum: + - Cluster + - Namespace + type: string + targetNamespace: + description: TargetNamespace is the namespace where + the extension will be created for a Namespace-scoped + extension. Required for Namespace-scoped extensions. + type: string + required: + - scopeType + type: object + version: + description: Version is the version of the extension. + type: string + required: + - extensionType + - name + - plan + type: object + type: array fleetsMember: description: "FleetsMember is the spec for the fleet this cluster is a member of. See also [AKS doc]. \n [AKS doc]: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ad06984359f..fb56aff03ed 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -315,6 +315,26 @@ rules: - get - patch - update +- apiGroups: + - kubernetesconfiguration.azure.com + resources: + - extensions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubernetesconfiguration.azure.com + resources: + - extensions/status + verbs: + - get + - list + - watch - apiGroups: - network.azure.com resources: diff --git a/controllers/azuremanagedcontrolplane_controller.go b/controllers/azuremanagedcontrolplane_controller.go index d8559614d83..ecf6917a0e5 100644 --- a/controllers/azuremanagedcontrolplane_controller.go +++ b/controllers/azuremanagedcontrolplane_controller.go @@ -120,6 +120,8 @@ func (amcpr *AzureManagedControlPlaneReconciler) SetupWithManager(ctx context.Co // +kubebuilder:rbac:groups=network.azure.com,resources=privateendpoints/status;virtualnetworks/status;virtualnetworkssubnets/status,verbs=get;list;watch // +kubebuilder:rbac:groups=containerservice.azure.com,resources=fleetsmembers,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=containerservice.azure.com,resources=fleetsmembers/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=kubernetesconfiguration.azure.com,resources=extensions,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=kubernetesconfiguration.azure.com,resources=extensions/status,verbs=get;list;watch // Reconcile idempotently gets, creates, and updates a managed control plane. func (amcpr *AzureManagedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { diff --git a/controllers/azuremanagedcontrolplane_reconciler.go b/controllers/azuremanagedcontrolplane_reconciler.go index 14f5f8660da..8f70b500a81 100644 --- a/controllers/azuremanagedcontrolplane_reconciler.go +++ b/controllers/azuremanagedcontrolplane_reconciler.go @@ -24,6 +24,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/scope" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/aksextensions" "sigs.k8s.io/cluster-api-provider-azure/azure/services/fleetsmembers" "sigs.k8s.io/cluster-api-provider-azure/azure/services/groups" "sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters" @@ -60,6 +61,7 @@ func newAzureManagedControlPlaneReconciler(scope *scope.ManagedControlPlaneScope managedclusters.New(scope), privateendpoints.New(scope), fleetsmembers.New(scope), + aksextensions.New(scope), resourceHealthSvc, }, }, nil diff --git a/docs/book/src/topics/managedcluster.md b/docs/book/src/topics/managedcluster.md index f6fa3842867..52478a6cddf 100644 --- a/docs/book/src/topics/managedcluster.md +++ b/docs/book/src/topics/managedcluster.md @@ -302,6 +302,32 @@ The `managerName` and `managerResourceGroup` fields are the name and resource gr When the `fleetMember` field is included, CAPZ will create an AKS fleet member resource which will join the CAPZ cluster to the AKS fleet. The AKS fleet member resource will be created in the same resource group as the CAPZ cluster. +### AKS Extensions + +CAPZ supports enabling AKS extensions on your managed AKS clusters. Cluster extensions provide an Azure Resource Manager driven experience for installation and lifecycle management of services like Azure Machine Learning or Kubernetes applications on an AKS cluster. For more documentation on AKS extensions, refer [AKS Docs](https://learn.microsoft.com/azure/aks/cluster-extensions). + +You can either provision official AKS extensions or Kubernetes applications through Marketplace. Please refer to [AKS Docs](https://learn.microsoft.com/en-us/azure/aks/cluster-extensions#currently-available-extensions) for the list of currently available extensions. + +To add an AKS extension to your managed cluster, simply add the `extensions` field to your AzureManagedControlPlane resource spec: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlane +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + extensions: + - name: my-extension + extensionType: "TraefikLabs.TraefikProxy" + plan: + name: "traefik-proxy" + product: "traefik-proxy" + publisher: "containous" +``` + +To find the `extensionType` and plan details for your desired extension, refer to the [az k8s-extension cli reference](https://learn.microsoft.com/cli/azure/k8s-extension). + ## Features AKS clusters deployed from CAPZ currently only support a limited, diff --git a/go.mod b/go.mod index 32d187516e9..33c67ea932a 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( require ( github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/kubernetesconfiguration/armkubernetesconfiguration v1.1.1 github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect diff --git a/go.sum b/go.sum index 87bd11641d4..b6d36e847d7 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventgrid/armeventgrid v1. github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/iothub/armiothub v1.3.0 h1:NZP+oPbAVFy7PhQ4PTD3SuGWbEziNhp7lphGkkN707s= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0 h1:HlZMUZW8S4P9oob1nCHxCCKrytxyLc+24nUJGssoEto= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/kubernetesconfiguration/armkubernetesconfiguration v1.1.1 h1:eAVG12uoHrqI8yiSrCtuQoLnKYYL5bnLrFbnP8lL+SE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/kubernetesconfiguration/armkubernetesconfiguration v1.1.1/go.mod h1:NLG7km6cZfoAnpkUypZQjAIIRADzps6vIusTTUE8rrE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/machinelearning/armmachinelearning v1.0.0 h1:KWvCVjnOTKCZAlqED5KPNoN9AfcK2BhUeveLdiwy33Q= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 h1:Ds0KRF8ggpEGg4Vo42oX1cIt/IfOhHWJBikksZbVxeg= diff --git a/main.go b/main.go index 0852b54f0e6..0195b39bead 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,7 @@ import ( // +kubebuilder:scaffold:imports asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview" asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001" + asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501" asonetworkv1api20201101 "github.com/Azure/azure-service-operator/v2/api/network/v1api20201101" asonetworkv1api20220701 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601" @@ -81,6 +82,7 @@ func init() { _ = asonetworkv1api20220701.AddToScheme(scheme) _ = asonetworkv1api20201101.AddToScheme(scheme) _ = asocontainerservicev1preview.AddToScheme(scheme) + _ = asokubernetesconfigurationv1.AddToScheme(scheme) // +kubebuilder:scaffold:scheme } diff --git a/test/e2e/aks_marketplace.go b/test/e2e/aks_marketplace.go new file mode 100644 index 00000000000..c444bd5f632 --- /dev/null +++ b/test/e2e/aks_marketplace.go @@ -0,0 +1,179 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/kubernetesconfiguration/armkubernetesconfiguration" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type AKSMarketplaceExtensionSpecInput struct { + Cluster *clusterv1.Cluster + WaitIntervals []interface{} +} + +const ( + extensionName = "aks-marketplace-extension" +) + +func AKSMarketplaceExtensionSpec(ctx context.Context, inputGetter func() AKSMarketplaceExtensionSpecInput) { + input := inputGetter() + + cred, err := azidentity.NewDefaultAzureCredential(nil) + Expect(err).NotTo(HaveOccurred()) + + mgmtClient := bootstrapClusterProxy.GetClient() + Expect(mgmtClient).NotTo(BeNil()) + + amcp := &infrav1.AzureManagedControlPlane{} + err = mgmtClient.Get(ctx, types.NamespacedName{ + Namespace: input.Cluster.Spec.ControlPlaneRef.Namespace, + Name: input.Cluster.Spec.ControlPlaneRef.Name, + }, amcp) + Expect(err).NotTo(HaveOccurred()) + + agentpoolsClient, err := armcontainerservice.NewAgentPoolsClient(amcp.Spec.SubscriptionID, cred, nil) + Expect(err).NotTo(HaveOccurred()) + + extensionClient, err := armkubernetesconfiguration.NewExtensionsClient(amcp.Spec.SubscriptionID, cred, nil) + Expect(err).NotTo(HaveOccurred()) + + By("Deleting all node taints for windows machine pool") + var ammp = &infrav1.AzureManagedMachinePool{} + Expect(mgmtClient.Get(ctx, types.NamespacedName{ + Namespace: input.Cluster.Namespace, + Name: input.Cluster.Name + "-pool2", + }, ammp)).To(Succeed()) + initialTaints := ammp.Spec.Taints + var expectedTaints []infrav1.Taint + expectedTaints = nil + checkTaints := func(g Gomega) { + var expectedTaintStrs []*string + if expectedTaints != nil { + expectedTaintStrs = make([]*string, 0, len(expectedTaints)) + for _, taint := range expectedTaints { + expectedTaintStrs = append(expectedTaintStrs, ptr.To(fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect))) + } + } + + resp, err := agentpoolsClient.Get(ctx, amcp.Spec.ResourceGroupName, amcp.Name, *ammp.Spec.Name, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(resp.Properties.ProvisioningState).To(Equal(ptr.To("Succeeded"))) + actualTaintStrs := resp.AgentPool.Properties.NodeTaints + if expectedTaintStrs == nil { + g.Expect(actualTaintStrs).To(BeNil()) + } else { + g.Expect(actualTaintStrs).To(Equal(expectedTaintStrs)) + } + } + Eventually(func(g Gomega) { + g.Expect(mgmtClient.Get(ctx, client.ObjectKeyFromObject(ammp), ammp)).To(Succeed()) + ammp.Spec.Taints = expectedTaints + g.Expect(mgmtClient.Update(ctx, ammp)).To(Succeed()) + }, inputGetter().WaitIntervals...).Should(Succeed()) + Eventually(checkTaints, input.WaitIntervals...).Should(Succeed()) + + By("Adding a taint to the Windows node pool") + expectedTaints = []infrav1.Taint{ + { + Effect: infrav1.TaintEffect(corev1.TaintEffectNoSchedule), + Key: "capz-e2e-1", + Value: "test1", + }, + } + Eventually(func(g Gomega) { + g.Expect(mgmtClient.Get(ctx, client.ObjectKeyFromObject(ammp), ammp)).To(Succeed()) + ammp.Spec.Taints = expectedTaints + g.Expect(mgmtClient.Update(ctx, ammp)).To(Succeed()) + Eventually(checkTaints, input.WaitIntervals...).Should(Succeed()) + }, input.WaitIntervals...).Should(Succeed()) + + By("Updating taints for Windows machine pool") + expectedTaints = expectedTaints[:1] + Eventually(func(g Gomega) { + g.Expect(mgmtClient.Get(ctx, client.ObjectKeyFromObject(ammp), ammp)).To(Succeed()) + ammp.Spec.Taints = expectedTaints + g.Expect(mgmtClient.Update(ctx, ammp)).To(Succeed()) + }, input.WaitIntervals...).Should(Succeed()) + Eventually(checkTaints, input.WaitIntervals...).Should(Succeed()) + + By("Adding an AKS Marketplace Extension to the AzureManagedControlPlane") + var infraControlPlane = &infrav1.AzureManagedControlPlane{} + Eventually(func(g Gomega) { + err = mgmtClient.Get(ctx, client.ObjectKey{ + Namespace: input.Cluster.Spec.ControlPlaneRef.Namespace, + Name: input.Cluster.Spec.ControlPlaneRef.Name, + }, infraControlPlane) + g.Expect(err).NotTo(HaveOccurred()) + infraControlPlane.Spec.Extensions = []infrav1.AKSExtension{ + { + Name: extensionName, + ExtensionType: ptr.To("TraefikLabs.TraefikProxy"), + Plan: &infrav1.ExtensionPlan{ + Name: "traefik-proxy", + Product: "traefik-proxy", + Publisher: "containous", + }, + }, + } + g.Expect(mgmtClient.Update(ctx, infraControlPlane)).To(Succeed()) + }, input.WaitIntervals...).Should(Succeed()) + + By("Ensuring the AKS Marketplace Extension status is ready on the AzureManagedControlPlane") + Eventually(func(g Gomega) { + err = mgmtClient.Get(ctx, client.ObjectKey{Namespace: input.Cluster.Spec.ControlPlaneRef.Namespace, Name: input.Cluster.Spec.ControlPlaneRef.Name}, infraControlPlane) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(conditions.IsTrue(infraControlPlane, infrav1.AKSExtensionsReadyCondition)).To(BeTrue()) + }, input.WaitIntervals...).Should(Succeed()) + + By("Ensuring the AKS Marketplace Extension is added to the AzureManagedControlPlane") + Eventually(func(g Gomega) { + resp, err := extensionClient.Get(ctx, amcp.Spec.ResourceGroupName, "Microsoft.ContainerService", "managedClusters", input.Cluster.Name, extensionName, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(resp.Properties.ProvisioningState).To(Equal(ptr.To(armkubernetesconfiguration.ProvisioningStateSucceeded))) + extension := resp.Extension + g.Expect(extension.Properties).NotTo(BeNil()) + g.Expect(extension.Name).To(Equal(ptr.To(extensionName))) + g.Expect(extension.Properties.AutoUpgradeMinorVersion).To(Equal(ptr.To(true))) + g.Expect(extension.Properties.ExtensionType).To(Equal(ptr.To("TraefikLabs.TraefikProxy"))) + }, input.WaitIntervals...).Should(Succeed()) + + By("Restoring initial taints for Windows machine pool") + expectedTaints = initialTaints + Eventually(func(g Gomega) { + g.Expect(mgmtClient.Get(ctx, client.ObjectKeyFromObject(ammp), ammp)).To(Succeed()) + ammp.Spec.Taints = expectedTaints + g.Expect(mgmtClient.Update(ctx, ammp)).To(Succeed()) + }, input.WaitIntervals...).Should(Succeed()) + Eventually(checkTaints, input.WaitIntervals...).Should(Succeed()) +} diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 3178862263a..34da41229c5 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -678,6 +678,15 @@ var _ = Describe("Workload cluster creation", func() { }), ), result) + By("adding an AKS marketplace extension", func() { + AKSMarketplaceExtensionSpec(ctx, func() AKSMarketplaceExtensionSpecInput { + return AKSMarketplaceExtensionSpecInput{ + Cluster: result.Cluster, + WaitIntervals: e2eConfig.GetIntervals(specName, "wait-machine-pool-nodes"), + } + }) + }) + By("attaching the cluster to azure fleet", func() { AKSFleetsMemberSpec(ctx, func() AKSFleetsMemberInput { return AKSFleetsMemberInput{