From c942ecf27f6608a9f9ab010d3688b3dcb010337d Mon Sep 17 00:00:00 2001 From: willie-yao Date: Mon, 4 Dec 2023 22:48:19 +0000 Subject: [PATCH] Add support for AKS Marketplace extensions --- Makefile | 2 +- .../azuremanagedcontrolplane_default.go | 11 + api/v1beta1/azuremanagedcontrolplane_types.go | 46 + .../azuremanagedcontrolplane_webhook.go | 128 +++ .../azuremanagedcontrolplane_webhook_test.go | 182 ++++ ...zuremanagedcontrolplanetemplate_webhook.go | 6 + ...anagedcontrolplanetemplate_webhook_test.go | 62 ++ api/v1beta1/consts.go | 2 + api/v1beta1/types.go | 69 ++ api/v1beta1/types_class.go | 4 + api/v1beta1/zz_generated.deepcopy.go | 89 ++ azure/scope/managedcontrolplane.go | 34 + azure/scope/managedcontrolplane_test.go | 106 +++ azure/services/aksextensions/aksextensions.go | 50 + azure/services/aksextensions/spec.go | 110 +++ azure/services/aksextensions/spec_test.go | 155 ++++ config/aso/crds.yaml | 874 ++++++++++++++++++ ...er.x-k8s.io_azuremanagedcontrolplanes.yaml | 105 +++ ....io_azuremanagedcontrolplanetemplates.yaml | 109 +++ config/rbac/role.yaml | 20 + .../azuremanagedcontrolplane_controller.go | 2 + .../azuremanagedcontrolplane_reconciler.go | 2 + go.mod | 1 + go.sum | 2 + main.go | 2 + test/e2e/aks_marketplace.go | 179 ++++ test/e2e/azure_test.go | 9 + 27 files changed, 2360 insertions(+), 1 deletion(-) create mode 100644 azure/services/aksextensions/aksextensions.go create mode 100644 azure/services/aksextensions/spec.go create mode 100644 azure/services/aksextensions/spec_test.go create mode 100644 test/e2e/aks_marketplace.go diff --git a/Makefile b/Makefile index 3534ff95f8f0..f2e2aee5d506 100644 --- a/Makefile +++ b/Makefile @@ -158,7 +158,7 @@ WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac ASO_CRDS_PATH := $(MANIFEST_ROOT)/aso/crds.yaml ASO_VERSION := v2.5.0 -ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com virtualnetworks.network.azure.com virtualnetworkssubnets.network.azure.com privateendpoints.network.azure.com +ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com virtualnetworks.network.azure.com virtualnetworkssubnets.network.azure.com privateendpoints.network.azure.com extensions.kubernetesconfiguration.azure.com # Allow overriding the imagePullPolicy PULL_POLICY ?= Always diff --git a/api/v1beta1/azuremanagedcontrolplane_default.go b/api/v1beta1/azuremanagedcontrolplane_default.go index f4b0cd4f4c6b..8329b3ca48c8 100644 --- a/api/v1beta1/azuremanagedcontrolplane_default.go +++ b/api/v1beta1/azuremanagedcontrolplane_default.go @@ -198,3 +198,14 @@ func (m *AzureManagedControlPlane) setDefaultDNSPrefix() { m.Spec.DNSPrefix = ptr.To(m.Name) } } + +func (m *AzureManagedControlPlane) setDefaultMarketplaceExtensions() { + for _, extension := range m.Spec.MarketplaceExtensions { + if extension.Plan.Name == "" { + extension.Plan.Name = fmt.Sprintf("%s-%s", m.Name, extension.Plan.Product) + } + if extension.AutoUpgradeMinorVersion == nil { + extension.AutoUpgradeMinorVersion = ptr.To(true) + } + } +} diff --git a/api/v1beta1/azuremanagedcontrolplane_types.go b/api/v1beta1/azuremanagedcontrolplane_types.go index 09535d9e25de..174e5e955734 100644 --- a/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/api/v1beta1/azuremanagedcontrolplane_types.go @@ -421,6 +421,52 @@ type OIDCIssuerProfile struct { Enabled *bool `json:"enabled,omitempty"` } +// MarketplaceExtension represents the configuration for a marketplace extension. +// See also [AKS doc]. +// +// [AKS doc]: https://learn.microsoft.com/en-us/azure/aks/cluster-extensions +type MarketplaceExtension struct { + // Name is the name of the extension. + Name string `json:"name"` + + // AKSAssignedIdentityType is the type of the AKS assigned identity. + // +optional + AKSAssignedIdentityType AKSAssignedIdentity `json:"aksAssignedIdentityType,omitempty"` + + // AutoUpgradeMinorVersion is a flag to note if this extension participates in auto upgrade of minor version, or not. + // +kubebuilder:default=true + // +optional + AutoUpgradeMinorVersion *bool `json:"autoUpgradeMinorVersion,omitempty"` + + // ConfigurationSettings are the name-value pairs for configuring this extension. + // +optional + ConfigurationSettings map[string]string `json:"configurationSettings,omitempty"` + + // ExtensionType is the type of the Extension of which this resource is an instance. + // It must be one of the Extension Types registered with Microsoft.KubernetesConfiguration by the Extension publisher. + ExtensionType *string `json:"extensionType"` + + // Plan is the plan of the extension. + Plan *MarketplacePlan `json:"plan"` + + // ReleaseTrain is the release train this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) + // This is only used if autoUpgradeMinorVersion is ‘true’. + // +optional + ReleaseTrain *string `json:"releaseTrain,omitempty"` + + // Scope is the scope at which this extension is enabled. + // +optional + Scope *ExtensionScope `json:"scope,omitempty"` + + // Version is the version of the extension. + // +optional + Version *string `json:"version,omitempty"` + + // Identity is the identity type of the Extension resource in an AKS cluster. + // +optional + Identity ExtensionIdentity `json:"identity,omitempty"` +} + // +kubebuilder:object:root=true // +kubebuilder:resource:path=azuremanagedcontrolplanes,scope=Namespaced,categories=cluster-api,shortName=amcp // +kubebuilder:storageversion diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook.go b/api/v1beta1/azuremanagedcontrolplane_webhook.go index 3285c0591433..a43a5b2d7808 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -94,6 +94,7 @@ func (mw *azureManagedControlPlaneWebhook) Default(ctx context.Context, obj runt m.setDefaultSubnet() m.setDefaultOIDCIssuerProfile() m.setDefaultDNSPrefix() + m.setDefaultMarketplaceExtensions() return nil } @@ -260,6 +261,10 @@ func (mw *azureManagedControlPlaneWebhook) ValidateUpdate(ctx context.Context, o allErrs = append(allErrs, errs...) } + if errs := validateMarketplaceExtensionsUpdate(old.Spec.MarketplaceExtensions, m.Spec.MarketplaceExtensions); len(errs) > 0 { + allErrs = append(allErrs, errs...) + } + if len(allErrs) == 0 { return nil, m.Validate(mw.Client) } @@ -309,6 +314,8 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { allErrs = append(allErrs, validateAutoScalerProfile(m.Spec.AutoScalerProfile, field.NewPath("spec").Child("AutoScalerProfile"))...) + allErrs = append(allErrs, validateMarketplaceExtensions(m.Spec.MarketplaceExtensions, field.NewPath("spec").Child("MarketplaceExtensions"))...) + return allErrs.ToAggregate() } @@ -687,6 +694,89 @@ func (m *AzureManagedControlPlane) validateOIDCIssuerProfileUpdate(old *AzureMan return allErrs } +// validateMarketplaceExtensionsUpdate validates update to Marketplace extensions. +func validateMarketplaceExtensionsUpdate(old []MarketplaceExtension, current []MarketplaceExtension) field.ErrorList { + var allErrs field.ErrorList + + oldMarketplaceExtensionsMap := make(map[string]MarketplaceExtension, len(old)) + oldMarketplaceExtensionsIndex := make(map[string]int, len(old)) + for i, extension := range old { + oldMarketplaceExtensionsMap[extension.Name] = extension + oldMarketplaceExtensionsIndex[extension.Name] = i + } + for i, extension := range current { + oldExtension, ok := oldMarketplaceExtensionsMap[extension.Name] + if !ok { + continue + } + if extension.Name != oldExtension.Name { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "MarketplaceExtensions", fmt.Sprintf("[%d]", i), "Name"), + extension.Name, + "field is immutable", + ), + ) + } + if (oldExtension.ExtensionType != nil && extension.ExtensionType != nil) && *extension.ExtensionType != *oldExtension.ExtensionType { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "MarketplaceExtensions", fmt.Sprintf("[%d]", i), "ExtensionType"), + extension.ExtensionType, + "field is immutable", + ), + ) + } + if (extension.Plan != nil && oldExtension.Plan != nil) && *extension.Plan != *oldExtension.Plan { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "MarketplaceExtensions", fmt.Sprintf("[%d]", i), "Plan"), + extension.Plan, + "field is immutable", + ), + ) + } + if extension.Scope != oldExtension.Scope { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "MarketplaceExtensions", fmt.Sprintf("[%d]", i), "Scope"), + extension.Scope, + "field is immutable", + ), + ) + } + if (extension.ReleaseTrain != nil && oldExtension.ReleaseTrain != nil) && *extension.ReleaseTrain != *oldExtension.ReleaseTrain { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "MarketplaceExtensions", fmt.Sprintf("[%d]", i), "ReleaseTrain"), + extension.ReleaseTrain, + "field is immutable", + ), + ) + } + if (extension.Version != nil && oldExtension.Version != nil) && *extension.Version != *oldExtension.Version { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "MarketplaceExtensions", fmt.Sprintf("[%d]", i), "Version"), + extension.Version, + "field is immutable", + ), + ) + } + if extension.Identity != oldExtension.Identity { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "MarketplaceExtensions", fmt.Sprintf("[%d]", i), "Identity"), + extension.Identity, + "field is immutable", + ), + ) + } + } + + return allErrs +} + func validateName(name string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList if lName := strings.ToLower(name); strings.Contains(lName, "microsoft") || @@ -698,6 +788,44 @@ func validateName(name string, fldPath *field.Path) field.ErrorList { return allErrs } +// validateMarketplaceExtensions validates the Marketplace extensions. +func validateMarketplaceExtensions(extensions []MarketplaceExtension, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + for _, extension := range extensions { + if extension.Version != nil && (extension.AutoUpgradeMinorVersion == nil || (extension.AutoUpgradeMinorVersion != nil && *extension.AutoUpgradeMinorVersion)) { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("Version"), "Version must not be given if AutoUpgradeMinorVersion is true (or not provided, as it is true by default)")) + } + if extension.Plan.Product == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("Plan", "Product"), "Product must be provided")) + } + if extension.Plan.Publisher == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("Plan", "Publisher"), "Publisher must be provided")) + } + if extension.AutoUpgradeMinorVersion == ptr.To(false) && extension.ReleaseTrain != nil { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("ReleaseTrain"), "ReleaseTrain must not be given if AutoUpgradeMinorVersion is false")) + } + if extension.Scope != nil { + if extension.Scope.ScopeType == ExtensionScopeCluster { + if extension.Scope.ReleaseNamespace == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("Scope", "ReleaseNamespace"), "ReleaseNamespace must be provided if Scope is Cluster")) + } + if extension.Scope.TargetNamespace != "" { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("Scope", "TargetNamespace"), "TargetNamespace can only be given if Scope is Namespace")) + } + } else if extension.Scope.ScopeType == ExtensionScopeNamespace { + if extension.Scope.TargetNamespace == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("Scope", "TargetNamespace"), "TargetNamespace must be provided if Scope is Namespace")) + } + if extension.Scope.ReleaseNamespace != "" { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("Scope", "ReleaseNamespace"), "ReleaseNamespace can only be given if Scope is Cluster")) + } + } + } + } + + return allErrs +} + // validateAutoScalerProfile validates an AutoScalerProfile. func validateAutoScalerProfile(autoScalerProfile *AutoScalerProfile, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index 3c695a57a8ec..b95327561353 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -42,6 +42,15 @@ func TestDefaultingWebhook(t *testing.T) { AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ Location: "fooLocation", Version: "1.17.5", + MarketplaceExtensions: []MarketplaceExtension{ + { + Name: "test-extension", + Plan: &MarketplacePlan{ + Product: "test-product", + Publisher: "test-publisher", + }, + }, + }, }, ResourceGroupName: "fooRg", SSHPublicKey: ptr.To(""), @@ -64,6 +73,7 @@ func TestDefaultingWebhook(t *testing.T) { g.Expect(*amcp.Spec.OIDCIssuerProfile.Enabled).To(BeFalse()) g.Expect(amcp.Spec.DNSPrefix).ToNot(BeNil()) g.Expect(*amcp.Spec.DNSPrefix).To(Equal(amcp.Name)) + g.Expect(amcp.Spec.MarketplaceExtensions[0].Plan.Name).To(Equal("fooName-test-product")) t.Logf("Testing amcp defaulting webhook with baseline") netPlug := "kubenet" @@ -1108,6 +1118,77 @@ func TestValidatingWebhook(t *testing.T) { }, expectErr: false, }, + { + name: "Testing valid Marketplace Extension", + amcp: AzureManagedControlPlane{ + ObjectMeta: getAMCPMetaData(), + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + MarketplaceExtensions: []MarketplaceExtension{ + { + Name: "extension1", + ExtensionType: ptr.To("test-type"), + Plan: &MarketplacePlan{ + Name: "test-plan", + Product: "test-product", + Publisher: "test-publisher", + }, + }, + }, + }, + }, + }, + expectErr: false, + }, + { + name: "Testing invalid Marketplace Extension: version given when AutoUpgradeMinorVersion is true", + amcp: AzureManagedControlPlane{ + ObjectMeta: getAMCPMetaData(), + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + MarketplaceExtensions: []MarketplaceExtension{ + { + Name: "extension1", + ExtensionType: ptr.To("test-type"), + Version: ptr.To("1.0.0"), + AutoUpgradeMinorVersion: ptr.To(true), + Plan: &MarketplacePlan{ + Name: "test-plan", + Product: "test-product", + Publisher: "test-publisher", + }, + }, + }, + }, + }, + }, + expectErr: true, + }, + { + name: "Testing invalid Marketplace Extension: missing plan.product and plan.publisher", + amcp: AzureManagedControlPlane{ + ObjectMeta: getAMCPMetaData(), + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + MarketplaceExtensions: []MarketplaceExtension{ + { + Name: "extension1", + ExtensionType: ptr.To("test-type"), + Version: ptr.To("1.0.0"), + AutoUpgradeMinorVersion: ptr.To(true), + Plan: &MarketplacePlan{ + Name: "test-plan", + }, + }, + }, + }, + }, + }, + expectErr: true, + }, } for _, tt := range tests { @@ -2647,6 +2728,107 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: false, }, + { + name: "AzureManagedControlPlane MarketplaceExtensions ConfigurationSettings and AutoUpgradeMinorVersion are mutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + MarketplaceExtensions: []MarketplaceExtension{ + { + Name: "extension1", + AutoUpgradeMinorVersion: ptr.To(false), + ConfigurationSettings: map[string]string{ + "key1": "value1", + }, + Plan: &MarketplacePlan{ + Name: "planName", + Product: "planProduct", + Publisher: "planPublisher", + }, + }, + }, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + MarketplaceExtensions: []MarketplaceExtension{ + { + Name: "extension1", + AutoUpgradeMinorVersion: ptr.To(true), + ConfigurationSettings: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + Plan: &MarketplacePlan{ + Name: "planName", + Product: "planProduct", + Publisher: "planPublisher", + }, + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "AzureManagedControlPlane all other fields are immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + MarketplaceExtensions: []MarketplaceExtension{ + { + Name: "extension1", + AKSAssignedIdentityType: AKSAssignedIdentitySystemAssigned, + ExtensionType: ptr.To("extensionType"), + Plan: &MarketplacePlan{ + Name: "planName", + Product: "planProduct", + Publisher: "planPublisher", + }, + Scope: &ExtensionScope{ + ScopeType: "Cluster", + ReleaseNamespace: "default", + }, + ReleaseTrain: ptr.To("releaseTrain"), + Version: ptr.To("v1.0.0"), + }, + }, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + MarketplaceExtensions: []MarketplaceExtension{ + { + Name: "extension2", + AKSAssignedIdentityType: AKSAssignedIdentityUserAssigned, + ExtensionType: ptr.To("extensionType1"), + Plan: &MarketplacePlan{ + Name: "planName1", + Product: "planProduct1", + Publisher: "planPublisher1", + }, + Scope: &ExtensionScope{ + ScopeType: "Namespace", + ReleaseNamespace: "default", + }, + ReleaseTrain: ptr.To("releaseTrain1"), + Version: ptr.To("v1.1.0"), + }, + }, + }, + }, + }, + wantErr: true, + }, } client := mockClient{ReturnError: false} for _, tc := range tests { diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go index 245d18113e1b..aea9a10eac5b 100644 --- a/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go @@ -172,6 +172,10 @@ func (mcpw *azureManagedControlPlaneTemplateWebhook) ValidateUpdate(ctx context. allErrs = append(allErrs, errs...) } + if errs := validateMarketplaceExtensionsUpdate(old.Spec.Template.Spec.MarketplaceExtensions, mcp.Spec.Template.Spec.MarketplaceExtensions); len(errs) > 0 { + allErrs = append(allErrs, errs...) + } + if len(allErrs) == 0 { return nil, mcp.validateManagedControlPlaneTemplate(mcpw.Client) } @@ -203,6 +207,8 @@ func (mcp *AzureManagedControlPlaneTemplate) validateManagedControlPlaneTemplate allErrs = append(allErrs, validateAutoScalerProfile(mcp.Spec.Template.Spec.AutoScalerProfile, field.NewPath("spec").Child("template").Child("spec").Child("AutoScalerProfile"))...) + allErrs = append(allErrs, validateMarketplaceExtensions(mcp.Spec.Template.Spec.MarketplaceExtensions, field.NewPath("spec").Child("MarketplaceExtensions"))...) + return allErrs.ToAggregate() } diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go index 326d9c40eebd..f2fea92c864f 100644 --- a/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go @@ -186,6 +186,68 @@ func TestControlPlaneTemplateUpdateWebhook(t *testing.T) { }), wantErr: true, }, + { + name: "azuremanagedcontrolplanetemplate MarketplaceExtension type and plan are immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.MarketplaceExtensions = []MarketplaceExtension{ + { + Name: "foo", + ExtensionType: ptr.To("foo-type"), + Plan: &MarketplacePlan{ + Name: "foo-name", + Product: "foo-product", + Publisher: "foo-publisher", + }, + }, + } + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.MarketplaceExtensions = []MarketplaceExtension{ + { + Name: "foo", + ExtensionType: ptr.To("bar"), + Plan: &MarketplacePlan{ + Name: "bar-name", + Product: "bar-product", + Publisher: "bar-publisher", + }, + }, + } + }), + wantErr: true, + }, + { + name: "azuremanagedcontrolplanetemplate MarketplaceExtension autoUpgradeMinorVersion is mutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.MarketplaceExtensions = []MarketplaceExtension{ + { + Name: "foo", + ExtensionType: ptr.To("foo"), + AutoUpgradeMinorVersion: ptr.To(true), + Plan: &MarketplacePlan{ + Name: "bar-name", + Product: "bar-product", + Publisher: "bar-publisher", + }, + }, + } + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.MarketplaceExtensions = []MarketplaceExtension{ + { + Name: "foo", + ExtensionType: ptr.To("foo"), + AutoUpgradeMinorVersion: ptr.To(false), + Plan: &MarketplacePlan{ + Name: "bar-name", + Product: "bar-product", + Publisher: "bar-publisher", + }, + }, + } + }), + wantErr: false, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/api/v1beta1/consts.go b/api/v1beta1/consts.go index 590da87c172d..bf70aaa4bf20 100644 --- a/api/v1beta1/consts.go +++ b/api/v1beta1/consts.go @@ -130,6 +130,8 @@ const ( NetworkInterfaceReadyCondition clusterv1.ConditionType = "NetworkInterfacesReady" // PrivateEndpointsReadyCondition means the private endpoints exist and are ready to be used. PrivateEndpointsReadyCondition clusterv1.ConditionType = "PrivateEndpointsReady" + // AKSExtensionsReadyCondition means the AKS Extensions exist and are ready to be used. + AKSExtensionsReadyCondition clusterv1.ConditionType = "AKSExtensionsReady" // CreatingReason means the resource is being created. CreatingReason = "Creating" diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 4893df71a8aa..dc7ff48771bb 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -1090,3 +1090,72 @@ const ( // UniformOrchestrationMode treats VMs as identical instances accessible by the VMSS VM API. UniformOrchestrationMode OrchestrationModeType = "Uniform" ) + +// MarketplacePlan represents the plan for an AKS marketplace extension. +type MarketplacePlan struct { + // Name is the user-defined name of the 3rd Party Artifact that is being procured. + // +optional + Name string `json:"name,omitempty"` + + // Product is the name of the 3rd Party artifact that is being procured. + Product string `json:"product"` + + // PromotionCode is a publisher-provided promotion code as provisioned in Data Market for the said product/artifact. + // +optional + PromotionCode string `json:"promotionCode,omitempty"` + + // Publisher is the name of the publisher of the 3rd Party Artifact that is being bought. + Publisher string `json:"publisher"` + + // Version is the version of the plan. + // +optional + Version string `json:"version,omitempty"` +} + +// ExtensionScope defines the scope of the AKS marketplace extension, if configured. +type ExtensionScope struct { + // ScopeType is the scope of the extension. It can be either Cluster or Namespace, but not both. + ScopeType ExtensionScopeType `json:"scopeType"` + + // ReleaseNamespace is the namespace where the extension Release must be placed, for a Cluster-scoped extension. + // Required for Cluster-scoped extensions. + // +optional + ReleaseNamespace string `json:"releaseNamespace,omitempty"` + + // TargetNamespace is the namespace where the extension will be created for a Namespace-scoped extension. + // Required for Namespace-scoped extensions. + // +optional + TargetNamespace string `json:"targetNamespace,omitempty"` +} + +// ExtensionScopeType defines the scope type of the AKS marketplace extension, if configured. +// +kubebuilder:validation:Enum=Cluster;Namespace +type ExtensionScopeType string + +const ( + // ExtensionScopeCluster ... + ExtensionScopeCluster ExtensionScopeType = "Cluster" + // ExtensionScopeNamespace ... + ExtensionScopeNamespace ExtensionScopeType = "Namespace" +) + +// ExtensionIdentity defines the identity of the AKS marketplace extension, if configured. +// +kubebuilder:validation:Enum=SystemAssigned +type ExtensionIdentity string + +const ( + // ExtensionIdentitySystemAssigned ... + ExtensionIdentitySystemAssigned ExtensionIdentity = "SystemAssigned" +) + +// AKSAssignedIdentity defines the AKS assigned-identity of the aks marketplace extension, if configured. +// +kubebuilder:validation:Enum=SystemAssigned;UserAssigned +type AKSAssignedIdentity string + +const ( + // AKSAssignedIdentitySystemAssigned ... + AKSAssignedIdentitySystemAssigned AKSAssignedIdentity = "SystemAssigned" + + // AKSAssignedIdentityUserAssigned ... + AKSAssignedIdentityUserAssigned AKSAssignedIdentity = "UserAssigned" +) diff --git a/api/v1beta1/types_class.go b/api/v1beta1/types_class.go index 46913ab7be7a..106e6a9d38a0 100644 --- a/api/v1beta1/types_class.go +++ b/api/v1beta1/types_class.go @@ -211,6 +211,10 @@ type AzureManagedControlPlaneClassSpec struct { // DisableLocalAccounts disables getting static credentials for this cluster when set. Expected to only be used for AAD clusters. // +optional DisableLocalAccounts *bool `json:"disableLocalAccounts,omitempty"` + + // MarketplaceExtensions is a list of marketplace extensions to be installed on the cluster. + // +optional + MarketplaceExtensions []MarketplaceExtension `json:"marketplaceExtensions,omitempty"` } // AzureManagedMachinePoolClassSpec defines the AzureManagedMachinePool properties that may be shared across several Azure managed machinepools. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 31b37491e4be..ff87e5566dd0 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1346,6 +1346,13 @@ func (in *AzureManagedControlPlaneClassSpec) DeepCopyInto(out *AzureManagedContr *out = new(bool) **out = **in } + if in.MarketplaceExtensions != nil { + in, out := &in.MarketplaceExtensions, &out.MarketplaceExtensions + *out = make([]MarketplaceExtension, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneClassSpec. @@ -2213,6 +2220,21 @@ func (in *ExtendedLocationSpec) DeepCopy() *ExtendedLocationSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExtensionScope) DeepCopyInto(out *ExtensionScope) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtensionScope. +func (in *ExtensionScope) DeepCopy() *ExtensionScope { + if in == nil { + return nil + } + out := new(ExtensionScope) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FrontendIP) DeepCopyInto(out *FrontendIP) { *out = *in @@ -2703,6 +2725,73 @@ func (in *ManagedMachinePoolScaling) DeepCopy() *ManagedMachinePoolScaling { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MarketplaceExtension) DeepCopyInto(out *MarketplaceExtension) { + *out = *in + if in.AutoUpgradeMinorVersion != nil { + in, out := &in.AutoUpgradeMinorVersion, &out.AutoUpgradeMinorVersion + *out = new(bool) + **out = **in + } + if in.ConfigurationSettings != nil { + in, out := &in.ConfigurationSettings, &out.ConfigurationSettings + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ExtensionType != nil { + in, out := &in.ExtensionType, &out.ExtensionType + *out = new(string) + **out = **in + } + if in.Plan != nil { + in, out := &in.Plan, &out.Plan + *out = new(MarketplacePlan) + **out = **in + } + if in.ReleaseTrain != nil { + in, out := &in.ReleaseTrain, &out.ReleaseTrain + *out = new(string) + **out = **in + } + if in.Scope != nil { + in, out := &in.Scope, &out.Scope + *out = new(ExtensionScope) + **out = **in + } + if in.Version != nil { + in, out := &in.Version, &out.Version + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MarketplaceExtension. +func (in *MarketplaceExtension) DeepCopy() *MarketplaceExtension { + if in == nil { + return nil + } + out := new(MarketplaceExtension) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MarketplacePlan) DeepCopyInto(out *MarketplacePlan) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MarketplacePlan. +func (in *MarketplacePlan) DeepCopy() *MarketplacePlan { + if in == nil { + return nil + } + out := new(MarketplacePlan) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NatGateway) DeepCopyInto(out *NatGateway) { *out = *in diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index bdc2e383b751..32f38624cdee 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -24,6 +24,7 @@ import ( "time" asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001" + asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501" asonetworkv1api20201101 "github.com/Azure/azure-service-operator/v2/api/network/v1api20201101" asonetworkv1api20220701 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601" @@ -38,6 +39,7 @@ import ( "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/aksextensions" "sigs.k8s.io/cluster-api-provider-azure/azure/services/groups" "sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters" "sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints" @@ -888,3 +890,35 @@ func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ASOResourceSpe func (s *ManagedControlPlaneScope) SetOIDCIssuerProfileStatus(oidc *infrav1.OIDCIssuerProfileStatus) { s.ControlPlane.Status.OIDCIssuerProfile = oidc } + +// AKSExtension returns the cluster AKS extensions. +func (s *ManagedControlPlaneScope) AKSExtension() []infrav1.MarketplaceExtension { + return s.ControlPlane.Spec.MarketplaceExtensions +} + +// AKSExtensionSpecs returns the AKS extension specs. +func (s *ManagedControlPlaneScope) AKSExtensionSpecs() []azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension] { + if s.AKSExtension() == nil { + return nil + } + extensionSpecs := make([]azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension], 0, len(s.ControlPlane.Spec.MarketplaceExtensions)) + for _, extension := range s.AKSExtension() { + extensionSpec := &aksextensions.AKSExtensionSpec{ + Name: extension.Name, + Namespace: s.Cluster.Namespace, + AutoUpgradeMinorVersion: extension.AutoUpgradeMinorVersion, + ConfigurationSettings: extension.ConfigurationSettings, + ExtensionType: extension.ExtensionType, + ReleaseTrain: extension.ReleaseTrain, + Version: extension.Version, + Owner: azure.ManagedClusterID(s.SubscriptionID(), s.ResourceGroup(), s.ControlPlane.Name), + Plan: *extension.Plan, + AKSAssignedIdentityType: extension.AKSAssignedIdentityType, + ExtensionIdentity: extension.Identity, + } + + extensionSpecs = append(extensionSpecs, extensionSpec) + } + + return extensionSpecs +} diff --git a/azure/scope/managedcontrolplane_test.go b/azure/scope/managedcontrolplane_test.go index 68bd056c607c..9dc6bcc229ee 100644 --- a/azure/scope/managedcontrolplane_test.go +++ b/azure/scope/managedcontrolplane_test.go @@ -23,6 +23,7 @@ import ( aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1" asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001" + asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501" asonetworkv1 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -32,6 +33,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/agentpools" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/aksextensions" "sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters" "sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -1443,3 +1445,107 @@ func TestManagedControlPlaneScope_PrivateEndpointSpecs(t *testing.T) { }) } } + +func TestManagedControlPlaneScope_AKSExtensionSpecs(t *testing.T) { + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected []azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension] + Err string + }{ + { + Name: "returns empty AKS extensions list if no extensions are specified", + Input: ManagedControlPlaneScopeParams{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "dummy-ns", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: "dummy-ns", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{}, + }, + }, + }, + }, + { + Name: "returns list of AKS extensions if extensions are specified", + Input: ManagedControlPlaneScopeParams{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: "dummy-ns", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: "dummy-ns", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + MarketplaceExtensions: []infrav1.MarketplaceExtension{ + { + Name: "my-extension", + AutoUpgradeMinorVersion: ptr.To(true), + ConfigurationSettings: map[string]string{ + "my-key": "my-value", + }, + ExtensionType: ptr.To("my-extension-type"), + ReleaseTrain: ptr.To("my-release-train"), + Version: ptr.To("my-version"), + Plan: &infrav1.MarketplacePlan{ + Name: "my-plan-name", + Product: "my-product", + Publisher: "my-publisher", + }, + AKSAssignedIdentityType: infrav1.AKSAssignedIdentitySystemAssigned, + Identity: infrav1.ExtensionIdentitySystemAssigned, + }, + }, + }, + }, + }, + }, + Expected: []azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension]{ + &aksextensions.AKSExtensionSpec{ + Name: "my-extension", + Namespace: "dummy-ns", + AutoUpgradeMinorVersion: ptr.To(true), + ConfigurationSettings: map[string]string{ + "my-key": "my-value", + }, + ExtensionType: ptr.To("my-extension-type"), + ReleaseTrain: ptr.To("my-release-train"), + Version: ptr.To("my-version"), + Owner: "/subscriptions//resourceGroups//providers/Microsoft.ContainerService/managedClusters/my-cluster", + Plan: infrav1.MarketplacePlan{ + Name: "my-plan-name", + Product: "my-product", + Publisher: "my-publisher", + }, + AKSAssignedIdentityType: infrav1.AKSAssignedIdentitySystemAssigned, + ExtensionIdentity: infrav1.ExtensionIdentitySystemAssigned, + }, + }, + }, + } + + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + s := &ManagedControlPlaneScope{ + ControlPlane: c.Input.ControlPlane, + Cluster: c.Input.Cluster, + } + if got := s.AKSExtensionSpecs(); !reflect.DeepEqual(got, c.Expected) { + t.Errorf("AKSExtensionSpecs() = %s, want %s", specArrayToString(got), specArrayToString(c.Expected)) + } + }) + } +} diff --git a/azure/services/aksextensions/aksextensions.go b/azure/services/aksextensions/aksextensions.go new file mode 100644 index 000000000000..9bff9a082ba1 --- /dev/null +++ b/azure/services/aksextensions/aksextensions.go @@ -0,0 +1,50 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aksextensions + +import ( + asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/aso" +) + +const serviceName = "extension" + +// AKSExtensionScope defines the scope interface for an AKS extensions service. +type AKSExtensionScope interface { + azure.ClusterScoper + aso.Scope + AKSExtensionSpecs() []azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension] +} + +// Service provides operations on Azure resources. +type Service struct { + Scope AKSExtensionScope + *aso.Service[*asokubernetesconfigurationv1.Extension, AKSExtensionScope] +} + +// New creates a new service. +func New(scope AKSExtensionScope) *Service { + svc := aso.NewService[*asokubernetesconfigurationv1.Extension, AKSExtensionScope](serviceName, scope) + svc.Specs = scope.AKSExtensionSpecs() + svc.ConditionType = infrav1.AKSExtensionsReadyCondition + return &Service{ + Scope: scope, + Service: svc, + } +} diff --git a/azure/services/aksextensions/spec.go b/azure/services/aksextensions/spec.go new file mode 100644 index 000000000000..7414820d9bca --- /dev/null +++ b/azure/services/aksextensions/spec.go @@ -0,0 +1,110 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aksextensions + +import ( + "context" + + asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501" + "github.com/Azure/azure-service-operator/v2/pkg/genruntime" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" +) + +// AKSExtensionSpec defines the specification for an AKS Extension. +type AKSExtensionSpec struct { + Name string + Namespace string + AKSAssignedIdentityType infrav1.AKSAssignedIdentity + ExtensionIdentity infrav1.ExtensionIdentity + AutoUpgradeMinorVersion *bool + ConfigurationSettings map[string]string + ExtensionType *string + ReleaseTrain *string + Version *string + Owner string + OwnerRef metav1.OwnerReference + Plan infrav1.MarketplacePlan + Scope infrav1.ExtensionScope +} + +// ResourceRef implements azure.ASOResourceSpecGetter. +func (s *AKSExtensionSpec) ResourceRef() *asokubernetesconfigurationv1.Extension { + return &asokubernetesconfigurationv1.Extension{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.Name, + Namespace: s.Namespace, + }, + } +} + +// Parameters implements azure.ASOResourceSpecGetter. +func (s *AKSExtensionSpec) Parameters(ctx context.Context, existingAKSExtension *asokubernetesconfigurationv1.Extension) (parameters *asokubernetesconfigurationv1.Extension, err error) { + aksExtension := &asokubernetesconfigurationv1.Extension{} + if existingAKSExtension != nil { + aksExtension = existingAKSExtension + } + + aksExtension.Spec = asokubernetesconfigurationv1.Extension_Spec{} + aksExtension.Spec.AzureName = s.Name + aksExtension.Spec.AutoUpgradeMinorVersion = s.AutoUpgradeMinorVersion + aksExtension.Spec.ConfigurationSettings = s.ConfigurationSettings + aksExtension.Spec.ExtensionType = s.ExtensionType + aksExtension.Spec.ReleaseTrain = s.ReleaseTrain + aksExtension.Spec.Version = s.Version + aksExtension.Spec.Owner = &genruntime.ArbitraryOwnerReference{ + ARMID: s.Owner, + } + aksExtension.Spec.Plan = &asokubernetesconfigurationv1.Plan{ + Name: ptr.To(s.Plan.Name), + Product: ptr.To(s.Plan.Product), + Publisher: ptr.To(s.Plan.Publisher), + Version: ptr.To(s.Plan.Version), + } + if s.ExtensionIdentity != "" { + aksExtension.Spec.Identity = &asokubernetesconfigurationv1.Identity{ + Type: (*asokubernetesconfigurationv1.Identity_Type)(ptr.To(s.ExtensionIdentity)), + } + } + if s.AKSAssignedIdentityType != "" { + aksExtension.Spec.AksAssignedIdentity = &asokubernetesconfigurationv1.Extension_Properties_AksAssignedIdentity_Spec{ + Type: (*asokubernetesconfigurationv1.Extension_Properties_AksAssignedIdentity_Type_Spec)(ptr.To(s.AKSAssignedIdentityType)), + } + } + if s.Scope.ScopeType == infrav1.ExtensionScopeCluster { + aksExtension.Spec.Scope = &asokubernetesconfigurationv1.Scope{ + Cluster: &asokubernetesconfigurationv1.ScopeCluster{ + ReleaseNamespace: ptr.To(s.Scope.ReleaseNamespace), + }, + } + } else if s.Scope.ScopeType == infrav1.ExtensionScopeNamespace { + aksExtension.Spec.Scope = &asokubernetesconfigurationv1.Scope{ + Namespace: &asokubernetesconfigurationv1.ScopeNamespace{ + TargetNamespace: ptr.To(s.Scope.TargetNamespace), + }, + } + } + + return aksExtension, nil +} + +// WasManaged implements azure.ASOResourceSpecGetter. +func (s *AKSExtensionSpec) WasManaged(resource *asokubernetesconfigurationv1.Extension) bool { + // returns always returns true as CAPZ does not support BYO extension. + return true +} diff --git a/azure/services/aksextensions/spec_test.go b/azure/services/aksextensions/spec_test.go new file mode 100644 index 000000000000..28d660618edd --- /dev/null +++ b/azure/services/aksextensions/spec_test.go @@ -0,0 +1,155 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aksextensions + +import ( + "context" + "testing" + + asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501" + "github.com/Azure/azure-service-operator/v2/pkg/genruntime" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" +) + +var ( + fakeAKSExtension = asokubernetesconfigurationv1.Extension{ + Spec: asokubernetesconfigurationv1.Extension_Spec{ + AksAssignedIdentity: &asokubernetesconfigurationv1.Extension_Properties_AksAssignedIdentity_Spec{ + Type: (*asokubernetesconfigurationv1.Extension_Properties_AksAssignedIdentity_Type_Spec)(&fakeAKSExtensionSpec1.AKSAssignedIdentityType), + }, + AzureName: fakeAKSExtensionSpec1.Name, + AutoUpgradeMinorVersion: ptr.To(true), + ConfigurationSettings: map[string]string{ + "fake-key": "fake-value", + }, + ExtensionType: fakeAKSExtensionSpec1.ExtensionType, + Owner: &genruntime.ArbitraryOwnerReference{ + ARMID: fakeAKSExtensionSpec1.Owner, + }, + Plan: &asokubernetesconfigurationv1.Plan{ + Name: ptr.To(fakeAKSExtensionSpec1.Plan.Name), + Product: ptr.To(fakeAKSExtensionSpec1.Plan.Product), + Publisher: ptr.To(fakeAKSExtensionSpec1.Plan.Publisher), + Version: ptr.To(fakeAKSExtensionSpec1.Plan.Version), + }, + ReleaseTrain: fakeAKSExtensionSpec1.ReleaseTrain, + Version: fakeAKSExtensionSpec1.Version, + Identity: &asokubernetesconfigurationv1.Identity{ + Type: (*asokubernetesconfigurationv1.Identity_Type)(&fakeAKSExtensionSpec1.ExtensionIdentity), + }, + }, + } + fakeAKSExtensionSpec1 = AKSExtensionSpec{ + Name: "fake-aks-extension", + Namespace: "fake-namespace", + AKSAssignedIdentityType: "SystemAssigned", + AutoUpgradeMinorVersion: ptr.To(true), + ConfigurationSettings: map[string]string{ + "fake-key": "fake-value", + }, + ExtensionType: ptr.To("fake-extension-type"), + ReleaseTrain: ptr.To("fake-release-train"), + Version: ptr.To("fake-version"), + Owner: "fake-owner", + Plan: infrav1.MarketplacePlan{ + Name: "fake-plan-name", + }, + ExtensionIdentity: "SystemAssigned", + } + fakeAKSExtensionStatus = asokubernetesconfigurationv1.Extension_STATUS{ + Name: ptr.To(fakeAKSExtensionSpec1.Name), + ProvisioningState: ptr.To(asokubernetesconfigurationv1.ProvisioningStateDefinition_STATUS_Succeeded), + } +) + +func getASOAKSExtension(changes ...func(*asokubernetesconfigurationv1.Extension)) *asokubernetesconfigurationv1.Extension { + aksExtension := fakeAKSExtension.DeepCopy() + for _, change := range changes { + change(aksExtension) + } + return aksExtension +} + +func TestAzureAKSExtensionSpec_Parameters(t *testing.T) { + testcases := []struct { + name string + spec *AKSExtensionSpec + existing *asokubernetesconfigurationv1.Extension + expect func(g *WithT, result asokubernetesconfigurationv1.Extension) + expectedError string + }{ + { + name: "Creating a new AKS Extension", + spec: &fakeAKSExtensionSpec1, + existing: nil, + expect: func(g *WithT, result asokubernetesconfigurationv1.Extension) { + g.Expect(result).To(Not(BeNil())) + + // ObjectMeta is populated later in the codeflow + g.Expect(result.ObjectMeta).To(Equal(metav1.ObjectMeta{})) + + // Spec is populated from the spec passed in + g.Expect(result.Spec).To(Equal(getASOAKSExtension().Spec)) + }, + }, + { + name: "user updates to AKS Extension resource and capz should overwrite it", + spec: &fakeAKSExtensionSpec1, + existing: getASOAKSExtension( + // user added AutoUpgradeMinorVersion which should be overwritten by capz + func(aksExtension *asokubernetesconfigurationv1.Extension) { + aksExtension.Spec.AutoUpgradeMinorVersion = ptr.To(false) + }, + // user added Status + func(aksExtension *asokubernetesconfigurationv1.Extension) { + aksExtension.Status = fakeAKSExtensionStatus + }, + ), + expect: func(g *WithT, result asokubernetesconfigurationv1.Extension) { + g.Expect(result).To(Not(BeNil())) + + // ObjectMeta is populated later in the codeflow + g.Expect(result.ObjectMeta).To(Equal(metav1.ObjectMeta{})) + + // Spec is populated from the spec passed in + g.Expect(result.Spec).To(Equal(getASOAKSExtension().Spec)) + + // Status should be carried over + g.Expect(result.Status).To(Equal(fakeAKSExtensionStatus)) + }, + }, + } + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + result, err := tc.spec.Parameters(context.TODO(), tc.existing) + if tc.expectedError != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(tc.expectedError)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + tc.expect(g, *result) + }) + } +} diff --git a/config/aso/crds.yaml b/config/aso/crds.yaml index df73499929c1..c225a27787cf 100644 --- a/config/aso/crds.yaml +++ b/config/aso/crds.yaml @@ -535,6 +535,880 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: azureserviceoperator-system/azureserviceoperator-serving-cert + controller-gen.kubebuilder.io/version: v0.13.0 + labels: + app.kubernetes.io/name: azure-service-operator + app.kubernetes.io/version: v2.5.0 + name: extensions.kubernetesconfiguration.azure.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: azureserviceoperator-webhook-service + namespace: azureserviceoperator-system + path: /convert + port: 443 + conversionReviewVersions: + - v1 + group: kubernetesconfiguration.azure.com + names: + kind: Extension + listKind: ExtensionList + plural: extensions + singular: extension + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].severity + name: Severity + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + type: string + name: v1api20230501 + schema: + openAPIV3Schema: + description: 'Generator information: - Generated from: /kubernetesconfiguration/resource-manager/Microsoft.KubernetesConfiguration/stable/2023-05-01/extensions.json - ARM URI: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{clusterRp}/{clusterResourceName}/{clusterName}/providers/Microsoft.KubernetesConfiguration/extensions/{extensionName}' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + aksAssignedIdentity: + description: 'AksAssignedIdentity: Identity of the Extension resource in an AKS cluster' + properties: + type: + description: 'Type: The identity type.' + enum: + - SystemAssigned + - UserAssigned + type: string + type: object + autoUpgradeMinorVersion: + description: 'AutoUpgradeMinorVersion: Flag to note if this extension participates in auto upgrade of minor version, or not.' + type: boolean + azureName: + description: 'AzureName: The name of the resource in Azure. This is often the same as the name of the resource in Kubernetes but it doesn''t have to be.' + type: string + configurationSettings: + additionalProperties: + type: string + description: 'ConfigurationSettings: Configuration settings, as name-value pairs for configuring this extension.' + type: object + extensionType: + description: 'ExtensionType: Type of the Extension, of which this resource is an instance of. It must be one of the Extension Types registered with Microsoft.KubernetesConfiguration by the Extension publisher.' + type: string + identity: + description: 'Identity: Identity of the Extension resource' + properties: + type: + description: 'Type: The identity type.' + enum: + - SystemAssigned + type: string + type: object + owner: + description: 'Owner: The owner of the resource. The owner controls where the resource goes when it is deployed. The owner also controls the resources lifecycle. When the owner is deleted the resource will also be deleted. This resource is an extension resource, which means that any other Azure resource can be its owner.' + properties: + armId: + description: Ownership across namespaces is not supported. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: This is the name of the Kubernetes resource to reference. + type: string + type: object + plan: + description: 'Plan: The plan information.' + properties: + name: + description: 'Name: A user defined name of the 3rd Party Artifact that is being procured.' + type: string + product: + description: 'Product: The 3rd Party artifact that is being procured. E.g. NewRelic. Product maps to the OfferID specified for the artifact at the time of Data Market onboarding.' + type: string + promotionCode: + description: 'PromotionCode: A publisher provided promotion code as provisioned in Data Market for the said product/artifact.' + type: string + publisher: + description: 'Publisher: The publisher of the 3rd Party Artifact that is being bought. E.g. NewRelic' + type: string + version: + description: 'Version: The version of the desired product/artifact.' + type: string + required: + - name + - product + - publisher + type: object + releaseTrain: + description: 'ReleaseTrain: ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is ''true''.' + type: string + scope: + description: 'Scope: Scope at which the extension is installed.' + properties: + cluster: + description: 'Cluster: Specifies that the scope of the extension is Cluster' + properties: + releaseNamespace: + description: 'ReleaseNamespace: Namespace where the extension Release must be placed, for a Cluster scoped extension. If this namespace does not exist, it will be created' + type: string + type: object + namespace: + description: 'Namespace: Specifies that the scope of the extension is Namespace' + properties: + targetNamespace: + description: 'TargetNamespace: Namespace where the extension will be created for an Namespace scoped extension. If this namespace does not exist, it will be created' + type: string + type: object + type: object + systemData: + description: 'SystemData: Top level metadata https://github.com/Azure/azure-resource-manager-rpc/blob/master/v1.0/common-api-contracts.md#system-metadata-for-all-azure-resources' + properties: + createdAt: + description: 'CreatedAt: The timestamp of resource creation (UTC).' + type: string + createdBy: + description: 'CreatedBy: The identity that created the resource.' + type: string + createdByType: + description: 'CreatedByType: The type of identity that created the resource.' + enum: + - Application + - Key + - ManagedIdentity + - User + type: string + lastModifiedAt: + description: 'LastModifiedAt: The timestamp of resource last modification (UTC)' + type: string + lastModifiedBy: + description: 'LastModifiedBy: The identity that last modified the resource.' + type: string + lastModifiedByType: + description: 'LastModifiedByType: The type of identity that last modified the resource.' + enum: + - Application + - Key + - ManagedIdentity + - User + type: string + type: object + version: + description: 'Version: User-specified version of the extension for this extension to ''pin''. To use ''version'', autoUpgradeMinorVersion must be ''false''.' + type: string + required: + - owner + type: object + status: + description: The Extension object. + properties: + aksAssignedIdentity: + description: 'AksAssignedIdentity: Identity of the Extension resource in an AKS cluster' + properties: + principalId: + description: 'PrincipalId: The principal ID of resource identity.' + type: string + tenantId: + description: 'TenantId: The tenant ID of resource.' + type: string + type: + description: 'Type: The identity type.' + type: string + type: object + autoUpgradeMinorVersion: + description: 'AutoUpgradeMinorVersion: Flag to note if this extension participates in auto upgrade of minor version, or not.' + type: boolean + conditions: + description: 'Conditions: The observed state of the resource' + items: + description: Condition defines an extension to status (an observation) of a resource + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: Message is a human readable message indicating details about the transition. This field may be empty. + type: string + observedGeneration: + description: ObservedGeneration is the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + type: integer + reason: + description: Reason for the condition's last transition. Reasons are upper CamelCase (PascalCase) with no spaces. A reason is always provided, this field will not be empty. + type: string + severity: + description: Severity with which to treat failures of this type of condition. For conditions which have positive polarity (Status == True is their normal/healthy state), this will be omitted when Status == True For conditions which have negative polarity (Status == False is their normal/healthy state), this will be omitted when Status == False. This is omitted in all cases when Status == Unknown + type: string + status: + description: Status of the condition, one of True, False, or Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - lastTransitionTime + - reason + - status + - type + type: object + type: array + configurationSettings: + additionalProperties: + type: string + description: 'ConfigurationSettings: Configuration settings, as name-value pairs for configuring this extension.' + type: object + currentVersion: + description: 'CurrentVersion: Currently installed version of the extension.' + type: string + customLocationSettings: + additionalProperties: + type: string + description: 'CustomLocationSettings: Custom Location settings properties.' + type: object + errorInfo: + description: 'ErrorInfo: Error information from the Agent - e.g. errors during installation.' + properties: + additionalInfo: + description: 'AdditionalInfo: The error additional info.' + items: + description: The resource management error additional info. + properties: + info: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + description: 'Info: The additional info.' + type: object + type: + description: 'Type: The additional info type.' + type: string + type: object + type: array + code: + description: 'Code: The error code.' + type: string + details: + description: 'Details: The error details.' + items: + properties: + additionalInfo: + description: 'AdditionalInfo: The error additional info.' + items: + description: The resource management error additional info. + properties: + info: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + description: 'Info: The additional info.' + type: object + type: + description: 'Type: The additional info type.' + type: string + type: object + type: array + code: + description: 'Code: The error code.' + type: string + message: + description: 'Message: The error message.' + type: string + target: + description: 'Target: The error target.' + type: string + type: object + type: array + message: + description: 'Message: The error message.' + type: string + target: + description: 'Target: The error target.' + type: string + type: object + extensionType: + description: 'ExtensionType: Type of the Extension, of which this resource is an instance of. It must be one of the Extension Types registered with Microsoft.KubernetesConfiguration by the Extension publisher.' + type: string + id: + description: 'Id: Fully qualified resource ID for the resource. Ex - /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}' + type: string + identity: + description: 'Identity: Identity of the Extension resource' + properties: + principalId: + description: 'PrincipalId: The principal ID of resource identity.' + type: string + tenantId: + description: 'TenantId: The tenant ID of resource.' + type: string + type: + description: 'Type: The identity type.' + type: string + type: object + isSystemExtension: + description: 'IsSystemExtension: Flag to note if this extension is a system extension' + type: boolean + name: + description: 'Name: The name of the resource' + type: string + packageUri: + description: 'PackageUri: Uri of the Helm package' + type: string + plan: + description: 'Plan: The plan information.' + properties: + name: + description: 'Name: A user defined name of the 3rd Party Artifact that is being procured.' + type: string + product: + description: 'Product: The 3rd Party artifact that is being procured. E.g. NewRelic. Product maps to the OfferID specified for the artifact at the time of Data Market onboarding.' + type: string + promotionCode: + description: 'PromotionCode: A publisher provided promotion code as provisioned in Data Market for the said product/artifact.' + type: string + publisher: + description: 'Publisher: The publisher of the 3rd Party Artifact that is being bought. E.g. NewRelic' + type: string + version: + description: 'Version: The version of the desired product/artifact.' + type: string + type: object + provisioningState: + description: 'ProvisioningState: Status of installation of this extension.' + type: string + releaseTrain: + description: 'ReleaseTrain: ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is ''true''.' + type: string + scope: + description: 'Scope: Scope at which the extension is installed.' + properties: + cluster: + description: 'Cluster: Specifies that the scope of the extension is Cluster' + properties: + releaseNamespace: + description: 'ReleaseNamespace: Namespace where the extension Release must be placed, for a Cluster scoped extension. If this namespace does not exist, it will be created' + type: string + type: object + namespace: + description: 'Namespace: Specifies that the scope of the extension is Namespace' + properties: + targetNamespace: + description: 'TargetNamespace: Namespace where the extension will be created for an Namespace scoped extension. If this namespace does not exist, it will be created' + type: string + type: object + type: object + statuses: + description: 'Statuses: Status from this extension.' + items: + description: Status from the extension. + properties: + code: + description: 'Code: Status code provided by the Extension' + type: string + displayStatus: + description: 'DisplayStatus: Short description of status of the extension.' + type: string + level: + description: 'Level: Level of the status.' + type: string + message: + description: 'Message: Detailed message of the status from the Extension.' + type: string + time: + description: 'Time: DateLiteral (per ISO8601) noting the time of installation status.' + type: string + type: object + type: array + systemData: + description: 'SystemData: Top level metadata https://github.com/Azure/azure-resource-manager-rpc/blob/master/v1.0/common-api-contracts.md#system-metadata-for-all-azure-resources' + properties: + createdAt: + description: 'CreatedAt: The timestamp of resource creation (UTC).' + type: string + createdBy: + description: 'CreatedBy: The identity that created the resource.' + type: string + createdByType: + description: 'CreatedByType: The type of identity that created the resource.' + type: string + lastModifiedAt: + description: 'LastModifiedAt: The timestamp of resource last modification (UTC)' + type: string + lastModifiedBy: + description: 'LastModifiedBy: The identity that last modified the resource.' + type: string + lastModifiedByType: + description: 'LastModifiedByType: The type of identity that last modified the resource.' + type: string + type: object + type: + description: 'Type: The type of the resource. E.g. "Microsoft.Compute/virtualMachines" or "Microsoft.Storage/storageAccounts"' + type: string + version: + description: 'Version: User-specified version of the extension for this extension to ''pin''. To use ''version'', autoUpgradeMinorVersion must be ''false''.' + type: string + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].severity + name: Severity + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + type: string + name: v1api20230501storage + schema: + openAPIV3Schema: + description: 'Storage version of v1api20230501.Extension Generator information: - Generated from: /kubernetesconfiguration/resource-manager/Microsoft.KubernetesConfiguration/stable/2023-05-01/extensions.json - ARM URI: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{clusterRp}/{clusterResourceName}/{clusterName}/providers/Microsoft.KubernetesConfiguration/extensions/{extensionName}' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Storage version of v1api20230501.Extension_Spec + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + aksAssignedIdentity: + description: Storage version of v1api20230501.Extension_Properties_AksAssignedIdentity_Spec + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + type: + type: string + type: object + autoUpgradeMinorVersion: + type: boolean + azureName: + description: 'AzureName: The name of the resource in Azure. This is often the same as the name of the resource in Kubernetes but it doesn''t have to be.' + type: string + configurationSettings: + additionalProperties: + type: string + type: object + extensionType: + type: string + identity: + description: Storage version of v1api20230501.Identity Identity for the resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + type: + type: string + type: object + originalVersion: + type: string + owner: + description: 'Owner: The owner of the resource. The owner controls where the resource goes when it is deployed. The owner also controls the resources lifecycle. When the owner is deleted the resource will also be deleted. This resource is an extension resource, which means that any other Azure resource can be its owner.' + properties: + armId: + description: Ownership across namespaces is not supported. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: This is the name of the Kubernetes resource to reference. + type: string + type: object + plan: + description: Storage version of v1api20230501.Plan Plan for the resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + name: + type: string + product: + type: string + promotionCode: + type: string + publisher: + type: string + version: + type: string + type: object + releaseTrain: + type: string + scope: + description: Storage version of v1api20230501.Scope Scope of the extension. It can be either Cluster or Namespace; but not both. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + cluster: + description: Storage version of v1api20230501.ScopeCluster Specifies that the scope of the extension is Cluster + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + releaseNamespace: + type: string + type: object + namespace: + description: Storage version of v1api20230501.ScopeNamespace Specifies that the scope of the extension is Namespace + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + targetNamespace: + type: string + type: object + type: object + systemData: + description: Storage version of v1api20230501.SystemData Metadata pertaining to creation and last modification of the resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + createdAt: + type: string + createdBy: + type: string + createdByType: + type: string + lastModifiedAt: + type: string + lastModifiedBy: + type: string + lastModifiedByType: + type: string + type: object + version: + type: string + required: + - owner + type: object + status: + description: Storage version of v1api20230501.Extension_STATUS The Extension object. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + aksAssignedIdentity: + description: Storage version of v1api20230501.Extension_Properties_AksAssignedIdentity_STATUS + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + principalId: + type: string + tenantId: + type: string + type: + type: string + type: object + autoUpgradeMinorVersion: + type: boolean + conditions: + items: + description: Condition defines an extension to status (an observation) of a resource + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: Message is a human readable message indicating details about the transition. This field may be empty. + type: string + observedGeneration: + description: ObservedGeneration is the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + type: integer + reason: + description: Reason for the condition's last transition. Reasons are upper CamelCase (PascalCase) with no spaces. A reason is always provided, this field will not be empty. + type: string + severity: + description: Severity with which to treat failures of this type of condition. For conditions which have positive polarity (Status == True is their normal/healthy state), this will be omitted when Status == True For conditions which have negative polarity (Status == False is their normal/healthy state), this will be omitted when Status == False. This is omitted in all cases when Status == Unknown + type: string + status: + description: Status of the condition, one of True, False, or Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - lastTransitionTime + - reason + - status + - type + type: object + type: array + configurationSettings: + additionalProperties: + type: string + type: object + currentVersion: + type: string + customLocationSettings: + additionalProperties: + type: string + type: object + errorInfo: + description: Storage version of v1api20230501.ErrorDetail_STATUS The error detail. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + additionalInfo: + items: + description: Storage version of v1api20230501.ErrorAdditionalInfo_STATUS The resource management error additional info. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + info: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + type: object + type: + type: string + type: object + type: array + code: + type: string + details: + items: + description: Storage version of v1api20230501.ErrorDetail_STATUS_Unrolled + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + additionalInfo: + items: + description: Storage version of v1api20230501.ErrorAdditionalInfo_STATUS The resource management error additional info. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + info: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + type: object + type: + type: string + type: object + type: array + code: + type: string + message: + type: string + target: + type: string + type: object + type: array + message: + type: string + target: + type: string + type: object + extensionType: + type: string + id: + type: string + identity: + description: Storage version of v1api20230501.Identity_STATUS Identity for the resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + principalId: + type: string + tenantId: + type: string + type: + type: string + type: object + isSystemExtension: + type: boolean + name: + type: string + packageUri: + type: string + plan: + description: Storage version of v1api20230501.Plan_STATUS Plan for the resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + name: + type: string + product: + type: string + promotionCode: + type: string + publisher: + type: string + version: + type: string + type: object + provisioningState: + type: string + releaseTrain: + type: string + scope: + description: Storage version of v1api20230501.Scope_STATUS Scope of the extension. It can be either Cluster or Namespace; but not both. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + cluster: + description: Storage version of v1api20230501.ScopeCluster_STATUS Specifies that the scope of the extension is Cluster + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + releaseNamespace: + type: string + type: object + namespace: + description: Storage version of v1api20230501.ScopeNamespace_STATUS Specifies that the scope of the extension is Namespace + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + targetNamespace: + type: string + type: object + type: object + statuses: + items: + description: Storage version of v1api20230501.ExtensionStatus_STATUS Status from the extension. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + code: + type: string + displayStatus: + type: string + level: + type: string + message: + type: string + time: + type: string + type: object + type: array + systemData: + description: Storage version of v1api20230501.SystemData_STATUS Metadata pertaining to creation and last modification of the resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + createdAt: + type: string + createdBy: + type: string + createdByType: + type: string + lastModifiedAt: + type: string + lastModifiedBy: + type: string + lastModifiedByType: + type: string + type: object + type: + type: string + version: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: azureserviceoperator-system/azureserviceoperator-serving-cert diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index 345f8dfd7bb8..19cc145e85fc 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -385,6 +385,111 @@ spec: the AzureManagedControlPlaneTemplate, this field is used only to fulfill the CAPI contract. type: object + marketplaceExtensions: + description: MarketplaceExtensions is a list of marketplace extensions + to be installed on the cluster. + items: + description: "MarketplaceExtension represents the configuration + for a marketplace extension. See also [AKS doc]. \n [AKS doc]: + https://learn.microsoft.com/en-us/azure/aks/cluster-extensions" + properties: + aksAssignedIdentityType: + description: AKSAssignedIdentityType is the type of the AKS + assigned identity. + enum: + - SystemAssigned + - UserAssigned + type: string + autoUpgradeMinorVersion: + default: true + description: AutoUpgradeMinorVersion is a flag to note if this + extension participates in auto upgrade of minor version, or + not. + type: boolean + configurationSettings: + additionalProperties: + type: string + description: ConfigurationSettings are the name-value pairs + for configuring this extension. + type: object + extensionType: + description: ExtensionType is the type of the Extension of which + this resource is an instance. It must be one of the Extension + Types registered with Microsoft.KubernetesConfiguration by + the Extension publisher. + type: string + identity: + description: Identity is the identity type of the Extension + resource in an AKS cluster. + enum: + - SystemAssigned + type: string + name: + description: Name is the name of the extension. + type: string + plan: + description: Plan is the plan of the extension. + properties: + name: + description: Name is the user-defined name of the 3rd Party + Artifact that is being procured. + type: string + product: + description: Product is the name of the 3rd Party artifact + that is being procured. + type: string + promotionCode: + description: PromotionCode is a publisher-provided promotion + code as provisioned in Data Market for the said product/artifact. + type: string + publisher: + description: Publisher is the name of the publisher of the + 3rd Party Artifact that is being bought. + type: string + version: + description: Version is the version of the plan. + type: string + required: + - product + - publisher + type: object + releaseTrain: + description: ReleaseTrain is the release train this extension + participates in for auto-upgrade (e.g. Stable, Preview, etc.) + This is only used if autoUpgradeMinorVersion is ‘true’. + type: string + scope: + description: Scope is the scope at which this extension is enabled. + properties: + releaseNamespace: + description: ReleaseNamespace is the namespace where the + extension Release must be placed, for a Cluster-scoped + extension. Required for Cluster-scoped extensions. + type: string + scopeType: + description: ScopeType is the scope of the extension. It + can be either Cluster or Namespace, but not both. + enum: + - Cluster + - Namespace + type: string + targetNamespace: + description: TargetNamespace is the namespace where the + extension will be created for a Namespace-scoped extension. + Required for Namespace-scoped extensions. + type: string + required: + - scopeType + type: object + version: + description: Version is the version of the extension. + type: string + required: + - extensionType + - name + - plan + type: object + type: array networkPlugin: description: NetworkPlugin used for building Kubernetes network. enum: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml index f92a346175f4..e2522406980e 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml @@ -394,6 +394,115 @@ spec: plane. For the AzureManagedControlPlaneTemplate, this field is used only to fulfill the CAPI contract. type: object + marketplaceExtensions: + description: MarketplaceExtensions is a list of marketplace + extensions to be installed on the cluster. + items: + description: "MarketplaceExtension represents the configuration + for a marketplace extension. See also [AKS doc]. \n [AKS + doc]: https://learn.microsoft.com/en-us/azure/aks/cluster-extensions" + properties: + aksAssignedIdentityType: + description: AKSAssignedIdentityType is the type of + the AKS assigned identity. + enum: + - SystemAssigned + - UserAssigned + type: string + autoUpgradeMinorVersion: + default: true + description: AutoUpgradeMinorVersion is a flag to note + if this extension participates in auto upgrade of + minor version, or not. + type: boolean + configurationSettings: + additionalProperties: + type: string + description: ConfigurationSettings are the name-value + pairs for configuring this extension. + type: object + extensionType: + description: ExtensionType is the type of the Extension + of which this resource is an instance. It must be + one of the Extension Types registered with Microsoft.KubernetesConfiguration + by the Extension publisher. + type: string + identity: + description: Identity is the identity type of the Extension + resource in an AKS cluster. + enum: + - SystemAssigned + type: string + name: + description: Name is the name of the extension. + type: string + plan: + description: Plan is the plan of the extension. + properties: + name: + description: Name is the user-defined name of the + 3rd Party Artifact that is being procured. + type: string + product: + description: Product is the name of the 3rd Party + artifact that is being procured. + type: string + promotionCode: + description: PromotionCode is a publisher-provided + promotion code as provisioned in Data Market for + the said product/artifact. + type: string + publisher: + description: Publisher is the name of the publisher + of the 3rd Party Artifact that is being bought. + type: string + version: + description: Version is the version of the plan. + type: string + required: + - product + - publisher + type: object + releaseTrain: + description: ReleaseTrain is the release train this + extension participates in for auto-upgrade (e.g. Stable, + Preview, etc.) This is only used if autoUpgradeMinorVersion + is ‘true’. + type: string + scope: + description: Scope is the scope at which this extension + is enabled. + properties: + releaseNamespace: + description: ReleaseNamespace is the namespace where + the extension Release must be placed, for a Cluster-scoped + extension. Required for Cluster-scoped extensions. + type: string + scopeType: + description: ScopeType is the scope of the extension. + It can be either Cluster or Namespace, but not + both. + enum: + - Cluster + - Namespace + type: string + targetNamespace: + description: TargetNamespace is the namespace where + the extension will be created for a Namespace-scoped + extension. Required for Namespace-scoped extensions. + type: string + required: + - scopeType + type: object + version: + description: Version is the version of the extension. + type: string + required: + - extensionType + - name + - plan + type: object + type: array networkPlugin: description: NetworkPlugin used for building Kubernetes network. enum: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6d98042d89fd..cdf447a4922a 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -313,6 +313,26 @@ rules: - get - patch - update +- apiGroups: + - kubernetesconfiguration.azure.com + resources: + - extensions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubernetesconfiguration.azure.com + resources: + - extensions/status + verbs: + - get + - list + - watch - apiGroups: - network.azure.com resources: diff --git a/controllers/azuremanagedcontrolplane_controller.go b/controllers/azuremanagedcontrolplane_controller.go index edc001b6031d..4a9739ef5711 100644 --- a/controllers/azuremanagedcontrolplane_controller.go +++ b/controllers/azuremanagedcontrolplane_controller.go @@ -118,6 +118,8 @@ func (amcpr *AzureManagedControlPlaneReconciler) SetupWithManager(ctx context.Co // +kubebuilder:rbac:groups=containerservice.azure.com,resources=managedclusters/status,verbs=get;list;watch // +kubebuilder:rbac:groups=network.azure.com,resources=privateendpoints;virtualnetworks;virtualnetworkssubnets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=network.azure.com,resources=privateendpoints/status;virtualnetworks/status;virtualnetworkssubnets/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=kubernetesconfiguration.azure.com,resources=extensions,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=kubernetesconfiguration.azure.com,resources=extensions/status,verbs=get;list;watch // Reconcile idempotently gets, creates, and updates a managed control plane. func (amcpr *AzureManagedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { diff --git a/controllers/azuremanagedcontrolplane_reconciler.go b/controllers/azuremanagedcontrolplane_reconciler.go index bd8ca504844e..1ecbc7767f39 100644 --- a/controllers/azuremanagedcontrolplane_reconciler.go +++ b/controllers/azuremanagedcontrolplane_reconciler.go @@ -24,6 +24,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/scope" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/aksextensions" "sigs.k8s.io/cluster-api-provider-azure/azure/services/groups" "sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters" "sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints" @@ -58,6 +59,7 @@ func newAzureManagedControlPlaneReconciler(scope *scope.ManagedControlPlaneScope subnets.New(scope), managedclusters.New(scope), privateendpoints.New(scope), + aksextensions.New(scope), resourceHealthSvc, }, }, nil diff --git a/go.mod b/go.mod index b78b3ab55f8b..3d08b56d8f9e 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( require ( github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/kubernetesconfiguration/armkubernetesconfiguration v1.1.1 github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect diff --git a/go.sum b/go.sum index 8fe3697b23d8..a81fba5e2d9b 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventgrid/armeventgrid v1. github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/iothub/armiothub v1.3.0 h1:NZP+oPbAVFy7PhQ4PTD3SuGWbEziNhp7lphGkkN707s= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0 h1:HlZMUZW8S4P9oob1nCHxCCKrytxyLc+24nUJGssoEto= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/kubernetesconfiguration/armkubernetesconfiguration v1.1.1 h1:eAVG12uoHrqI8yiSrCtuQoLnKYYL5bnLrFbnP8lL+SE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/kubernetesconfiguration/armkubernetesconfiguration v1.1.1/go.mod h1:NLG7km6cZfoAnpkUypZQjAIIRADzps6vIusTTUE8rrE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/machinelearning/armmachinelearning v1.0.0 h1:KWvCVjnOTKCZAlqED5KPNoN9AfcK2BhUeveLdiwy33Q= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 h1:Ds0KRF8ggpEGg4Vo42oX1cIt/IfOhHWJBikksZbVxeg= diff --git a/main.go b/main.go index 24d1b0593fc7..c3d6c04a714a 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,7 @@ import ( // +kubebuilder:scaffold:imports aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1" asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001" + asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501" asonetworkv1api20201101 "github.com/Azure/azure-service-operator/v2/api/network/v1api20201101" asonetworkv1api20220701 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601" @@ -82,6 +83,7 @@ func init() { _ = asocontainerservicev1.AddToScheme(scheme) _ = asonetworkv1api20220701.AddToScheme(scheme) _ = asonetworkv1api20201101.AddToScheme(scheme) + _ = asokubernetesconfigurationv1.AddToScheme(scheme) // +kubebuilder:scaffold:scheme // Add aadpodidentity v1 to the scheme. diff --git a/test/e2e/aks_marketplace.go b/test/e2e/aks_marketplace.go new file mode 100644 index 000000000000..53e3f94cb53e --- /dev/null +++ b/test/e2e/aks_marketplace.go @@ -0,0 +1,179 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/kubernetesconfiguration/armkubernetesconfiguration" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type AKSMarketplaceExtensionSpecInput struct { + Cluster *clusterv1.Cluster + WaitIntervals []interface{} +} + +const ( + extensionName = "aks-marketplace-extension" +) + +func AKSMarketplaceExtensionSpec(ctx context.Context, inputGetter func() AKSMarketplaceExtensionSpecInput) { + input := inputGetter() + + cred, err := azidentity.NewDefaultAzureCredential(nil) + Expect(err).NotTo(HaveOccurred()) + + mgmtClient := bootstrapClusterProxy.GetClient() + Expect(mgmtClient).NotTo(BeNil()) + + amcp := &infrav1.AzureManagedControlPlane{} + err = mgmtClient.Get(ctx, types.NamespacedName{ + Namespace: input.Cluster.Spec.ControlPlaneRef.Namespace, + Name: input.Cluster.Spec.ControlPlaneRef.Name, + }, amcp) + Expect(err).NotTo(HaveOccurred()) + + agentpoolsClient, err := armcontainerservice.NewAgentPoolsClient(amcp.Spec.SubscriptionID, cred, nil) + Expect(err).NotTo(HaveOccurred()) + + extensionClient, err := armkubernetesconfiguration.NewExtensionsClient(amcp.Spec.SubscriptionID, cred, nil) + Expect(err).NotTo(HaveOccurred()) + + By("Deleting all node taints for windows machine pool") + var ammp = &infrav1.AzureManagedMachinePool{} + Expect(mgmtClient.Get(ctx, types.NamespacedName{ + Namespace: input.Cluster.Namespace, + Name: input.Cluster.Name + "-pool2", + }, ammp)).To(Succeed()) + initialTaints := ammp.Spec.Taints + var expectedTaints []infrav1.Taint + expectedTaints = nil + checkTaints := func(g Gomega) { + var expectedTaintStrs []*string + if expectedTaints != nil { + expectedTaintStrs = make([]*string, 0, len(expectedTaints)) + for _, taint := range expectedTaints { + expectedTaintStrs = append(expectedTaintStrs, ptr.To(fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect))) + } + } + + resp, err := agentpoolsClient.Get(ctx, amcp.Spec.ResourceGroupName, amcp.Name, *ammp.Spec.Name, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(resp.Properties.ProvisioningState).To(Equal(ptr.To("Succeeded"))) + actualTaintStrs := resp.AgentPool.Properties.NodeTaints + if expectedTaintStrs == nil { + g.Expect(actualTaintStrs).To(BeNil()) + } else { + g.Expect(actualTaintStrs).To(Equal(expectedTaintStrs)) + } + } + Eventually(func(g Gomega) { + g.Expect(mgmtClient.Get(ctx, client.ObjectKeyFromObject(ammp), ammp)).To(Succeed()) + ammp.Spec.Taints = expectedTaints + g.Expect(mgmtClient.Update(ctx, ammp)).To(Succeed()) + }, inputGetter().WaitIntervals...).Should(Succeed()) + Eventually(checkTaints, input.WaitIntervals...).Should(Succeed()) + + By("Adding a taint to the Windows node pool") + expectedTaints = []infrav1.Taint{ + { + Effect: infrav1.TaintEffect(corev1.TaintEffectNoSchedule), + Key: "capz-e2e-1", + Value: "test1", + }, + } + Eventually(func(g Gomega) { + g.Expect(mgmtClient.Get(ctx, client.ObjectKeyFromObject(ammp), ammp)).To(Succeed()) + ammp.Spec.Taints = expectedTaints + g.Expect(mgmtClient.Update(ctx, ammp)).To(Succeed()) + Eventually(checkTaints, input.WaitIntervals...).Should(Succeed()) + }, input.WaitIntervals...).Should(Succeed()) + + By("Updating taints for Windows machine pool") + expectedTaints = expectedTaints[:1] + Eventually(func(g Gomega) { + g.Expect(mgmtClient.Get(ctx, client.ObjectKeyFromObject(ammp), ammp)).To(Succeed()) + ammp.Spec.Taints = expectedTaints + g.Expect(mgmtClient.Update(ctx, ammp)).To(Succeed()) + }, input.WaitIntervals...).Should(Succeed()) + Eventually(checkTaints, input.WaitIntervals...).Should(Succeed()) + + By("Adding an AKS Marketplace Extension to the AzureManagedControlPlane") + var infraControlPlane = &infrav1.AzureManagedControlPlane{} + Eventually(func(g Gomega) { + err = mgmtClient.Get(ctx, client.ObjectKey{ + Namespace: input.Cluster.Spec.ControlPlaneRef.Namespace, + Name: input.Cluster.Spec.ControlPlaneRef.Name, + }, infraControlPlane) + g.Expect(err).NotTo(HaveOccurred()) + infraControlPlane.Spec.MarketplaceExtensions = []infrav1.MarketplaceExtension{ + { + Name: extensionName, + ExtensionType: ptr.To("TraefikLabs.TraefikProxy"), + Plan: &infrav1.MarketplacePlan{ + Name: "traefik-proxy", + Product: "traefik-proxy", + Publisher: "containous", + }, + }, + } + g.Expect(mgmtClient.Update(ctx, infraControlPlane)).To(Succeed()) + }, input.WaitIntervals...).Should(Succeed()) + + By("Ensuring the AKS Marketplace Extension status is ready on the AzureManagedControlPlane") + Eventually(func(g Gomega) { + err = mgmtClient.Get(ctx, client.ObjectKey{Namespace: input.Cluster.Spec.ControlPlaneRef.Namespace, Name: input.Cluster.Spec.ControlPlaneRef.Name}, infraControlPlane) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(conditions.IsTrue(infraControlPlane, infrav1.AKSExtensionsReadyCondition)).To(BeTrue()) + }, input.WaitIntervals...).Should(Succeed()) + + By("Ensuring the AKS Marketplace Extension is added to the AzureManagedControlPlane") + Eventually(func(g Gomega) { + resp, err := extensionClient.Get(ctx, amcp.Spec.ResourceGroupName, "Microsoft.ContainerService", "managedClusters", input.Cluster.Name, extensionName, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(resp.Properties.ProvisioningState).To(Equal(ptr.To(armkubernetesconfiguration.ProvisioningStateSucceeded))) + extension := resp.Extension + g.Expect(extension.Properties).NotTo(BeNil()) + g.Expect(extension.Name).To(Equal(ptr.To(extensionName))) + g.Expect(extension.Properties.AutoUpgradeMinorVersion).To(Equal(ptr.To(true))) + g.Expect(extension.Properties.ExtensionType).To(Equal(ptr.To("TraefikLabs.TraefikProxy"))) + }, input.WaitIntervals...).Should(Succeed()) + + By("Restoring initial taints for Windows machine pool") + expectedTaints = initialTaints + Eventually(func(g Gomega) { + g.Expect(mgmtClient.Get(ctx, client.ObjectKeyFromObject(ammp), ammp)).To(Succeed()) + ammp.Spec.Taints = expectedTaints + g.Expect(mgmtClient.Update(ctx, ammp)).To(Succeed()) + }, input.WaitIntervals...).Should(Succeed()) + Eventually(checkTaints, input.WaitIntervals...).Should(Succeed()) +} diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 5c3039cd97cf..b6d23cd52ad3 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -844,6 +844,15 @@ var _ = Describe("Workload cluster creation", func() { } }) }) + + By("adding a marketplace extension", func() { + AKSMarketplaceExtensionSpec(ctx, func() AKSMarketplaceExtensionSpecInput { + return AKSMarketplaceExtensionSpecInput{ + Cluster: result.Cluster, + WaitIntervals: e2eConfig.GetIntervals(specName, "wait-machine-pool-nodes"), + } + }) + }) }) })