From 224b7dc26ac523579f4296261808ab861135bd6f Mon Sep 17 00:00:00 2001 From: willie-yao Date: Wed, 18 Oct 2023 21:46:59 +0000 Subject: [PATCH] Add ClusterClass support for Managed Clusters --- .../azuremanagedclustertemplate_types.go | 56 + .../azuremanagedclustertemplate_webhook.go | 61 + .../azuremanagedcontrolplane_default.go | 112 +- .../azuremanagedcontrolplane_default_test.go | 88 +- api/v1beta1/azuremanagedcontrolplane_types.go | 143 +- .../azuremanagedcontrolplane_webhook.go | 287 ++- .../azuremanagedcontrolplane_webhook_test.go | 1563 ++++++++++++----- ...zuremanagedcontrolplanetemplate_default.go | 69 + ...anagedcontrolplanetemplate_default_test.go | 326 ++++ .../azuremanagedcontrolplanetemplate_types.go | 56 + ...zuremanagedcontrolplanetemplate_webhook.go | 289 +++ ...anagedcontrolplanetemplate_webhook_test.go | 370 ++++ api/v1beta1/azuremanagedmachinepool_types.go | 148 +- .../azuremanagedmachinepool_webhook.go | 205 ++- .../azuremanagedmachinepool_webhook_test.go | 683 ++++--- .../azuremanagedmachinepooltemplate_types.go | 56 + ...azuremanagedmachinepooltemplate_webhook.go | 287 +++ ...managedmachinepooltemplate_webhook_test.go | 269 +++ api/v1beta1/consts.go | 2 + api/v1beta1/types_class.go | 297 ++++ api/v1beta1/types_template.go | 17 + api/v1beta1/zz_generated.deepcopy.go | 495 +++++- azure/defaults.go | 5 + azure/scope/managedcontrolplane_test.go | 148 +- azure/scope/managedmachinepool_test.go | 100 +- ...x-k8s.io_azuremanagedclustertemplates.yaml | 60 + ...er.x-k8s.io_azuremanagedcontrolplanes.yaml | 42 +- ....io_azuremanagedcontrolplanetemplates.yaml | 594 +++++++ ...ter.x-k8s.io_azuremanagedmachinepools.yaml | 12 +- ...s.io_azuremanagedmachinepooltemplates.yaml | 584 ++++++ config/crd/kustomization.yaml | 3 + config/webhook/manifests.yaml | 105 ++ .../azuremanagedcontrolplane_controller.go | 1 + ...zuremanagedcontrolplane_controller_test.go | 4 +- controllers/helpers.go | 4 + controllers/helpers_test.go | 22 +- docs/book/src/topics/clusterclass.md | 79 + main.go | 15 + .../cluster-template-aks-clusterclass.yaml | 125 ++ templates/cluster-template-aks-topology.yaml | 21 + templates/cluster-template-clusterclass.yaml | 24 - templates/cluster-template-topology.yaml | 23 + .../azure-managed-cluster-template.yaml | 8 + .../azure-managed-controlplane-template.yaml | 15 + .../azure-managed-machinepool-template.yaml | 23 + .../aks-clusterclass/clusterclass.yaml | 43 + .../kubeadm-config-template.yaml | 17 + .../aks-clusterclass/kustomization.yaml | 10 + .../managedazurecluster-identity-ref.yaml | 9 + templates/flavors/aks-topology/cluster.yaml | 21 + .../flavors/aks-topology/kustomization.yaml | 5 + .../flavors/clusterclass/kustomization.yaml | 1 - .../{clusterclass => topology}/cluster.yaml | 0 templates/flavors/topology/kustomization.yaml | 3 + ...luster-template-prow-aks-clusterclass.yaml | 273 +++ .../prow-aks-clusterclass/kustomization.yaml | 14 + .../ci/prow-aks-clusterclass/patches.yaml | 54 + .../prow-aks-clusterclass/patches/addons.yaml | 11 + .../patches/aks-clusterclass-pool0.yaml | 15 + .../patches/aks-clusterclass-pool1.yaml | 67 + .../patches/cluster.yaml | 7 + .../patches/kubeadm-config-template.yaml | 45 + .../patches/tags-aks-clusterclass.yaml | 12 + .../test/ci/prow-topology/kustomization.yaml | 2 +- test/e2e/aks.go | 4 +- test/e2e/aks_clusterclass.go | 127 ++ test/e2e/aks_public_ip_prefix.go | 11 +- test/e2e/aks_spot.go | 14 +- test/e2e/aks_upgrade.go | 1 + test/e2e/azure_test.go | 50 +- test/e2e/common.go | 4 +- test/e2e/config/azure-dev.yaml | 22 +- test/e2e/data/shared/v1beta1/metadata.yaml | 3 + 73 files changed, 7216 insertions(+), 1525 deletions(-) create mode 100644 api/v1beta1/azuremanagedclustertemplate_types.go create mode 100644 api/v1beta1/azuremanagedclustertemplate_webhook.go create mode 100644 api/v1beta1/azuremanagedcontrolplanetemplate_default.go create mode 100644 api/v1beta1/azuremanagedcontrolplanetemplate_default_test.go create mode 100644 api/v1beta1/azuremanagedcontrolplanetemplate_types.go create mode 100644 api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go create mode 100644 api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go create mode 100644 api/v1beta1/azuremanagedmachinepooltemplate_types.go create mode 100644 api/v1beta1/azuremanagedmachinepooltemplate_webhook.go create mode 100644 api/v1beta1/azuremanagedmachinepooltemplate_webhook_test.go create mode 100644 config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedclustertemplates.yaml create mode 100644 config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml create mode 100644 config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepooltemplates.yaml create mode 100644 docs/book/src/topics/clusterclass.md create mode 100644 templates/cluster-template-aks-clusterclass.yaml create mode 100644 templates/cluster-template-aks-topology.yaml create mode 100644 templates/cluster-template-topology.yaml create mode 100644 templates/flavors/aks-clusterclass/azure-managed-cluster-template.yaml create mode 100644 templates/flavors/aks-clusterclass/azure-managed-controlplane-template.yaml create mode 100644 templates/flavors/aks-clusterclass/azure-managed-machinepool-template.yaml create mode 100644 templates/flavors/aks-clusterclass/clusterclass.yaml create mode 100644 templates/flavors/aks-clusterclass/kubeadm-config-template.yaml create mode 100644 templates/flavors/aks-clusterclass/kustomization.yaml create mode 100644 templates/flavors/aks-clusterclass/patches/managedazurecluster-identity-ref.yaml create mode 100644 templates/flavors/aks-topology/cluster.yaml create mode 100644 templates/flavors/aks-topology/kustomization.yaml rename templates/flavors/{clusterclass => topology}/cluster.yaml (100%) create mode 100644 templates/flavors/topology/kustomization.yaml create mode 100644 templates/test/ci/cluster-template-prow-aks-clusterclass.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/kustomization.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/patches.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/patches/addons.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool0.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool1.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/patches/cluster.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/patches/kubeadm-config-template.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/patches/tags-aks-clusterclass.yaml create mode 100644 test/e2e/aks_clusterclass.go diff --git a/api/v1beta1/azuremanagedclustertemplate_types.go b/api/v1beta1/azuremanagedclustertemplate_types.go new file mode 100644 index 00000000000..c26afc7f93d --- /dev/null +++ b/api/v1beta1/azuremanagedclustertemplate_types.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// AzureManagedClusterTemplateSpec defines the desired state of AzureManagedClusterTemplate. +type AzureManagedClusterTemplateSpec struct { + Template AzureManagedClusterTemplateResource `json:"template"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=azuremanagedclustertemplates,scope=Namespaced,categories=cluster-api,shortName=amct +// +kubebuilder:storageversion + +// AzureManagedClusterTemplate is the Schema for the AzureManagedClusterTemplates API. +type AzureManagedClusterTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AzureManagedClusterTemplateSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// AzureManagedClusterTemplateList contains a list of AzureManagedClusterTemplates. +type AzureManagedClusterTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AzureManagedClusterTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&AzureManagedClusterTemplate{}, &AzureManagedClusterTemplateList{}) +} + +// AzureManagedClusterTemplateResource describes the data needed to create an AzureManagedCluster from a template. +type AzureManagedClusterTemplateResource struct { + Spec AzureManagedClusterTemplateResourceSpec `json:"spec"` +} diff --git a/api/v1beta1/azuremanagedclustertemplate_webhook.go b/api/v1beta1/azuremanagedclustertemplate_webhook.go new file mode 100644 index 00000000000..048d79adf06 --- /dev/null +++ b/api/v1beta1/azuremanagedclustertemplate_webhook.go @@ -0,0 +1,61 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/cluster-api-provider-azure/feature" + capifeature "sigs.k8s.io/cluster-api/feature" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// SetupWebhookWithManager sets up and registers the webhook with the manager. +func (r *AzureManagedClusterTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// +kubebuilder:webhook:verbs=update,path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedclustertemplate,mutating=false,failurePolicy=fail,groups=infrastructure.cluster.x-k8s.io,resources=azuremanagedclustertemplates,versions=v1beta1,name=validation.azuremanagedclustertemplates.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 + +var _ webhook.Validator = &AzureManagedClusterTemplate{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (r *AzureManagedClusterTemplate) ValidateCreate() (admission.Warnings, error) { + // NOTE: AzureManagedClusterTemplate relies upon MachinePools, which is behind a feature gate flag. + // The webhook must prevent creating new objects in case the feature flag is disabled. + if !feature.Gates.Enabled(capifeature.MachinePool) { + return nil, field.Forbidden( + field.NewPath("spec"), + "cannot be set if the Cluster API 'MachinePool' feature flag is not enabled", + ) + } + return nil, nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (r *AzureManagedClusterTemplate) ValidateUpdate(_ runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (r *AzureManagedClusterTemplate) ValidateDelete() (admission.Warnings, error) { + return nil, nil +} diff --git a/api/v1beta1/azuremanagedcontrolplane_default.go b/api/v1beta1/azuremanagedcontrolplane_default.go index 9e2940f64fc..f4b0cd4f4c6 100644 --- a/api/v1beta1/azuremanagedcontrolplane_default.go +++ b/api/v1beta1/azuremanagedcontrolplane_default.go @@ -19,10 +19,13 @@ package v1beta1 import ( "encoding/base64" "fmt" + "strings" "golang.org/x/crypto/ssh" "k8s.io/utils/ptr" utilSSH "sigs.k8s.io/cluster-api-provider-azure/util/ssh" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" ) const ( @@ -36,6 +39,15 @@ const ( defaultAKSNodeSubnetCIDRForOverlay = "10.224.0.0/16" ) +// setDefaultResourceGroupName sets the default ResourceGroupName for an AzureManagedControlPlane. +func (m *AzureManagedControlPlane) setDefaultResourceGroupName() { + if m.Spec.ResourceGroupName == "" { + if clusterName, ok := m.Labels[clusterv1.ClusterNameLabel]; ok { + m.Spec.ResourceGroupName = clusterName + } + } +} + // setDefaultSSHPublicKey sets the default SSHPublicKey for an AzureManagedControlPlane. func (m *AzureManagedControlPlane) setDefaultSSHPublicKey() error { if sshKey := m.Spec.SSHPublicKey; sshKey != nil && *sshKey == "" { @@ -86,73 +98,89 @@ func (m *AzureManagedControlPlane) setDefaultSubnet() { } } -func (m *AzureManagedControlPlane) setDefaultSku() { - if m.Spec.SKU == nil { - m.Spec.SKU = &AKSSku{ - Tier: FreeManagedControlPlaneTier, - } +func setDefaultSku(sku *AKSSku) *AKSSku { + result := sku.DeepCopy() + if sku == nil { + result = new(AKSSku) + result.Tier = FreeManagedControlPlaneTier + } else if sku.Tier == PaidManagedControlPlaneTier { + result.Tier = StandardManagedControlPlaneTier + ctrl.Log.WithName("AzureManagedControlPlaneWebHookLogger").Info("Paid SKU tier is deprecated and has been replaced by Standard") } + return result } -func (m *AzureManagedControlPlane) setDefaultAutoScalerProfile() { - if m.Spec.AutoScalerProfile == nil { - return +func setDefaultVersion(version string) string { + if version != "" && !strings.HasPrefix(version, "v") { + normalizedVersion := "v" + version + version = normalizedVersion + } + return version +} + +func setDefaultAutoScalerProfile(autoScalerProfile *AutoScalerProfile) *AutoScalerProfile { + if autoScalerProfile == nil { + return nil } + result := autoScalerProfile.DeepCopy() + // Default values are from https://learn.microsoft.com/en-us/azure/aks/cluster-autoscaler#using-the-autoscaler-profile // If any values are set, they all need to be set. - if m.Spec.AutoScalerProfile.BalanceSimilarNodeGroups == nil { - m.Spec.AutoScalerProfile.BalanceSimilarNodeGroups = (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsFalse))) + if autoScalerProfile.BalanceSimilarNodeGroups == nil { + result.BalanceSimilarNodeGroups = (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsFalse))) } - if m.Spec.AutoScalerProfile.Expander == nil { - m.Spec.AutoScalerProfile.Expander = (*Expander)(ptr.To(string(ExpanderRandom))) + if autoScalerProfile.Expander == nil { + result.Expander = (*Expander)(ptr.To(string(ExpanderRandom))) } - if m.Spec.AutoScalerProfile.MaxEmptyBulkDelete == nil { - m.Spec.AutoScalerProfile.MaxEmptyBulkDelete = ptr.To("10") + if autoScalerProfile.MaxEmptyBulkDelete == nil { + result.MaxEmptyBulkDelete = ptr.To("10") } - if m.Spec.AutoScalerProfile.MaxGracefulTerminationSec == nil { - m.Spec.AutoScalerProfile.MaxGracefulTerminationSec = ptr.To("600") + if autoScalerProfile.MaxGracefulTerminationSec == nil { + result.MaxGracefulTerminationSec = ptr.To("600") } - if m.Spec.AutoScalerProfile.MaxNodeProvisionTime == nil { - m.Spec.AutoScalerProfile.MaxNodeProvisionTime = ptr.To("15m") + if autoScalerProfile.MaxNodeProvisionTime == nil { + result.MaxNodeProvisionTime = ptr.To("15m") } - if m.Spec.AutoScalerProfile.MaxTotalUnreadyPercentage == nil { - m.Spec.AutoScalerProfile.MaxTotalUnreadyPercentage = ptr.To("45") + if autoScalerProfile.MaxTotalUnreadyPercentage == nil { + result.MaxTotalUnreadyPercentage = ptr.To("45") } - if m.Spec.AutoScalerProfile.NewPodScaleUpDelay == nil { - m.Spec.AutoScalerProfile.NewPodScaleUpDelay = ptr.To("0s") + if autoScalerProfile.NewPodScaleUpDelay == nil { + result.NewPodScaleUpDelay = ptr.To("0s") } - if m.Spec.AutoScalerProfile.OkTotalUnreadyCount == nil { - m.Spec.AutoScalerProfile.OkTotalUnreadyCount = ptr.To("3") + if autoScalerProfile.OkTotalUnreadyCount == nil { + result.OkTotalUnreadyCount = ptr.To("3") } - if m.Spec.AutoScalerProfile.ScanInterval == nil { - m.Spec.AutoScalerProfile.ScanInterval = ptr.To("10s") + if autoScalerProfile.ScanInterval == nil { + result.ScanInterval = ptr.To("10s") } - if m.Spec.AutoScalerProfile.ScaleDownDelayAfterAdd == nil { - m.Spec.AutoScalerProfile.ScaleDownDelayAfterAdd = ptr.To("10m") + if autoScalerProfile.ScaleDownDelayAfterAdd == nil { + result.ScaleDownDelayAfterAdd = ptr.To("10m") } - if m.Spec.AutoScalerProfile.ScaleDownDelayAfterDelete == nil { + if autoScalerProfile.ScaleDownDelayAfterDelete == nil { // Default is the same as the ScanInterval so default to that same value if it isn't set - m.Spec.AutoScalerProfile.ScaleDownDelayAfterDelete = m.Spec.AutoScalerProfile.ScanInterval + result.ScaleDownDelayAfterDelete = result.ScanInterval } - if m.Spec.AutoScalerProfile.ScaleDownDelayAfterFailure == nil { - m.Spec.AutoScalerProfile.ScaleDownDelayAfterFailure = ptr.To("3m") + if autoScalerProfile.ScaleDownDelayAfterFailure == nil { + result.ScaleDownDelayAfterFailure = ptr.To("3m") } - if m.Spec.AutoScalerProfile.ScaleDownUnneededTime == nil { - m.Spec.AutoScalerProfile.ScaleDownUnneededTime = ptr.To("10m") + if autoScalerProfile.ScaleDownUnneededTime == nil { + result.ScaleDownUnneededTime = ptr.To("10m") } - if m.Spec.AutoScalerProfile.ScaleDownUnreadyTime == nil { - m.Spec.AutoScalerProfile.ScaleDownUnreadyTime = ptr.To("20m") + if autoScalerProfile.ScaleDownUnreadyTime == nil { + result.ScaleDownUnreadyTime = ptr.To("20m") } - if m.Spec.AutoScalerProfile.ScaleDownUtilizationThreshold == nil { - m.Spec.AutoScalerProfile.ScaleDownUtilizationThreshold = ptr.To("0.5") + if autoScalerProfile.ScaleDownUtilizationThreshold == nil { + result.ScaleDownUtilizationThreshold = ptr.To("0.5") } - if m.Spec.AutoScalerProfile.SkipNodesWithLocalStorage == nil { - m.Spec.AutoScalerProfile.SkipNodesWithLocalStorage = (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageFalse))) + if autoScalerProfile.SkipNodesWithLocalStorage == nil { + result.SkipNodesWithLocalStorage = (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageFalse))) } - if m.Spec.AutoScalerProfile.SkipNodesWithSystemPods == nil { - m.Spec.AutoScalerProfile.SkipNodesWithSystemPods = (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsTrue))) + if autoScalerProfile.SkipNodesWithSystemPods == nil { + result.SkipNodesWithSystemPods = (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsTrue))) } + + return result } func (m *AzureManagedControlPlane) setDefaultOIDCIssuerProfile() { diff --git a/api/v1beta1/azuremanagedcontrolplane_default_test.go b/api/v1beta1/azuremanagedcontrolplane_default_test.go index 1058b3f4c62..ae9eca5415b 100644 --- a/api/v1beta1/azuremanagedcontrolplane_default_test.go +++ b/api/v1beta1/azuremanagedcontrolplane_default_test.go @@ -64,69 +64,77 @@ func TestSetDefaultAutoScalerProfile(t *testing.T) { defaultAMP := &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - AutoScalerProfile: &AutoScalerProfile{ - BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsFalse))), - Expander: (*Expander)(ptr.To(string(ExpanderRandom))), - MaxEmptyBulkDelete: ptr.To("10"), - MaxGracefulTerminationSec: ptr.To("600"), - MaxNodeProvisionTime: ptr.To("15m"), - MaxTotalUnreadyPercentage: ptr.To("45"), - NewPodScaleUpDelay: ptr.To("0s"), - OkTotalUnreadyCount: ptr.To("3"), - ScanInterval: ptr.To("10s"), - ScaleDownDelayAfterAdd: ptr.To("10m"), - ScaleDownDelayAfterDelete: ptr.To("10s"), - ScaleDownDelayAfterFailure: ptr.To("3m"), - ScaleDownUnneededTime: ptr.To("10m"), - ScaleDownUnreadyTime: ptr.To("20m"), - ScaleDownUtilizationThreshold: ptr.To("0.5"), - SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageFalse))), - SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsTrue))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + AutoScalerProfile: &AutoScalerProfile{ + BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsFalse))), + Expander: (*Expander)(ptr.To(string(ExpanderRandom))), + MaxEmptyBulkDelete: ptr.To("10"), + MaxGracefulTerminationSec: ptr.To("600"), + MaxNodeProvisionTime: ptr.To("15m"), + MaxTotalUnreadyPercentage: ptr.To("45"), + NewPodScaleUpDelay: ptr.To("0s"), + OkTotalUnreadyCount: ptr.To("3"), + ScanInterval: ptr.To("10s"), + ScaleDownDelayAfterAdd: ptr.To("10m"), + ScaleDownDelayAfterDelete: ptr.To("10s"), + ScaleDownDelayAfterFailure: ptr.To("3m"), + ScaleDownUnneededTime: ptr.To("10m"), + ScaleDownUnreadyTime: ptr.To("20m"), + ScaleDownUtilizationThreshold: ptr.To("0.5"), + SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageFalse))), + SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsTrue))), + }, }, }, } allFieldsAreNilTest := test{amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - AutoScalerProfile: &AutoScalerProfile{}, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + AutoScalerProfile: &AutoScalerProfile{}, + }, }, }} - allFieldsAreNilTest.amcp.setDefaultAutoScalerProfile() + allFieldsAreNilTest.amcp.Spec.AutoScalerProfile = setDefaultAutoScalerProfile(allFieldsAreNilTest.amcp.Spec.AutoScalerProfile) g.Expect(allFieldsAreNilTest.amcp.Spec.AutoScalerProfile).To(Equal(defaultAMP.Spec.AutoScalerProfile)) expectedNotNil := &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - AutoScalerProfile: &AutoScalerProfile{ - BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsTrue))), - Expander: (*Expander)(ptr.To(string(ExpanderLeastWaste))), - MaxEmptyBulkDelete: ptr.To("5"), - MaxGracefulTerminationSec: ptr.To("300"), - MaxNodeProvisionTime: ptr.To("10m"), - MaxTotalUnreadyPercentage: ptr.To("30"), - NewPodScaleUpDelay: ptr.To("30s"), - OkTotalUnreadyCount: ptr.To("5"), - ScanInterval: ptr.To("20s"), - ScaleDownDelayAfterAdd: ptr.To("5m"), - ScaleDownDelayAfterDelete: ptr.To("1m"), - ScaleDownDelayAfterFailure: ptr.To("2m"), - ScaleDownUnneededTime: ptr.To("5m"), - ScaleDownUnreadyTime: ptr.To("10m"), - ScaleDownUtilizationThreshold: ptr.To("0.4"), - SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageTrue))), - SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsFalse))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + AutoScalerProfile: &AutoScalerProfile{ + BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsTrue))), + Expander: (*Expander)(ptr.To(string(ExpanderLeastWaste))), + MaxEmptyBulkDelete: ptr.To("5"), + MaxGracefulTerminationSec: ptr.To("300"), + MaxNodeProvisionTime: ptr.To("10m"), + MaxTotalUnreadyPercentage: ptr.To("30"), + NewPodScaleUpDelay: ptr.To("30s"), + OkTotalUnreadyCount: ptr.To("5"), + ScanInterval: ptr.To("20s"), + ScaleDownDelayAfterAdd: ptr.To("5m"), + ScaleDownDelayAfterDelete: ptr.To("1m"), + ScaleDownDelayAfterFailure: ptr.To("2m"), + ScaleDownUnneededTime: ptr.To("5m"), + ScaleDownUnreadyTime: ptr.To("10m"), + ScaleDownUtilizationThreshold: ptr.To("0.4"), + SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageTrue))), + SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsFalse))), + }, }, }, } allFieldsAreNotNilTest := test{amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - AutoScalerProfile: expectedNotNil.Spec.AutoScalerProfile, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + AutoScalerProfile: ptr.To(*expectedNotNil.Spec.AutoScalerProfile), + }, }, }} - allFieldsAreNotNilTest.amcp.setDefaultAutoScalerProfile() + allFieldsAreNotNilTest.amcp.Spec.AutoScalerProfile = setDefaultAutoScalerProfile(allFieldsAreNotNilTest.amcp.Spec.AutoScalerProfile) g.Expect(allFieldsAreNotNilTest.amcp.Spec.AutoScalerProfile).To(Equal(expectedNotNil.Spec.AutoScalerProfile)) } diff --git a/api/v1beta1/azuremanagedcontrolplane_types.go b/api/v1beta1/azuremanagedcontrolplane_types.go index 24c04f5f8d5..09535d9e25d 100644 --- a/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/api/v1beta1/azuremanagedcontrolplane_types.go @@ -17,7 +17,6 @@ limitations under the License. package v1beta1 import ( - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) @@ -83,9 +82,7 @@ const ( // AzureManagedControlPlaneSpec defines the desired state of AzureManagedControlPlane. type AzureManagedControlPlaneSpec struct { - // Version defines the desired Kubernetes version. - // +kubebuilder:validation:MinLength:=2 - Version string `json:"version"` + AzureManagedControlPlaneClassSpec `json:",inline"` // ResourceGroupName is the name of the Azure resource group for this AKS Cluster. // Immutable. @@ -98,136 +95,21 @@ type AzureManagedControlPlaneSpec struct { // +optional NodeResourceGroupName string `json:"nodeResourceGroupName,omitempty"` - // VirtualNetwork describes the vnet for the AKS cluster. Will be created if it does not exist. - // Immutable except for `subnet`. - // +optional - VirtualNetwork ManagedControlPlaneVirtualNetwork `json:"virtualNetwork,omitempty"` - - // SubscriptionID is the GUID of the Azure subscription to hold this cluster. - // Immutable. - // +optional - SubscriptionID string `json:"subscriptionID,omitempty"` - - // Location is a string matching one of the canonical Azure region names. Examples: "westus2", "eastus". - // Immutable. - Location string `json:"location"` - // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. // Immutable, populated by the AKS API at create. // +optional ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint,omitempty"` - // AdditionalTags is an optional set of tags to add to Azure resources managed by the Azure provider, in addition to the - // ones added by default. - // +optional - AdditionalTags Tags `json:"additionalTags,omitempty"` - - // NetworkPlugin used for building Kubernetes network. - // Allowed values are "azure", "kubenet". - // Immutable. - // +kubebuilder:validation:Enum=azure;kubenet;none - // +optional - NetworkPlugin *string `json:"networkPlugin,omitempty"` - - // NetworkPluginMode is the mode the network plugin should use. - // Allowed value is "overlay". - // +kubebuilder:validation:Enum=overlay - // +optional - NetworkPluginMode *NetworkPluginMode `json:"networkPluginMode,omitempty"` - - // NetworkPolicy used for building Kubernetes network. - // Allowed values are "azure", "calico". - // Immutable. - // +kubebuilder:validation:Enum=azure;calico - // +optional - NetworkPolicy *string `json:"networkPolicy,omitempty"` - - // Outbound configuration used by Nodes. - // Immutable. - // +kubebuilder:validation:Enum=loadBalancer;managedNATGateway;userAssignedNATGateway;userDefinedRouting - // +optional - OutboundType *ManagedControlPlaneOutboundType `json:"outboundType,omitempty"` - // SSHPublicKey is a string literal containing an ssh public key base64 encoded. // Use empty string to autogenerate new key. Use null value to not set key. // Immutable. // +optional SSHPublicKey *string `json:"sshPublicKey,omitempty"` - // DNSServiceIP is an IP address assigned to the Kubernetes DNS service. - // It must be within the Kubernetes service address range specified in serviceCidr. - // Immutable. - // +optional - DNSServiceIP *string `json:"dnsServiceIP,omitempty"` - - // LoadBalancerSKU is the SKU of the loadBalancer to be provisioned. - // Immutable. - // +kubebuilder:validation:Enum=Basic;Standard - // +kubebuilder:default:=Standard - // +optional - LoadBalancerSKU *string `json:"loadBalancerSKU,omitempty"` - - // IdentityRef is a reference to a AzureClusterIdentity to be used when reconciling this cluster - IdentityRef *corev1.ObjectReference `json:"identityRef"` - - // AadProfile is Azure Active Directory configuration to integrate with AKS for aad authentication. - // +optional - AADProfile *AADProfile `json:"aadProfile,omitempty"` - - // AddonProfiles are the profiles of managed cluster add-on. - // +optional - AddonProfiles []AddonProfile `json:"addonProfiles,omitempty"` - - // SKU is the SKU of the AKS to be provisioned. - // +optional - SKU *AKSSku `json:"sku,omitempty"` - - // LoadBalancerProfile is the profile of the cluster load balancer. - // +optional - LoadBalancerProfile *LoadBalancerProfile `json:"loadBalancerProfile,omitempty"` - - // APIServerAccessProfile is the access profile for AKS API server. - // Immutable except for `authorizedIPRanges`. - // +optional - APIServerAccessProfile *APIServerAccessProfile `json:"apiServerAccessProfile,omitempty"` - - // AutoscalerProfile is the parameters to be applied to the cluster-autoscaler when enabled - // +optional - AutoScalerProfile *AutoScalerProfile `json:"autoscalerProfile,omitempty"` - - // AzureEnvironment is the name of the AzureCloud to be used. - // The default value that would be used by most users is "AzurePublicCloud", other values are: - // - ChinaCloud: "AzureChinaCloud" - // - PublicCloud: "AzurePublicCloud" - // - USGovernmentCloud: "AzureUSGovernmentCloud" - // +optional - AzureEnvironment string `json:"azureEnvironment,omitempty"` - - // Identity configuration used by the AKS control plane. - // +optional - Identity *Identity `json:"identity,omitempty"` - - // KubeletUserAssignedIdentity is the user-assigned identity for kubelet. - // For authentication with Azure Container Registry. - // +optional - KubeletUserAssignedIdentity string `json:"kubeletUserAssignedIdentity,omitempty"` - - // HTTPProxyConfig is the HTTP proxy configuration for the cluster. - // Immutable. - // +optional - HTTPProxyConfig *HTTPProxyConfig `json:"httpProxyConfig,omitempty"` - - // OIDCIssuerProfile is the OIDC issuer profile of the Managed Cluster. - // +optional - OIDCIssuerProfile *OIDCIssuerProfile `json:"oidcIssuerProfile,omitempty"` - // DNSPrefix allows the user to customize dns prefix. // Immutable. // +optional DNSPrefix *string `json:"dnsPrefix,omitempty"` - // DisableLocalAccounts disables getting static credentials for this cluster when set. Expected to only be used for AAD clusters. - // +optional - DisableLocalAccounts *bool `json:"disableLocalAccounts,omitempty"` } // HTTPProxyConfig is the HTTP proxy configuration for the cluster. @@ -331,28 +213,17 @@ type APIServerAccessProfile struct { // AuthorizedIPRanges - Authorized IP Ranges to kubernetes API server. // +optional AuthorizedIPRanges []string `json:"authorizedIPRanges,omitempty"` - // EnablePrivateCluster - Whether to create the cluster as a private cluster or not. - // +optional - EnablePrivateCluster *bool `json:"enablePrivateCluster,omitempty"` - // PrivateDNSZone - Private dns zone mode for private cluster. - // +kubebuilder:validation:Enum=System;None - // +optional - PrivateDNSZone *string `json:"privateDNSZone,omitempty"` - // EnablePrivateClusterPublicFQDN - Whether to create additional public FQDN for private cluster or not. - // +optional - EnablePrivateClusterPublicFQDN *bool `json:"enablePrivateClusterPublicFQDN,omitempty"` + + APIServerAccessProfileClassSpec `json:",inline"` } // ManagedControlPlaneVirtualNetwork describes a virtual network required to provision AKS clusters. type ManagedControlPlaneVirtualNetwork struct { - Name string `json:"name"` - CIDRBlock string `json:"cidrBlock"` - // Immutable except for `serviceEndpoints`. - // +optional - Subnet ManagedControlPlaneSubnet `json:"subnet,omitempty"` // ResourceGroup is the name of the Azure resource group for the VNet and Subnet. // +optional ResourceGroup string `json:"resourceGroup,omitempty"` + + ManagedControlPlaneVirtualNetworkClassSpec `json:",inline"` } // ManagedControlPlaneSubnet describes a subnet for an AKS cluster. @@ -393,6 +264,10 @@ type AzureManagedControlPlaneStatus struct { // OIDCIssuerProfile is the OIDC issuer profile of the Managed Cluster. // +optional OIDCIssuerProfile *OIDCIssuerProfileStatus `json:"oidcIssuerProfile,omitempty"` + + // Version defines the Kubernetes version for the control plane instance. + // +optional + Version string `json:"version"` } // OIDCIssuerProfileStatus is the OIDC issuer profile of the Managed Cluster. diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook.go b/api/v1beta1/azuremanagedcontrolplane_webhook.go index 6e5129b7292..1e5cc196caf 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -18,7 +18,6 @@ package v1beta1 import ( "context" - "errors" "fmt" "net" "reflect" @@ -29,7 +28,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api-provider-azure/feature" @@ -73,36 +71,27 @@ func (mw *azureManagedControlPlaneWebhook) Default(ctx context.Context, obj runt return apierrors.NewBadRequest("expected an AzureManagedControlPlane") } if m.Spec.NetworkPlugin == nil { - networkPlugin := "azure" + networkPlugin := AzureNetworkPluginName m.Spec.NetworkPlugin = &networkPlugin } - if m.Spec.Version != "" && !strings.HasPrefix(m.Spec.Version, "v") { - normalizedVersion := "v" + m.Spec.Version - m.Spec.Version = normalizedVersion - } - - if m.Spec.Identity == nil { - m.Spec.Identity = &Identity{ - Type: ManagedControlPlaneIdentityTypeSystemAssigned, - } - } + setDefault[*string](&m.Spec.NetworkPlugin, ptr.To(AzureNetworkPluginName)) + setDefault[*string](&m.Spec.LoadBalancerSKU, ptr.To("Standard")) + setDefault[*Identity](&m.Spec.Identity, &Identity{ + Type: ManagedControlPlaneIdentityTypeSystemAssigned, + }) + m.Spec.Version = setDefaultVersion(m.Spec.Version) + m.Spec.SKU = setDefaultSku(m.Spec.SKU) + m.Spec.AutoScalerProfile = setDefaultAutoScalerProfile(m.Spec.AutoScalerProfile) if err := m.setDefaultSSHPublicKey(); err != nil { ctrl.Log.WithName("AzureManagedControlPlaneWebHookLogger").Error(err, "setDefaultSSHPublicKey failed") } - // PaidManagedControlPlaneTier has been replaced with StandardManagedControlPlaneTier since v2023-02-01. - if m.Spec.SKU != nil && m.Spec.SKU.Tier == PaidManagedControlPlaneTier { - m.Spec.SKU.Tier = StandardManagedControlPlaneTier - ctrl.Log.WithName("AzureManagedControlPlaneWebHookLogger").Info("Paid SKU tier is deprecated and has been replaced by Standard") - } - + m.setDefaultResourceGroupName() m.setDefaultNodeResourceGroupName() m.setDefaultVirtualNetwork() m.setDefaultSubnet() - m.setDefaultSku() - m.setDefaultAutoScalerProfile() m.setDefaultOIDCIssuerProfile() m.setDefaultDNSPrefix() @@ -274,31 +263,45 @@ func (mw *azureManagedControlPlaneWebhook) ValidateDelete(ctx context.Context, o // Validate the Azure Managed Control Plane and return an aggregate error. func (m *AzureManagedControlPlane) Validate(cli client.Client) error { - validators := []func(client client.Client) error{ - m.validateName, - m.validateVersion, + var allErrs field.ErrorList + validators := []func(client client.Client) field.ErrorList{ m.validateSSHKey, - m.validateLoadBalancerProfile, m.validateAPIServerAccessProfile, - m.validateManagedClusterNetwork, - m.validateAutoScalerProfile, m.validateIdentity, m.validateNetworkPluginMode, m.validateDNSPrefix, m.validateDisableLocalAccounts, } - - var errs []error for _, validator := range validators { if err := validator(cli); err != nil { - errs = append(errs, err) + allErrs = append(allErrs, err...) } } - return kerrors.NewAggregate(errs) + allErrs = append(allErrs, validateVersion( + m.Spec.Version, + field.NewPath("Spec").Child("Version"))...) + + allErrs = append(allErrs, validateLoadBalancerProfile( + m.Spec.LoadBalancerProfile, + field.NewPath("Spec").Child("LoadBalancerProfile"))...) + + allErrs = append(allErrs, validateManagedClusterNetwork( + cli, + m.Labels, + m.Namespace, + m.Spec.DNSServiceIP, + m.Spec.VirtualNetwork.Subnet, + field.NewPath("Spec"))...) + + allErrs = append(allErrs, validateName(m.Name, field.NewPath("Name"))...) + + allErrs = append(allErrs, validateAutoScalerProfile(m.Spec.AutoScalerProfile, field.NewPath("spec").Child("AutoScalerProfile"))...) + + return allErrs.ToAggregate() } -func (m *AzureManagedControlPlane) validateDNSPrefix(_ client.Client) error { +func (m *AzureManagedControlPlane) validateDNSPrefix(_ client.Client) field.ErrorList { if m.Spec.DNSPrefix == nil { return nil } @@ -315,31 +318,34 @@ func (m *AzureManagedControlPlane) validateDNSPrefix(_ client.Client) error { allErrs := field.ErrorList{ field.Invalid(field.NewPath("Spec", "DNSPrefix"), *m.Spec.DNSPrefix, "DNSPrefix is invalid, does not match regex: "+pattern), } - return kerrors.NewAggregate(allErrs.ToAggregate().Errors()) + return allErrs } // validateVersion disabling local accounts for AAD based clusters. -func (m *AzureManagedControlPlane) validateDisableLocalAccounts(_ client.Client) error { +func (m *AzureManagedControlPlane) validateDisableLocalAccounts(_ client.Client) field.ErrorList { if m.Spec.DisableLocalAccounts != nil && m.Spec.AADProfile == nil { - return errors.New("DisableLocalAccounts should be set only for AAD enabled clusters") + return field.ErrorList{ + field.Invalid(field.NewPath("Spec", "DisableLocalAccounts"), *m.Spec.DisableLocalAccounts, "DisableLocalAccounts should be set only for AAD enabled clusters"), + } } return nil } // validateVersion validates the Kubernetes version. -func (m *AzureManagedControlPlane) validateVersion(_ client.Client) error { - if !kubeSemver.MatchString(m.Spec.Version) { - return errors.New("must be a valid semantic version") +func validateVersion(version string, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + if !kubeSemver.MatchString(version) { + allErrs = append(allErrs, field.Invalid(fldPath, version, "must be a valid semantic version")) } - return nil + return allErrs } // validateSSHKey validates an SSHKey. -func (m *AzureManagedControlPlane) validateSSHKey(_ client.Client) error { +func (m *AzureManagedControlPlane) validateSSHKey(_ client.Client) field.ErrorList { if sshKey := m.Spec.SSHPublicKey; sshKey != nil && *sshKey != "" { if errs := ValidateSSHKey(*sshKey, field.NewPath("sshKey")); len(errs) > 0 { - return kerrors.NewAggregate(errs.ToAggregate().Errors()) + return errs } } @@ -347,56 +353,48 @@ func (m *AzureManagedControlPlane) validateSSHKey(_ client.Client) error { } // validateLoadBalancerProfile validates a LoadBalancerProfile. -func (m *AzureManagedControlPlane) validateLoadBalancerProfile(_ client.Client) error { - if m.Spec.LoadBalancerProfile != nil { - var errs []error - var allErrs field.ErrorList +func validateLoadBalancerProfile(loadBalancerProfile *LoadBalancerProfile, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + if loadBalancerProfile != nil { numOutboundIPTypes := 0 - if m.Spec.LoadBalancerProfile.ManagedOutboundIPs != nil { - if *m.Spec.LoadBalancerProfile.ManagedOutboundIPs < 1 || *m.Spec.LoadBalancerProfile.ManagedOutboundIPs > 100 { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "LoadBalancerProfile", "ManagedOutboundIPs"), *m.Spec.LoadBalancerProfile.ManagedOutboundIPs, "value should be in between 1 and 100")) + if loadBalancerProfile.ManagedOutboundIPs != nil { + if *loadBalancerProfile.ManagedOutboundIPs < 1 || *loadBalancerProfile.ManagedOutboundIPs > 100 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("ManagedOutboundIPs"), *loadBalancerProfile.ManagedOutboundIPs, "value should be in between 1 and 100")) } } - if m.Spec.LoadBalancerProfile.AllocatedOutboundPorts != nil { - if *m.Spec.LoadBalancerProfile.AllocatedOutboundPorts < 0 || *m.Spec.LoadBalancerProfile.AllocatedOutboundPorts > 64000 { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "LoadBalancerProfile", "AllocatedOutboundPorts"), *m.Spec.LoadBalancerProfile.AllocatedOutboundPorts, "value should be in between 0 and 64000")) + if loadBalancerProfile.AllocatedOutboundPorts != nil { + if *loadBalancerProfile.AllocatedOutboundPorts < 0 || *loadBalancerProfile.AllocatedOutboundPorts > 64000 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("AllocatedOutboundPorts"), *loadBalancerProfile.AllocatedOutboundPorts, "value should be in between 0 and 64000")) } } - if m.Spec.LoadBalancerProfile.IdleTimeoutInMinutes != nil { - if *m.Spec.LoadBalancerProfile.IdleTimeoutInMinutes < 4 || *m.Spec.LoadBalancerProfile.IdleTimeoutInMinutes > 120 { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "LoadBalancerProfile", "IdleTimeoutInMinutes"), *m.Spec.LoadBalancerProfile.IdleTimeoutInMinutes, "value should be in between 4 and 120")) + if loadBalancerProfile.IdleTimeoutInMinutes != nil { + if *loadBalancerProfile.IdleTimeoutInMinutes < 4 || *loadBalancerProfile.IdleTimeoutInMinutes > 120 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("IdleTimeoutInMinutes"), *loadBalancerProfile.IdleTimeoutInMinutes, "value should be in between 4 and 120")) } } - if m.Spec.LoadBalancerProfile.ManagedOutboundIPs != nil { + if loadBalancerProfile.ManagedOutboundIPs != nil { numOutboundIPTypes++ } - if len(m.Spec.LoadBalancerProfile.OutboundIPPrefixes) > 0 { + if len(loadBalancerProfile.OutboundIPPrefixes) > 0 { numOutboundIPTypes++ } - if len(m.Spec.LoadBalancerProfile.OutboundIPs) > 0 { + if len(loadBalancerProfile.OutboundIPs) > 0 { numOutboundIPTypes++ } if numOutboundIPTypes > 1 { - errs = append(errs, errors.New("load balancer profile must specify at most one of ManagedOutboundIPs, OutboundIPPrefixes and OutboundIPs")) - } - - if len(allErrs) > 0 { - agg := kerrors.NewAggregate(allErrs.ToAggregate().Errors()) - errs = append(errs, agg) + allErrs = append(allErrs, field.Forbidden(fldPath, "load balancer profile must specify at most one of ManagedOutboundIPs, OutboundIPPrefixes and OutboundIPs")) } - - return kerrors.NewAggregate(errs) } - return nil + return allErrs } // validateAPIServerAccessProfile validates an APIServerAccessProfile. -func (m *AzureManagedControlPlane) validateAPIServerAccessProfile(_ client.Client) error { +func (m *AzureManagedControlPlane) validateAPIServerAccessProfile(_ client.Client) field.ErrorList { if m.Spec.APIServerAccessProfile != nil { var allErrs field.ErrorList for _, ipRange := range m.Spec.APIServerAccessProfile.AuthorizedIPRanges { @@ -405,37 +403,38 @@ func (m *AzureManagedControlPlane) validateAPIServerAccessProfile(_ client.Clien } } if len(allErrs) > 0 { - return kerrors.NewAggregate(allErrs.ToAggregate().Errors()) + return allErrs } } return nil } // validateManagedClusterNetwork validates the Cluster network values. -func (m *AzureManagedControlPlane) validateManagedClusterNetwork(cli client.Client) error { +func validateManagedClusterNetwork(cli client.Client, labels map[string]string, namespace string, dnsServiceIP *string, subnet ManagedControlPlaneSubnet, fldPath *field.Path) field.ErrorList { + var ( + allErrs field.ErrorList + serviceCIDR string + ) + ctx := context.Background() // Fetch the Cluster. - clusterName, ok := m.Labels[clusterv1.ClusterNameLabel] + clusterName, ok := labels[clusterv1.ClusterNameLabel] if !ok { return nil } ownerCluster := &clusterv1.Cluster{} key := client.ObjectKey{ - Namespace: m.Namespace, + Namespace: namespace, Name: clusterName, } if err := cli.Get(ctx, key, ownerCluster); err != nil { - return err + allErrs = append(allErrs, field.InternalError(field.NewPath("Cluster", "Spec", "ClusterNetwork"), err)) + return allErrs } - var ( - allErrs field.ErrorList - serviceCIDR string - ) - if clusterNetwork := ownerCluster.Spec.ClusterNetwork; clusterNetwork != nil { if clusterNetwork.Services != nil { // A user may provide zero or one CIDR blocks. If they provide an empty array, @@ -456,7 +455,7 @@ func (m *AzureManagedControlPlane) validateManagedClusterNetwork(cli client.Clie } } - if m.Spec.DNSServiceIP != nil { + if dnsServiceIP != nil { if serviceCIDR == "" { allErrs = append(allErrs, field.Required(field.NewPath("Cluster", "Spec", "ClusterNetwork", "Services", "CIDRBlocks"), "service CIDR must be specified if specifying DNSServiceIP")) } @@ -465,9 +464,9 @@ func (m *AzureManagedControlPlane) validateManagedClusterNetwork(cli client.Clie allErrs = append(allErrs, field.Invalid(field.NewPath("Cluster", "Spec", "ClusterNetwork", "Services", "CIDRBlocks"), serviceCIDR, fmt.Sprintf("failed to parse cluster service cidr: %v", err))) } - dnsIP := net.ParseIP(*m.Spec.DNSServiceIP) + dnsIP := net.ParseIP(*dnsServiceIP) if dnsIP == nil { // dnsIP will be nil if the string is not a valid IP - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "DNSServiceIP"), *m.Spec.DNSServiceIP, "must be a valid IP address")) + allErrs = append(allErrs, field.Invalid(field.NewPath("Cluster", "Spec", "ClusterNetwork", "Services", "DNSServiceIP"), *dnsServiceIP, "must be a valid IP address")) } if dnsIP != nil && !cidr.Contains(dnsIP) { @@ -478,18 +477,15 @@ func (m *AzureManagedControlPlane) validateManagedClusterNetwork(cli client.Clie // Refer to: https://learn.microsoft.com/en-us/azure/aks/configure-kubenet#create-an-aks-cluster-with-system-assigned-managed-identities targetSuffix := ".10" if dnsIP != nil && !strings.HasSuffix(dnsIP.String(), targetSuffix) { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "DNSServiceIP"), *m.Spec.DNSServiceIP, fmt.Sprintf("must end with %q", targetSuffix))) + allErrs = append(allErrs, field.Invalid(field.NewPath("Cluster", "Spec", "ClusterNetwork", "Services", "DNSServiceIP"), *dnsServiceIP, fmt.Sprintf("must end with %q", targetSuffix))) } } - if errs := validatePrivateEndpoints(m.Spec.VirtualNetwork.Subnet.PrivateEndpoints, []string{m.Spec.VirtualNetwork.Subnet.CIDRBlock}, field.NewPath("Spec", "VirtualNetwork.Subnet.PrivateEndpoints")); len(errs) > 0 { + if errs := validatePrivateEndpoints(subnet.PrivateEndpoints, []string{subnet.CIDRBlock}, fldPath.Child("VirtualNetwork.Subnet.PrivateEndpoints")); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if len(allErrs) > 0 { - return kerrors.NewAggregate(allErrs.ToAggregate().Errors()) - } - return nil + return allErrs } // validateAPIServerAccessProfileUpdate validates update to APIServerAccessProfile. @@ -500,16 +496,20 @@ func (m *AzureManagedControlPlane) validateAPIServerAccessProfileUpdate(old *Azu oldAPIServerAccessProfileNormalized := &APIServerAccessProfile{} if m.Spec.APIServerAccessProfile != nil { newAPIServerAccessProfileNormalized = &APIServerAccessProfile{ - EnablePrivateCluster: m.Spec.APIServerAccessProfile.EnablePrivateCluster, - PrivateDNSZone: m.Spec.APIServerAccessProfile.PrivateDNSZone, - EnablePrivateClusterPublicFQDN: m.Spec.APIServerAccessProfile.EnablePrivateClusterPublicFQDN, + APIServerAccessProfileClassSpec: APIServerAccessProfileClassSpec{ + EnablePrivateCluster: m.Spec.APIServerAccessProfile.EnablePrivateCluster, + PrivateDNSZone: m.Spec.APIServerAccessProfile.PrivateDNSZone, + EnablePrivateClusterPublicFQDN: m.Spec.APIServerAccessProfile.EnablePrivateClusterPublicFQDN, + }, } } if old.Spec.APIServerAccessProfile != nil { oldAPIServerAccessProfileNormalized = &APIServerAccessProfile{ - EnablePrivateCluster: old.Spec.APIServerAccessProfile.EnablePrivateCluster, - PrivateDNSZone: old.Spec.APIServerAccessProfile.PrivateDNSZone, - EnablePrivateClusterPublicFQDN: old.Spec.APIServerAccessProfile.EnablePrivateClusterPublicFQDN, + APIServerAccessProfileClassSpec: APIServerAccessProfileClassSpec{ + EnablePrivateCluster: old.Spec.APIServerAccessProfile.EnablePrivateCluster, + PrivateDNSZone: old.Spec.APIServerAccessProfile.PrivateDNSZone, + EnablePrivateClusterPublicFQDN: old.Spec.APIServerAccessProfile.EnablePrivateClusterPublicFQDN, + }, } } @@ -676,153 +676,150 @@ func (m *AzureManagedControlPlane) validateOIDCIssuerProfileUpdate(old *AzureMan return allErrs } -func (m *AzureManagedControlPlane) validateName(_ client.Client) error { - if lName := strings.ToLower(m.Name); strings.Contains(lName, "microsoft") || +func validateName(name string, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + if lName := strings.ToLower(name); strings.Contains(lName, "microsoft") || strings.Contains(lName, "windows") { - return field.Invalid(field.NewPath("Name"), m.Name, - "cluster name is invalid because 'MICROSOFT' and 'WINDOWS' can't be used as either a whole word or a substring in the name") + allErrs = append(allErrs, field.Invalid(fldPath.Child("Name"), name, + "cluster name is invalid because 'MICROSOFT' and 'WINDOWS' can't be used as either a whole word or a substring in the name")) } - return nil + return allErrs } // validateAutoScalerProfile validates an AutoScalerProfile. -func (m *AzureManagedControlPlane) validateAutoScalerProfile(_ client.Client) error { +func validateAutoScalerProfile(autoScalerProfile *AutoScalerProfile, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - if m.Spec.AutoScalerProfile == nil { + if autoScalerProfile == nil { return nil } - if errs := m.validateIntegerStringGreaterThanZero(m.Spec.AutoScalerProfile.MaxEmptyBulkDelete, "MaxEmptyBulkDelete"); len(errs) > 0 { + if errs := validateIntegerStringGreaterThanZero(autoScalerProfile.MaxEmptyBulkDelete, fldPath, "MaxEmptyBulkDelete"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateIntegerStringGreaterThanZero(m.Spec.AutoScalerProfile.MaxGracefulTerminationSec, "MaxGracefulTerminationSec"); len(errs) > 0 { + if errs := validateIntegerStringGreaterThanZero(autoScalerProfile.MaxGracefulTerminationSec, fldPath, "MaxGracefulTerminationSec"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateMaxNodeProvisionTime(); len(errs) > 0 { + if errs := validateMaxNodeProvisionTime(autoScalerProfile.MaxNodeProvisionTime, fldPath); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if m.Spec.AutoScalerProfile.MaxTotalUnreadyPercentage != nil { - val, err := strconv.Atoi(*m.Spec.AutoScalerProfile.MaxTotalUnreadyPercentage) + if autoScalerProfile.MaxTotalUnreadyPercentage != nil { + val, err := strconv.Atoi(*autoScalerProfile.MaxTotalUnreadyPercentage) if err != nil || val < 0 || val > 100 { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "MaxTotalUnreadyPercentage"), m.Spec.AutoScalerProfile.MaxTotalUnreadyPercentage, "invalid value")) + allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "MaxTotalUnreadyPercentage"), autoScalerProfile.MaxTotalUnreadyPercentage, "invalid value")) } } - if errs := m.validateNewPodScaleUpDelay(); len(errs) > 0 { + if errs := validateNewPodScaleUpDelay(autoScalerProfile.NewPodScaleUpDelay, fldPath); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateIntegerStringGreaterThanZero(m.Spec.AutoScalerProfile.OkTotalUnreadyCount, "OkTotalUnreadyCount"); len(errs) > 0 { + if errs := validateIntegerStringGreaterThanZero(autoScalerProfile.OkTotalUnreadyCount, fldPath, "OkTotalUnreadyCount"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateScanInterval(); len(errs) > 0 { + if errs := validateScanInterval(autoScalerProfile.ScanInterval, fldPath); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateScaleDownTime(m.Spec.AutoScalerProfile.ScaleDownDelayAfterAdd, "ScaleDownDelayAfterAdd"); len(errs) > 0 { + if errs := validateScaleDownTime(autoScalerProfile.ScaleDownDelayAfterAdd, fldPath, "ScaleDownDelayAfterAdd"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateScaleDownDelayAfterDelete(); len(errs) > 0 { + if errs := validateScaleDownDelayAfterDelete(autoScalerProfile.ScaleDownDelayAfterDelete, fldPath); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateScaleDownTime(m.Spec.AutoScalerProfile.ScaleDownDelayAfterFailure, "ScaleDownDelayAfterFailure"); len(errs) > 0 { + if errs := validateScaleDownTime(autoScalerProfile.ScaleDownDelayAfterFailure, fldPath, "ScaleDownDelayAfterFailure"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateScaleDownTime(m.Spec.AutoScalerProfile.ScaleDownUnneededTime, "ScaleDownUnneededTime"); len(errs) > 0 { + if errs := validateScaleDownTime(autoScalerProfile.ScaleDownUnneededTime, fldPath, "ScaleDownUnneededTime"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateScaleDownTime(m.Spec.AutoScalerProfile.ScaleDownUnreadyTime, "ScaleDownUnreadyTime"); len(errs) > 0 { + if errs := validateScaleDownTime(autoScalerProfile.ScaleDownUnreadyTime, fldPath, "ScaleDownUnreadyTime"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if m.Spec.AutoScalerProfile.ScaleDownUtilizationThreshold != nil { - val, err := strconv.ParseFloat(*m.Spec.AutoScalerProfile.ScaleDownUtilizationThreshold, 32) + if autoScalerProfile.ScaleDownUtilizationThreshold != nil { + val, err := strconv.ParseFloat(*autoScalerProfile.ScaleDownUtilizationThreshold, 32) if err != nil || val < 0 || val > 1 { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "ScaleDownUtilizationThreshold"), m.Spec.AutoScalerProfile.ScaleDownUtilizationThreshold, "invalid value")) + allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "ScaleDownUtilizationThreshold"), autoScalerProfile.ScaleDownUtilizationThreshold, "invalid value")) } } - if len(allErrs) > 0 { - return kerrors.NewAggregate(allErrs.ToAggregate().Errors()) - } - - return nil + return allErrs } // validateMaxNodeProvisionTime validates update to AutoscalerProfile.MaxNodeProvisionTime. -func (m *AzureManagedControlPlane) validateMaxNodeProvisionTime() field.ErrorList { +func validateMaxNodeProvisionTime(maxNodeProvisionTime *string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - if ptr.Deref(m.Spec.AutoScalerProfile.MaxNodeProvisionTime, "") != "" { - if !rMaxNodeProvisionTime.MatchString(ptr.Deref(m.Spec.AutoScalerProfile.MaxNodeProvisionTime, "")) { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "MaxNodeProvisionTime"), m.Spec.AutoScalerProfile.MaxNodeProvisionTime, "invalid value")) + if ptr.Deref(maxNodeProvisionTime, "") != "" { + if !rMaxNodeProvisionTime.MatchString(ptr.Deref(maxNodeProvisionTime, "")) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("MaxNodeProvisionTime"), maxNodeProvisionTime, "invalid value")) } } return allErrs } // validateScanInterval validates update to AutoscalerProfile.ScanInterval. -func (m *AzureManagedControlPlane) validateScanInterval() field.ErrorList { +func validateScanInterval(scanInterval *string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - if ptr.Deref(m.Spec.AutoScalerProfile.ScanInterval, "") != "" { - if !rScanInterval.MatchString(ptr.Deref(m.Spec.AutoScalerProfile.ScanInterval, "")) { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "ScanInterval"), m.Spec.AutoScalerProfile.ScanInterval, "invalid value")) + if ptr.Deref(scanInterval, "") != "" { + if !rScanInterval.MatchString(ptr.Deref(scanInterval, "")) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("ScanInterval"), scanInterval, "invalid value")) } } return allErrs } // validateNewPodScaleUpDelay validates update to AutoscalerProfile.NewPodScaleUpDelay. -func (m *AzureManagedControlPlane) validateNewPodScaleUpDelay() field.ErrorList { +func validateNewPodScaleUpDelay(newPodScaleUpDelay *string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - if ptr.Deref(m.Spec.AutoScalerProfile.NewPodScaleUpDelay, "") != "" { - _, err := time.ParseDuration(ptr.Deref(m.Spec.AutoScalerProfile.NewPodScaleUpDelay, "")) + if ptr.Deref(newPodScaleUpDelay, "") != "" { + _, err := time.ParseDuration(ptr.Deref(newPodScaleUpDelay, "")) if err != nil { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "NewPodScaleUpDelay"), m.Spec.AutoScalerProfile.NewPodScaleUpDelay, "invalid value")) + allErrs = append(allErrs, field.Invalid(fldPath.Child("NewPodScaleUpDelay"), newPodScaleUpDelay, "invalid value")) } } return allErrs } // validateScaleDownDelayAfterDelete validates update to AutoscalerProfile.ScaleDownDelayAfterDelete value. -func (m *AzureManagedControlPlane) validateScaleDownDelayAfterDelete() field.ErrorList { +func validateScaleDownDelayAfterDelete(scaleDownDelayAfterDelete *string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - if ptr.Deref(m.Spec.AutoScalerProfile.ScaleDownDelayAfterDelete, "") != "" { - if !rScaleDownDelayAfterDelete.MatchString(ptr.Deref(m.Spec.AutoScalerProfile.ScaleDownDelayAfterDelete, "")) { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "ScaleDownDelayAfterDelete"), ptr.Deref(m.Spec.AutoScalerProfile.ScaleDownDelayAfterDelete, ""), "invalid value")) + if ptr.Deref(scaleDownDelayAfterDelete, "") != "" { + if !rScaleDownDelayAfterDelete.MatchString(ptr.Deref(scaleDownDelayAfterDelete, "")) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("ScaleDownDelayAfterDelete"), ptr.Deref(scaleDownDelayAfterDelete, ""), "invalid value")) } } return allErrs } // validateScaleDownTime validates update to AutoscalerProfile.ScaleDown* values. -func (m *AzureManagedControlPlane) validateScaleDownTime(scaleDownValue *string, fieldName string) field.ErrorList { +func validateScaleDownTime(scaleDownValue *string, fldPath *field.Path, fieldName string) field.ErrorList { var allErrs field.ErrorList if ptr.Deref(scaleDownValue, "") != "" { if !rScaleDownTime.MatchString(ptr.Deref(scaleDownValue, "")) { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", fieldName), ptr.Deref(scaleDownValue, ""), "invalid value")) + allErrs = append(allErrs, field.Invalid(fldPath.Child(fieldName), ptr.Deref(scaleDownValue, ""), "invalid value")) } } return allErrs } // validateIntegerStringGreaterThanZero validates that a string value is an integer greater than zero. -func (m *AzureManagedControlPlane) validateIntegerStringGreaterThanZero(input *string, fieldName string) field.ErrorList { +func validateIntegerStringGreaterThanZero(input *string, fldPath *field.Path, fieldName string) field.ErrorList { var allErrs field.ErrorList if input != nil { val, err := strconv.Atoi(*input) if err != nil || val < 0 { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", fieldName), input, "invalid value")) + allErrs = append(allErrs, field.Invalid(fldPath.Child(fieldName), input, "invalid value")) } } @@ -830,7 +827,7 @@ func (m *AzureManagedControlPlane) validateIntegerStringGreaterThanZero(input *s } // validateIdentity validates an Identity. -func (m *AzureManagedControlPlane) validateIdentity(_ client.Client) error { +func (m *AzureManagedControlPlane) validateIdentity(_ client.Client) field.ErrorList { var allErrs field.ErrorList if m.Spec.Identity != nil { @@ -846,14 +843,14 @@ func (m *AzureManagedControlPlane) validateIdentity(_ client.Client) error { } if len(allErrs) > 0 { - return kerrors.NewAggregate(allErrs.ToAggregate().Errors()) + return allErrs } return nil } // validateNetworkPluginMode validates a NetworkPluginMode. -func (m *AzureManagedControlPlane) validateNetworkPluginMode(_ client.Client) error { +func (m *AzureManagedControlPlane) validateNetworkPluginMode(_ client.Client) field.ErrorList { var allErrs field.ErrorList const kubenet = "kubenet" @@ -863,7 +860,7 @@ func (m *AzureManagedControlPlane) validateNetworkPluginMode(_ client.Client) er } if len(allErrs) > 0 { - return kerrors.NewAggregate(allErrs.ToAggregate().Errors()) + return allErrs } return nil diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index 870155cfbb5..a8ca849dedc 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation/field" utilfeature "k8s.io/component-base/featuregate/testing" "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api-provider-azure/feature" @@ -38,16 +39,19 @@ func TestDefaultingWebhook(t *testing.T) { Name: "fooName", }, Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Location: "fooLocation", + Version: "1.17.5", + }, ResourceGroupName: "fooRg", - Location: "fooLocation", - Version: "1.17.5", SSHPublicKey: ptr.To(""), }, } mcpw := &azureManagedControlPlaneWebhook{} err := mcpw.Default(context.Background(), amcp) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(*amcp.Spec.NetworkPlugin).To(Equal("azure")) + g.Expect(amcp.Spec.NetworkPlugin).To(Equal(ptr.To(AzureNetworkPluginName))) + g.Expect(amcp.Spec.LoadBalancerSKU).To(Equal(ptr.To("Standard"))) g.Expect(amcp.Spec.Version).To(Equal("v1.17.5")) g.Expect(*amcp.Spec.SSHPublicKey).NotTo(BeEmpty()) g.Expect(amcp.Spec.NodeResourceGroupName).To(Equal("MC_fooRg_fooName_fooLocation")) @@ -96,11 +100,13 @@ func TestDefaultingWebhook(t *testing.T) { Name: "fooName", }, Spec: AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Location: "fooLocation", + Version: "1.17.5", + NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + }, ResourceGroupName: "fooRg", - Location: "fooLocation", - Version: "1.17.5", SSHPublicKey: ptr.To(""), - NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), }, } err = mcpw.Default(context.Background(), amcp) @@ -109,11 +115,312 @@ func TestDefaultingWebhook(t *testing.T) { g.Expect(amcp.Spec.VirtualNetwork.Subnet.CIDRBlock).To(Equal(defaultAKSNodeSubnetCIDRForOverlay)) } +func TestValidateVersion(t *testing.T) { + tests := []struct { + name string + version string + expectErr bool + }{ + { + name: "Invalid Version", + version: "honk", + expectErr: true, + }, + { + name: "not following the Kubernetes Version pattern: missing leading v", + version: "1.19.0", + expectErr: true, + }, + { + name: "Version not set", + version: "", + expectErr: true, + }, + { + name: "Valid Version", + version: "v1.17.8", + expectErr: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + allErrs := validateVersion(tt.version, field.NewPath("spec").Child("Version")) + if tt.expectErr { + g.Expect(allErrs).NotTo(BeNil()) + } else { + g.Expect(allErrs).To(BeNil()) + } + }) + } +} + +func TestValidateLoadBalancerProfile(t *testing.T) { + tests := []struct { + name string + profile *LoadBalancerProfile + expectedErr field.Error + }{ + { + name: "Valid LoadBalancerProfile", + profile: &LoadBalancerProfile{ + ManagedOutboundIPs: ptr.To(10), + AllocatedOutboundPorts: ptr.To(1000), + IdleTimeoutInMinutes: ptr.To(60), + }, + }, + { + name: "Invalid LoadBalancerProfile.ManagedOutboundIPs", + profile: &LoadBalancerProfile{ + ManagedOutboundIPs: ptr.To(200), + }, + expectedErr: field.Error{ + Type: field.ErrorTypeInvalid, + Field: "spec.LoadBalancerProfile.ManagedOutboundIPs", + BadValue: ptr.To(200), + Detail: "value should be in between 1 and 100", + }, + }, + { + name: "Invalid LoadBalancerProfile.IdleTimeoutInMinutes", + profile: &LoadBalancerProfile{ + IdleTimeoutInMinutes: ptr.To(600), + }, + expectedErr: field.Error{ + Type: field.ErrorTypeInvalid, + Field: "spec.LoadBalancerProfile.IdleTimeoutInMinutes", + BadValue: ptr.To(600), + Detail: "value should be in between 4 and 120", + }, + }, + { + name: "LoadBalancerProfile must specify at most one of ManagedOutboundIPs, OutboundIPPrefixes and OutboundIPs", + profile: &LoadBalancerProfile{ + ManagedOutboundIPs: ptr.To(1), + OutboundIPs: []string{ + "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/foo-bar/providers/Microsoft.Network/publicIPAddresses/my-public-ip", + }, + }, + expectedErr: field.Error{ + Type: field.ErrorTypeForbidden, + Field: "spec.LoadBalancerProfile", + BadValue: ptr.To(2), + Detail: "load balancer profile must specify at most one of ManagedOutboundIPs, OutboundIPPrefixes and OutboundIPs", + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + allErrs := validateLoadBalancerProfile(tt.profile, field.NewPath("spec").Child("LoadBalancerProfile")) + if tt.expectedErr != (field.Error{}) { + g.Expect(allErrs).To(ContainElement(MatchError(tt.expectedErr.Error()))) + } else { + g.Expect(allErrs).To(BeNil()) + } + }) + } +} + +func TestValidateAutoScalerProfile(t *testing.T) { + tests := []struct { + name string + profile *AutoScalerProfile + expectErr bool + }{ + { + name: "Valid AutoScalerProfile", + profile: &AutoScalerProfile{ + BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsFalse))), + Expander: (*Expander)(ptr.To(string(ExpanderRandom))), + MaxEmptyBulkDelete: ptr.To("10"), + MaxGracefulTerminationSec: ptr.To("600"), + MaxNodeProvisionTime: ptr.To("10m"), + MaxTotalUnreadyPercentage: ptr.To("45"), + NewPodScaleUpDelay: ptr.To("10m"), + OkTotalUnreadyCount: ptr.To("3"), + ScanInterval: ptr.To("60s"), + ScaleDownDelayAfterAdd: ptr.To("10m"), + ScaleDownDelayAfterDelete: ptr.To("10s"), + ScaleDownDelayAfterFailure: ptr.To("10m"), + ScaleDownUnneededTime: ptr.To("10m"), + ScaleDownUnreadyTime: ptr.To("10m"), + ScaleDownUtilizationThreshold: ptr.To("0.5"), + SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageTrue))), + SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsTrue))), + }, + expectErr: false, + }, + { + name: "Testing valid AutoScalerProfile.ExpanderRandom", + profile: &AutoScalerProfile{ + Expander: (*Expander)(ptr.To(string(ExpanderRandom))), + }, + expectErr: false, + }, + { + name: "Testing valid AutoScalerProfile.ExpanderLeastWaste", + profile: &AutoScalerProfile{ + Expander: (*Expander)(ptr.To(string(ExpanderLeastWaste))), + }, + expectErr: false, + }, + { + name: "Testing valid AutoScalerProfile.ExpanderMostPods", + profile: &AutoScalerProfile{ + Expander: (*Expander)(ptr.To(string(ExpanderMostPods))), + }, + expectErr: false, + }, + { + name: "Testing valid AutoScalerProfile.ExpanderPriority", + profile: &AutoScalerProfile{ + Expander: (*Expander)(ptr.To(string(ExpanderPriority))), + }, + expectErr: false, + }, + { + name: "Testing valid AutoScalerProfile.BalanceSimilarNodeGroupsTrue", + profile: &AutoScalerProfile{ + BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsTrue))), + }, + expectErr: false, + }, + { + name: "Testing valid AutoScalerProfile.BalanceSimilarNodeGroupsFalse", + profile: &AutoScalerProfile{ + BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsFalse))), + }, + expectErr: false, + }, + { + name: "Testing invalid AutoScalerProfile.MaxEmptyBulkDelete", + profile: &AutoScalerProfile{ + MaxEmptyBulkDelete: ptr.To("invalid"), + }, + expectErr: true, + }, + { + name: "Testing invalid AutoScalerProfile.MaxGracefulTerminationSec", + profile: &AutoScalerProfile{ + MaxGracefulTerminationSec: ptr.To("invalid"), + }, + expectErr: true, + }, + { + name: "Testing invalid AutoScalerProfile.MaxNodeProvisionTime", + profile: &AutoScalerProfile{ + MaxNodeProvisionTime: ptr.To("invalid"), + }, + expectErr: true, + }, + { + name: "Testing invalid AutoScalerProfile.MaxTotalUnreadyPercentage", + profile: &AutoScalerProfile{ + MaxTotalUnreadyPercentage: ptr.To("invalid"), + }, + expectErr: true, + }, + { + name: "Testing invalid AutoScalerProfile.NewPodScaleUpDelay", + profile: &AutoScalerProfile{ + NewPodScaleUpDelay: ptr.To("invalid"), + }, + expectErr: true, + }, + { + name: "Testing invalid AutoScalerProfile.OkTotalUnreadyCount", + profile: &AutoScalerProfile{ + OkTotalUnreadyCount: ptr.To("invalid"), + }, + expectErr: true, + }, + { + name: "Testing invalid AutoScalerProfile.ScanInterval", + profile: &AutoScalerProfile{ + ScanInterval: ptr.To("invalid"), + }, + expectErr: true, + }, + { + name: "Testing invalid AutoScalerProfile.ScaleDownDelayAfterAdd", + profile: &AutoScalerProfile{ + ScaleDownDelayAfterAdd: ptr.To("invalid"), + }, + expectErr: true, + }, + { + name: "Testing invalid AutoScalerProfile.ScaleDownDelayAfterDelete", + profile: &AutoScalerProfile{ + ScaleDownDelayAfterDelete: ptr.To("invalid"), + }, + expectErr: true, + }, + { + name: "Testing invalid AutoScalerProfile.ScaleDownDelayAfterFailure", + profile: &AutoScalerProfile{ + ScaleDownDelayAfterFailure: ptr.To("invalid"), + }, + expectErr: true, + }, + { + name: "Testing invalid AutoScalerProfile.ScaleDownUnneededTime", + profile: &AutoScalerProfile{ + ScaleDownUnneededTime: ptr.To("invalid"), + }, + expectErr: true, + }, + { + name: "Testing invalid AutoScalerProfile.ScaleDownUnreadyTime", + profile: &AutoScalerProfile{ + ScaleDownUnreadyTime: ptr.To("invalid"), + }, + expectErr: true, + }, + { + name: "Testing invalid AutoScalerProfile.ScaleDownUtilizationThreshold", + profile: &AutoScalerProfile{ + ScaleDownUtilizationThreshold: ptr.To("invalid"), + }, + expectErr: true, + }, + { + name: "Testing valid AutoScalerProfile.SkipNodesWithLocalStorageTrue", + profile: &AutoScalerProfile{ + SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageTrue))), + }, + expectErr: false, + }, + { + name: "Testing valid AutoScalerProfile.SkipNodesWithLocalStorageFalse", + profile: &AutoScalerProfile{ + SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsFalse))), + }, + expectErr: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + allErrs := validateAutoScalerProfile(tt.profile, field.NewPath("spec").Child("AutoScalerProfile")) + if tt.expectErr { + g.Expect(allErrs).NotTo(BeNil()) + } else { + g.Expect(allErrs).To(BeNil()) + } + }) + } +} + func TestValidatingWebhook(t *testing.T) { // NOTE: AzureManageControlPlane is behind AKS feature gate flag; the webhook // must prevent creating new objects in case the feature flag is disabled. defer utilfeature.SetFeatureGateDuringTest(t, feature.Gates, capifeature.MachinePool, true)() - tests := []struct { name string amcp AzureManagedControlPlane @@ -124,8 +431,10 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.17.8", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.17.8", + }, }, }, expectErr: false, @@ -135,8 +444,10 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10.3"), - Version: "v1.17.8", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10.3"), + Version: "v1.17.8", + }, }, }, expectErr: true, @@ -146,8 +457,10 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.11"), - Version: "v1.17.8", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.11"), + Version: "v1.17.8", + }, }, }, expectErr: true, @@ -157,7 +470,9 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.17.8", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + }, }, }, expectErr: false, @@ -167,8 +482,10 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "honk", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "honk", + }, }, }, expectErr: true, @@ -178,8 +495,10 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "1.19.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "1.19.0", + }, }, }, expectErr: true, @@ -189,8 +508,10 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "", + }, }, }, expectErr: true, @@ -200,8 +521,10 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.17.8", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.17.8", + }, }, }, expectErr: false, @@ -211,11 +534,13 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{ - "616077a8-5db7-4c98-b856-b34619afg75h", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.21.2", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{ + "616077a8-5db7-4c98-b856-b34619afg75h", + }, }, }, }, @@ -227,11 +552,13 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - LoadBalancerProfile: &LoadBalancerProfile{ - ManagedOutboundIPs: ptr.To(10), - AllocatedOutboundPorts: ptr.To(1000), - IdleTimeoutInMinutes: ptr.To(60), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.21.2", + LoadBalancerProfile: &LoadBalancerProfile{ + ManagedOutboundIPs: ptr.To(10), + AllocatedOutboundPorts: ptr.To(1000), + IdleTimeoutInMinutes: ptr.To(60), + }, }, }, }, @@ -242,9 +569,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - LoadBalancerProfile: &LoadBalancerProfile{ - ManagedOutboundIPs: ptr.To(200), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.21.2", + LoadBalancerProfile: &LoadBalancerProfile{ + ManagedOutboundIPs: ptr.To(200), + }, }, }, }, @@ -255,9 +584,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - LoadBalancerProfile: &LoadBalancerProfile{ - AllocatedOutboundPorts: ptr.To(80000), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.21.2", + LoadBalancerProfile: &LoadBalancerProfile{ + AllocatedOutboundPorts: ptr.To(80000), + }, }, }, }, @@ -268,9 +599,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - LoadBalancerProfile: &LoadBalancerProfile{ - IdleTimeoutInMinutes: ptr.To(600), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.21.2", + LoadBalancerProfile: &LoadBalancerProfile{ + IdleTimeoutInMinutes: ptr.To(600), + }, }, }, }, @@ -281,11 +614,13 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - LoadBalancerProfile: &LoadBalancerProfile{ - ManagedOutboundIPs: ptr.To(1), - OutboundIPs: []string{ - "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/foo-bar/providers/Microsoft.Network/publicIPAddresses/my-public-ip", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.21.2", + LoadBalancerProfile: &LoadBalancerProfile{ + ManagedOutboundIPs: ptr.To(1), + OutboundIPs: []string{ + "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/foo-bar/providers/Microsoft.Network/publicIPAddresses/my-public-ip", + }, }, }, }, @@ -297,9 +632,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - APIServerAccessProfile: &APIServerAccessProfile{ - AuthorizedIPRanges: []string{"1.2.3.400/32"}, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.21.2", + APIServerAccessProfile: &APIServerAccessProfile{ + AuthorizedIPRanges: []string{"1.2.3.400/32"}, + }, }, }, }, @@ -310,25 +647,27 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsFalse))), - Expander: (*Expander)(ptr.To(string(ExpanderRandom))), - MaxEmptyBulkDelete: ptr.To("10"), - MaxGracefulTerminationSec: ptr.To("600"), - MaxNodeProvisionTime: ptr.To("10m"), - MaxTotalUnreadyPercentage: ptr.To("45"), - NewPodScaleUpDelay: ptr.To("10m"), - OkTotalUnreadyCount: ptr.To("3"), - ScanInterval: ptr.To("60s"), - ScaleDownDelayAfterAdd: ptr.To("10m"), - ScaleDownDelayAfterDelete: ptr.To("10s"), - ScaleDownDelayAfterFailure: ptr.To("10m"), - ScaleDownUnneededTime: ptr.To("10m"), - ScaleDownUnreadyTime: ptr.To("10m"), - ScaleDownUtilizationThreshold: ptr.To("0.5"), - SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageTrue))), - SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsTrue))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsFalse))), + Expander: (*Expander)(ptr.To(string(ExpanderRandom))), + MaxEmptyBulkDelete: ptr.To("10"), + MaxGracefulTerminationSec: ptr.To("600"), + MaxNodeProvisionTime: ptr.To("10m"), + MaxTotalUnreadyPercentage: ptr.To("45"), + NewPodScaleUpDelay: ptr.To("10m"), + OkTotalUnreadyCount: ptr.To("3"), + ScanInterval: ptr.To("60s"), + ScaleDownDelayAfterAdd: ptr.To("10m"), + ScaleDownDelayAfterDelete: ptr.To("10s"), + ScaleDownDelayAfterFailure: ptr.To("10m"), + ScaleDownUnneededTime: ptr.To("10m"), + ScaleDownUnreadyTime: ptr.To("10m"), + ScaleDownUtilizationThreshold: ptr.To("0.5"), + SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageTrue))), + SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsTrue))), + }, }, }, }, @@ -339,9 +678,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - Expander: (*Expander)(ptr.To(string(ExpanderRandom))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + Expander: (*Expander)(ptr.To(string(ExpanderRandom))), + }, }, }, }, @@ -352,9 +693,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - Expander: (*Expander)(ptr.To(string(ExpanderLeastWaste))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + Expander: (*Expander)(ptr.To(string(ExpanderLeastWaste))), + }, }, }, }, @@ -365,9 +708,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - Expander: (*Expander)(ptr.To(string(ExpanderMostPods))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + Expander: (*Expander)(ptr.To(string(ExpanderMostPods))), + }, }, }, }, @@ -378,9 +723,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - Expander: (*Expander)(ptr.To(string(ExpanderPriority))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + Expander: (*Expander)(ptr.To(string(ExpanderPriority))), + }, }, }, }, @@ -391,9 +738,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsTrue))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsTrue))), + }, }, }, }, @@ -404,9 +753,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsFalse))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(ptr.To(string(BalanceSimilarNodeGroupsFalse))), + }, }, }, }, @@ -417,9 +768,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - MaxEmptyBulkDelete: ptr.To("invalid"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + MaxEmptyBulkDelete: ptr.To("invalid"), + }, }, }, }, @@ -430,9 +783,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - MaxGracefulTerminationSec: ptr.To("invalid"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + MaxGracefulTerminationSec: ptr.To("invalid"), + }, }, }, }, @@ -443,9 +798,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - MaxNodeProvisionTime: ptr.To("invalid"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + MaxNodeProvisionTime: ptr.To("invalid"), + }, }, }, }, @@ -456,9 +813,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - MaxTotalUnreadyPercentage: ptr.To("invalid"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + MaxTotalUnreadyPercentage: ptr.To("invalid"), + }, }, }, }, @@ -469,9 +828,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - NewPodScaleUpDelay: ptr.To("invalid"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + NewPodScaleUpDelay: ptr.To("invalid"), + }, }, }, }, @@ -482,9 +843,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - OkTotalUnreadyCount: ptr.To("invalid"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + OkTotalUnreadyCount: ptr.To("invalid"), + }, }, }, }, @@ -495,9 +858,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScanInterval: ptr.To("invalid"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + ScanInterval: ptr.To("invalid"), + }, }, }, }, @@ -508,9 +873,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScaleDownDelayAfterAdd: ptr.To("invalid"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + ScaleDownDelayAfterAdd: ptr.To("invalid"), + }, }, }, }, @@ -521,9 +888,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScaleDownDelayAfterDelete: ptr.To("invalid"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + ScaleDownDelayAfterDelete: ptr.To("invalid"), + }, }, }, }, @@ -534,9 +903,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScaleDownDelayAfterFailure: ptr.To("invalid"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + ScaleDownDelayAfterFailure: ptr.To("invalid"), + }, }, }, }, @@ -547,9 +918,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScaleDownUnneededTime: ptr.To("invalid"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + ScaleDownUnneededTime: ptr.To("invalid"), + }, }, }, }, @@ -560,9 +933,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScaleDownUnreadyTime: ptr.To("invalid"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + ScaleDownUnreadyTime: ptr.To("invalid"), + }, }, }, }, @@ -573,9 +948,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScaleDownUtilizationThreshold: ptr.To("invalid"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + ScaleDownUtilizationThreshold: ptr.To("invalid"), + }, }, }, }, @@ -586,9 +963,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageTrue))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageTrue))), + }, }, }, }, @@ -599,9 +978,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageFalse))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(ptr.To(string(SkipNodesWithLocalStorageFalse))), + }, }, }, }, @@ -612,9 +993,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsTrue))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsTrue))), + }, }, }, }, @@ -625,9 +1008,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsFalse))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + AutoScalerProfile: &AutoScalerProfile{ + SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(ptr.To(string(SkipNodesWithSystemPodsFalse))), + }, }, }, }, @@ -638,9 +1023,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - Identity: &Identity{ - Type: ManagedControlPlaneIdentityTypeSystemAssigned, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + Identity: &Identity{ + Type: ManagedControlPlaneIdentityTypeSystemAssigned, + }, }, }, }, @@ -651,10 +1038,12 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - Identity: &Identity{ - Type: ManagedControlPlaneIdentityTypeUserAssigned, - UserAssignedIdentityResourceID: "/resource/id", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + Identity: &Identity{ + Type: ManagedControlPlaneIdentityTypeUserAssigned, + UserAssignedIdentityResourceID: "/resource/id", + }, }, }, }, @@ -665,10 +1054,12 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - Identity: &Identity{ - Type: ManagedControlPlaneIdentityTypeSystemAssigned, - UserAssignedIdentityResourceID: "/resource/id", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + Identity: &Identity{ + Type: ManagedControlPlaneIdentityTypeSystemAssigned, + UserAssignedIdentityResourceID: "/resource/id", + }, }, }, }, @@ -679,9 +1070,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - Identity: &Identity{ - Type: ManagedControlPlaneIdentityTypeUserAssigned, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + Identity: &Identity{ + Type: ManagedControlPlaneIdentityTypeUserAssigned, + }, }, }, }, @@ -692,9 +1085,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - NetworkPlugin: ptr.To("kubenet"), - NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + NetworkPlugin: ptr.To("kubenet"), + NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + }, }, }, expectErr: true, @@ -704,9 +1099,11 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - NetworkPlugin: ptr.To("azure"), - NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.24.1", + NetworkPlugin: ptr.To("azure"), + NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + }, }, }, expectErr: false, @@ -783,7 +1180,9 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: ptr.To("-thisi$"), - Version: "v1.17.8", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + }, }, }, wantErr: true, @@ -793,7 +1192,9 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: ptr.To("thisisaverylong$^clusternameconsistingofmorethan54characterswhichshouldbeinvalid"), - Version: "v1.17.8", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + }, }, }, wantErr: true, @@ -803,7 +1204,9 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: ptr.To("no_underscore"), - Version: "v1.17.8", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + }, }, }, wantErr: true, @@ -813,7 +1216,9 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: ptr.To("no-dollar$@%"), - Version: "v1.17.8", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + }, }, }, wantErr: true, @@ -823,7 +1228,9 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: ptr.To("hyphen-allowed"), - Version: "v1.17.8", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + }, }, }, wantErr: false, @@ -833,7 +1240,9 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: ptr.To("palette-test07"), - Version: "v1.17.8", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + }, }, }, wantErr: false, @@ -843,7 +1252,9 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: ptr.To("thisisavlerylongclu7l0sternam3leconsistingofmorethan54"), - Version: "v1.17.8", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.17.8", + }, }, }, wantErr: false, @@ -856,8 +1267,10 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { }, Spec: AzureManagedControlPlaneSpec{ SSHPublicKey: ptr.To(generateSSHPublicKey(true)), - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.23.5", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.23.5", + }, }, }, wantErr: true, @@ -871,8 +1284,10 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { }, Spec: AzureManagedControlPlaneSpec{ SSHPublicKey: ptr.To(generateSSHPublicKey(true)), - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.23.5", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.23.5", + }, }, }, wantErr: true, @@ -885,13 +1300,15 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { ControlPlaneEndpoint: clusterv1.APIEndpoint{ Host: "my-host", }, - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", SSHPublicKey: ptr.To(generateSSHPublicKey(true)), - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{ - "616077a8-5db7-4c98-b856-b34619afg75h", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{ + "616077a8-5db7-4c98-b856-b34619afg75h", + }, }, }, }, @@ -905,13 +1322,15 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { ControlPlaneEndpoint: clusterv1.APIEndpoint{ Port: 444, }, - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", SSHPublicKey: ptr.To(generateSSHPublicKey(true)), - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{ - "616077a8-5db7-4c98-b856-b34619afg75h", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{ + "616077a8-5db7-4c98-b856-b34619afg75h", + }, }, }, }, @@ -922,8 +1341,10 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { name: "DisableLocalAccounts cannot be set for non AAD clusters", amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - DisableLocalAccounts: ptr.To[bool](true), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.21.2", + DisableLocalAccounts: ptr.To[bool](true), + }, }, }, wantErr: true, @@ -932,12 +1353,14 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { name: "DisableLocalAccounts can be set for AAD clusters", amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.21.2", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To[bool](true), }, - DisableLocalAccounts: ptr.To[bool](true), }, }, wantErr: false, @@ -983,8 +1406,8 @@ func TestAzureManagedControlPlane_ValidateCreateFailure(t *testing.T) { client := mockClient{ReturnError: false} for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - defer tc.deferFunc() g := NewWithT(t) + defer tc.deferFunc() mcpw := &azureManagedControlPlaneWebhook{ Client: client, } @@ -1036,16 +1459,20 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane AddonProfiles is mutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AddonProfiles: []AddonProfile{ - { - Name: "first-addon-profile", - Enabled: true, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AddonProfiles: []AddonProfile{ + { + Name: "first-addon-profile", + Enabled: true, + }, }, }, }, @@ -1056,22 +1483,26 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane AddonProfiles can be disabled", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - AddonProfiles: []AddonProfile{ - { - Name: "first-addon-profile", - Enabled: true, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + AddonProfiles: []AddonProfile{ + { + Name: "first-addon-profile", + Enabled: true, + }, }, + Version: "v1.18.0", }, - Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AddonProfiles: []AddonProfile{ - { - Name: "first-addon-profile", - Enabled: false, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AddonProfiles: []AddonProfile{ + { + Name: "first-addon-profile", + Enabled: false, + }, }, }, }, @@ -1082,18 +1513,22 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane AddonProfiles cannot update to empty array", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - AddonProfiles: []AddonProfile{ - { - Name: "first-addon-profile", - Enabled: true, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + AddonProfiles: []AddonProfile{ + { + Name: "first-addon-profile", + Enabled: true, + }, }, + Version: "v1.18.0", }, - Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1102,28 +1537,32 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane AddonProfiles cannot be completely removed", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - AddonProfiles: []AddonProfile{ - { - Name: "first-addon-profile", - Enabled: true, - }, - { - Name: "second-addon-profile", - Enabled: true, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + AddonProfiles: []AddonProfile{ + { + Name: "first-addon-profile", + Enabled: true, + }, + { + Name: "second-addon-profile", + Enabled: true, + }, }, + Version: "v1.18.0", }, - Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - AddonProfiles: []AddonProfile{ - { - Name: "first-addon-profile", - Enabled: true, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + AddonProfiles: []AddonProfile{ + { + Name: "first-addon-profile", + Enabled: true, + }, }, + Version: "v1.18.0", }, - Version: "v1.18.0", }, }, wantErr: true, @@ -1132,16 +1571,20 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane SubscriptionID is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - SubscriptionID: "212ec1q8", - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + SubscriptionID: "212ec1q8", + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - SubscriptionID: "212ec1q9", - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + SubscriptionID: "212ec1q9", + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1150,16 +1593,20 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane ResourceGroupName is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, ResourceGroupName: "hello-1", - Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, ResourceGroupName: "hello-2", - Version: "v1.18.0", }, }, wantErr: true, @@ -1168,16 +1615,20 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane NodeResourceGroupName is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, NodeResourceGroupName: "hello-1", - Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, NodeResourceGroupName: "hello-2", - Version: "v1.18.0", }, }, wantErr: true, @@ -1186,16 +1637,20 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane Location is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Location: "westeurope", - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Location: "westeurope", + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Location: "eastus", - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Location: "eastus", + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1204,16 +1659,20 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane SSHPublicKey is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, SSHPublicKey: ptr.To(generateSSHPublicKey(true)), - Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, SSHPublicKey: ptr.To(generateSSHPublicKey(true)), - Version: "v1.18.0", }, }, wantErr: true, @@ -1222,14 +1681,18 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane DNSServiceIP is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.1.1"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.1.1"), + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1238,13 +1701,17 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane DNSServiceIP is immutable, unsetting is not allowed", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1253,16 +1720,20 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane NetworkPlugin is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - NetworkPlugin: ptr.To("azure"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + NetworkPlugin: ptr.To("azure"), + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - NetworkPlugin: ptr.To("kubenet"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + NetworkPlugin: ptr.To("kubenet"), + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1271,15 +1742,19 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane NetworkPlugin is immutable, unsetting is not allowed", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - NetworkPlugin: ptr.To("azure"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + NetworkPlugin: ptr.To("azure"), + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1288,16 +1763,20 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane NetworkPolicy is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - NetworkPolicy: ptr.To("azure"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + NetworkPolicy: ptr.To("azure"), + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - NetworkPolicy: ptr.To("calico"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + NetworkPolicy: ptr.To("calico"), + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1306,15 +1785,19 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane NetworkPolicy is immutable, unsetting is not allowed", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - NetworkPolicy: ptr.To("azure"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + NetworkPolicy: ptr.To("azure"), + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1323,16 +1806,20 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane LoadBalancerSKU is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - LoadBalancerSKU: ptr.To(LoadBalancerSKUStandard), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + LoadBalancerSKU: ptr.To("Standard"), + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - LoadBalancerSKU: ptr.To(LoadBalancerSKUBasic), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + LoadBalancerSKU: ptr.To(LoadBalancerSKUBasic), + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1341,15 +1828,19 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane LoadBalancerSKU is immutable, unsetting is not allowed", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - LoadBalancerSKU: ptr.To(LoadBalancerSKUStandard), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + LoadBalancerSKU: ptr.To(LoadBalancerSKUStandard), + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1358,16 +1849,20 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane ManagedAad can be set after cluster creation", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{ - "616077a8-5db7-4c98-b856-b34619afg75h", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{ + "616077a8-5db7-4c98-b856-b34619afg75h", + }, }, }, }, @@ -1378,19 +1873,23 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane ManagedAad cannot be disabled", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{ - "616077a8-5db7-4c98-b856-b34619afg75h", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{ + "616077a8-5db7-4c98-b856-b34619afg75h", + }, }, }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AADProfile: &AADProfile{}, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{}, + }, }, }, wantErr: true, @@ -1399,22 +1898,26 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane managed field cannot set to false", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{ - "616077a8-5db7-4c98-b856-b34619afg75h", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{ + "616077a8-5db7-4c98-b856-b34619afg75h", + }, }, }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AADProfile: &AADProfile{ - Managed: false, - AdminGroupObjectIDs: []string{ - "616077a8-5db7-4c98-b856-b34619afg75h", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: false, + AdminGroupObjectIDs: []string{ + "616077a8-5db7-4c98-b856-b34619afg75h", + }, }, }, }, @@ -1425,21 +1928,25 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane adminGroupObjectIDs cannot set to empty", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{ - "616077a8-5db7-4c98-b856-b34619afg75h", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{ + "616077a8-5db7-4c98-b856-b34619afg75h", + }, }, }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{}, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{}, + }, }, }, }, @@ -1449,18 +1956,22 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane ManagedAad cannot be disabled", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{ - "616077a8-5db7-4c98-b856-b34619afg75h", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{ + "616077a8-5db7-4c98-b856-b34619afg75h", + }, }, }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1469,16 +1980,22 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane EnablePrivateCluster is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", - APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: ptr.To(true), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + APIServerAccessProfileClassSpec: APIServerAccessProfileClassSpec{ + EnablePrivateCluster: ptr.To(true), + }, + }, }, }, }, @@ -1488,16 +2005,20 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane AuthorizedIPRanges is mutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", - APIServerAccessProfile: &APIServerAccessProfile{ - AuthorizedIPRanges: []string{"192.168.0.1/32"}, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + AuthorizedIPRanges: []string{"192.168.0.1/32"}, + }, }, }, }, @@ -1510,15 +2031,19 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", - VirtualNetwork: ManagedControlPlaneVirtualNetwork{ - Name: "test-network", - CIDRBlock: "10.0.0.0/8", - ResourceGroup: "test-rg", - Subnet: ManagedControlPlaneSubnet{ - Name: "test-subnet", - CIDRBlock: "10.0.2.0/24", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Name: "test-network", + CIDRBlock: "10.0.0.0/8", + Subnet: ManagedControlPlaneSubnet{ + Name: "test-subnet", + CIDRBlock: "10.0.2.0/24", + }, + }, + ResourceGroup: "test-rg", }, }, }, @@ -1528,8 +2053,10 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1541,8 +2068,10 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ @@ -1550,15 +2079,19 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", - VirtualNetwork: ManagedControlPlaneVirtualNetwork{ - Name: "test-network", - CIDRBlock: "10.0.0.0/8", - ResourceGroup: "test-rg", - Subnet: ManagedControlPlaneSubnet{ - Name: "test-subnet", - CIDRBlock: "10.0.2.0/24", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Name: "test-network", + CIDRBlock: "10.0.0.0/8", + Subnet: ManagedControlPlaneSubnet{ + Name: "test-subnet", + CIDRBlock: "10.0.2.0/24", + }, + }, + ResourceGroup: "test-rg", }, }, }, @@ -1572,15 +2105,19 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", - VirtualNetwork: ManagedControlPlaneVirtualNetwork{ - Name: "test-network", - CIDRBlock: "10.0.0.0/8", - ResourceGroup: "test-rg", - Subnet: ManagedControlPlaneSubnet{ - Name: "test-subnet", - CIDRBlock: "10.0.2.0/24", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Name: "test-network", + CIDRBlock: "10.0.0.0/8", + Subnet: ManagedControlPlaneSubnet{ + Name: "test-subnet", + CIDRBlock: "10.0.2.0/24", + }, + }, + ResourceGroup: "test-rg", }, }, }, @@ -1590,15 +2127,19 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", - VirtualNetwork: ManagedControlPlaneVirtualNetwork{ - Name: "test-network", - CIDRBlock: "10.0.0.0/8", - ResourceGroup: "test-rg", - Subnet: ManagedControlPlaneSubnet{ - Name: "test-subnet", - CIDRBlock: "10.0.2.0/24", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Name: "test-network", + CIDRBlock: "10.0.0.0/8", + Subnet: ManagedControlPlaneSubnet{ + Name: "test-subnet", + CIDRBlock: "10.0.2.0/24", + }, + }, + ResourceGroup: "test-rg", }, }, }, @@ -1612,7 +2153,9 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - OutboundType: (*ManagedControlPlaneOutboundType)(ptr.To(string(ManagedControlPlaneOutboundTypeUserDefinedRouting))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + OutboundType: (*ManagedControlPlaneOutboundType)(ptr.To(string(ManagedControlPlaneOutboundTypeUserDefinedRouting))), + }, }, }, amcp: &AzureManagedControlPlane{ @@ -1620,7 +2163,9 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - OutboundType: (*ManagedControlPlaneOutboundType)(ptr.To(string(ManagedControlPlaneOutboundTypeLoadBalancer))), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + OutboundType: (*ManagedControlPlaneOutboundType)(ptr.To(string(ManagedControlPlaneOutboundTypeLoadBalancer))), + }, }, }, wantErr: true, @@ -1632,11 +2177,13 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - HTTPProxyConfig: &HTTPProxyConfig{ - HTTPProxy: ptr.To("http://1.2.3.4:8080"), - HTTPSProxy: ptr.To("https://5.6.7.8:8443"), - NoProxy: []string{"endpoint1", "endpoint2"}, - TrustedCA: ptr.To("ca"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + HTTPProxyConfig: &HTTPProxyConfig{ + HTTPProxy: ptr.To("http://1.2.3.4:8080"), + HTTPSProxy: ptr.To("https://5.6.7.8:8443"), + NoProxy: []string{"endpoint1", "endpoint2"}, + TrustedCA: ptr.To("ca"), + }, }, }, }, @@ -1645,11 +2192,13 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - HTTPProxyConfig: &HTTPProxyConfig{ - HTTPProxy: ptr.To("http://10.20.3.4:8080"), - HTTPSProxy: ptr.To("https://5.6.7.8:8443"), - NoProxy: []string{"endpoint1", "endpoint2"}, - TrustedCA: ptr.To("ca"), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + HTTPProxyConfig: &HTTPProxyConfig{ + HTTPProxy: ptr.To("http://10.20.3.4:8080"), + HTTPSProxy: ptr.To("https://5.6.7.8:8443"), + NoProxy: []string{"endpoint1", "endpoint2"}, + TrustedCA: ptr.To("ca"), + }, }, }, }, @@ -1662,8 +2211,10 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - NetworkPolicy: ptr.To("anything"), - NetworkPluginMode: nil, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + NetworkPolicy: ptr.To("anything"), + NetworkPluginMode: nil, + }, }, }, amcp: &AzureManagedControlPlane{ @@ -1671,8 +2222,10 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - NetworkPolicy: ptr.To("anything"), - NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + NetworkPolicy: ptr.To("anything"), + NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + }, }, }, wantErr: true, @@ -1684,8 +2237,10 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - NetworkPolicy: nil, - NetworkPluginMode: nil, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + NetworkPolicy: nil, + NetworkPluginMode: nil, + }, }, }, amcp: &AzureManagedControlPlane{ @@ -1693,8 +2248,10 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), - Version: "v0.0.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v0.0.0", + NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + }, }, }, wantErr: false, @@ -1706,8 +2263,10 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - NetworkPolicy: ptr.To("anything"), - NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + NetworkPolicy: ptr.To("anything"), + NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + }, }, }, amcp: &AzureManagedControlPlane{ @@ -1715,9 +2274,11 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - NetworkPolicy: ptr.To("anything"), - NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), - Version: "v0.0.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + NetworkPolicy: ptr.To("anything"), + Version: "v0.0.0", + NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + }, }, }, wantErr: false, @@ -1729,8 +2290,10 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - OIDCIssuerProfile: &OIDCIssuerProfile{ - Enabled: ptr.To(false), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + OIDCIssuerProfile: &OIDCIssuerProfile{ + Enabled: ptr.To(false), + }, }, }, }, @@ -1739,9 +2302,11 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - Version: "v0.0.0", - OIDCIssuerProfile: &OIDCIssuerProfile{ - Enabled: ptr.To(false), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v0.0.0", + OIDCIssuerProfile: &OIDCIssuerProfile{ + Enabled: ptr.To(false), + }, }, }, }, @@ -1754,8 +2319,10 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - OIDCIssuerProfile: &OIDCIssuerProfile{ - Enabled: ptr.To(false), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + OIDCIssuerProfile: &OIDCIssuerProfile{ + Enabled: ptr.To(false), + }, }, }, }, @@ -1764,9 +2331,11 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - Version: "v0.0.0", - OIDCIssuerProfile: &OIDCIssuerProfile{ - Enabled: ptr.To(true), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v0.0.0", + OIDCIssuerProfile: &OIDCIssuerProfile{ + Enabled: ptr.To(true), + }, }, }, }, @@ -1779,8 +2348,10 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - OIDCIssuerProfile: &OIDCIssuerProfile{ - Enabled: ptr.To(true), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + OIDCIssuerProfile: &OIDCIssuerProfile{ + Enabled: ptr.To(true), + }, }, }, }, @@ -1789,9 +2360,11 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - Version: "v0.0.0", - OIDCIssuerProfile: &OIDCIssuerProfile{ - Enabled: ptr.To(false), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v0.0.0", + OIDCIssuerProfile: &OIDCIssuerProfile{ + Enabled: ptr.To(false), + }, }, }, }, @@ -1804,8 +2377,10 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - OIDCIssuerProfile: &OIDCIssuerProfile{ - Enabled: ptr.To(true), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + OIDCIssuerProfile: &OIDCIssuerProfile{ + Enabled: ptr.To(true), + }, }, }, }, @@ -1814,9 +2389,11 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - Version: "v0.0.0", - OIDCIssuerProfile: &OIDCIssuerProfile{ - Enabled: ptr.To(true), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v0.0.0", + OIDCIssuerProfile: &OIDCIssuerProfile{ + Enabled: ptr.To(true), + }, }, }, }, @@ -1827,13 +2404,17 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: ptr.To("capz-aks-1"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: ptr.To("capz-aks"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1843,13 +2424,17 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: ptr.To("capz-aks"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: ptr.To("capz-aks"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, wantErr: false, @@ -1861,10 +2446,12 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, }, }, }, @@ -1873,11 +2460,13 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - DisableLocalAccounts: ptr.To[bool](true), - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To[bool](true), }, }, }, @@ -1890,12 +2479,14 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To[bool](true), }, - DisableLocalAccounts: ptr.To[bool](true), }, }, amcp: &AzureManagedControlPlane{ @@ -1903,10 +2494,12 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, }, }, }, @@ -1919,12 +2512,14 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To[bool](true), }, - DisableLocalAccounts: ptr.To[bool](true), }, }, amcp: &AzureManagedControlPlane{ @@ -1932,11 +2527,13 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - DisableLocalAccounts: ptr.To[bool](false), - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To[bool](false), }, }, }, @@ -1947,13 +2544,17 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: nil, - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: ptr.To("capz-aks"), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -1965,7 +2566,9 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ @@ -1973,8 +2576,10 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Name: "test-cluster", }, Spec: AzureManagedControlPlaneSpec{ - Version: "v1.18.0", - DisableLocalAccounts: ptr.To[bool](true), + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + DisableLocalAccounts: ptr.To[bool](true), + }, }, }, wantErr: true, @@ -1984,13 +2589,17 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: nil, - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: ptr.To(""), - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, wantErr: true, @@ -2000,13 +2609,17 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: nil, - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: nil, - Version: "v1.18.0", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Version: "v1.18.0", + }, }, }, wantErr: false, @@ -2034,8 +2647,10 @@ func createAzureManagedControlPlane(serviceIP, version, sshKey string) *AzureMan ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ SSHPublicKey: &sshKey, - DNSServiceIP: ptr.To(serviceIP), - Version: version, + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To(serviceIP), + Version: version, + }, }, } } @@ -2044,15 +2659,17 @@ func getKnownValidAzureManagedControlPlane() *AzureManagedControlPlane { return &AzureManagedControlPlane{ ObjectMeta: getAMCPMetaData(), Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: ptr.To("192.168.0.10"), - Version: "v1.18.0", - SSHPublicKey: ptr.To(generateSSHPublicKey(true)), - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{ - "616077a8-5db7-4c98-b856-b34619afg75h", + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + DNSServiceIP: ptr.To("192.168.0.10"), + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{ + "616077a8-5db7-4c98-b856-b34619afg75h", + }, }, }, + SSHPublicKey: ptr.To(generateSSHPublicKey(true)), }, } } diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_default.go b/api/v1beta1/azuremanagedcontrolplanetemplate_default.go new file mode 100644 index 00000000000..fce6a8182d0 --- /dev/null +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_default.go @@ -0,0 +1,69 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "strings" + + "k8s.io/utils/ptr" +) + +func (mcp *AzureManagedControlPlaneTemplate) setDefaults() { + setDefault[*string](&mcp.Spec.Template.Spec.NetworkPlugin, ptr.To(AzureNetworkPluginName)) + setDefault[*string](&mcp.Spec.Template.Spec.LoadBalancerSKU, ptr.To("Standard")) + + if mcp.Spec.Template.Spec.Version != "" && !strings.HasPrefix(mcp.Spec.Template.Spec.Version, "v") { + mcp.Spec.Template.Spec.Version = setDefaultVersion(mcp.Spec.Template.Spec.Version) + } + + mcp.setDefaultVirtualNetwork() + mcp.setDefaultSubnet() + mcp.Spec.Template.Spec.SKU = setDefaultSku(mcp.Spec.Template.Spec.SKU) + mcp.Spec.Template.Spec.AutoScalerProfile = setDefaultAutoScalerProfile(mcp.Spec.Template.Spec.AutoScalerProfile) +} + +// setDefaultVirtualNetwork sets the default VirtualNetwork for an AzureManagedControlPlaneTemplate. +func (mcp *AzureManagedControlPlaneTemplate) setDefaultVirtualNetwork() { + if mcp.Spec.Template.Spec.VirtualNetwork.Name == "" { + mcp.Spec.Template.Spec.VirtualNetwork.Name = mcp.Name + } + if mcp.Spec.Template.Spec.VirtualNetwork.CIDRBlock == "" { + mcp.Spec.Template.Spec.VirtualNetwork.CIDRBlock = defaultAKSVnetCIDR + } +} + +// setDefaultSubnet sets the default Subnet for an AzureManagedControlPlaneTemplate. +func (mcp *AzureManagedControlPlaneTemplate) setDefaultSubnet() { + if mcp.Spec.Template.Spec.VirtualNetwork.Subnet.Name == "" { + mcp.Spec.Template.Spec.VirtualNetwork.Subnet.Name = mcp.Name + } + if mcp.Spec.Template.Spec.VirtualNetwork.Subnet.CIDRBlock == "" { + mcp.Spec.Template.Spec.VirtualNetwork.Subnet.CIDRBlock = defaultAKSNodeSubnetCIDR + } +} + +// setDefault sets the default value for a pointer to a value for any comparable type. +func setDefault[T comparable](field *T, value T) { + if field == nil { + // shouldn't happen with proper use + return + } + var zero T + if *field == zero { + *field = value + } +} diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_default_test.go b/api/v1beta1/azuremanagedcontrolplanetemplate_default_test.go new file mode 100644 index 00000000000..6d18d47502f --- /dev/null +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_default_test.go @@ -0,0 +1,326 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "encoding/json" + "reflect" + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestDefaultVirtualNetworkTemplate(t *testing.T) { + cases := []struct { + name string + controlPlaneTemplate *AzureManagedControlPlaneTemplate + outputTemplate *AzureManagedControlPlaneTemplate + }{ + { + name: "virtual network not specified", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{}, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Name: "test-cluster-template", + CIDRBlock: defaultAKSVnetCIDR, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "custom name", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Name: "custom-vnet-name", + }, + }, + }, + }, + }, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Name: "custom-vnet-name", + CIDRBlock: defaultAKSVnetCIDR, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "custom cidr block", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + CIDRBlock: "10.0.0.16/24", + }, + }, + }, + }, + }, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Name: "test-cluster-template", + CIDRBlock: "10.0.0.16/24", + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, c := range cases { + tc := c + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.controlPlaneTemplate.setDefaultVirtualNetwork() + if !reflect.DeepEqual(tc.controlPlaneTemplate, tc.outputTemplate) { + expected, _ := json.MarshalIndent(tc.outputTemplate, "", "\t") + actual, _ := json.MarshalIndent(tc.controlPlaneTemplate, "", "\t") + t.Errorf("Expected %s, got %s", string(expected), string(actual)) + } + }) + } +} + +func TestDefaultSubnetTemplate(t *testing.T) { + cases := []struct { + name string + controlPlaneTemplate *AzureManagedControlPlaneTemplate + outputTemplate *AzureManagedControlPlaneTemplate + }{ + { + name: "subnet not specified", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{}, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Subnet: ManagedControlPlaneSubnet{ + Name: "test-cluster-template", + CIDRBlock: defaultAKSNodeSubnetCIDR, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "custom name", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Subnet: ManagedControlPlaneSubnet{ + Name: "custom-subnet-name", + }, + }, + }, + }, + }, + }, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Subnet: ManagedControlPlaneSubnet{ + Name: "custom-subnet-name", + CIDRBlock: defaultAKSNodeSubnetCIDR, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "custom cidr block", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Subnet: ManagedControlPlaneSubnet{ + CIDRBlock: "10.0.0.16/24", + }, + }, + }, + }, + }, + }, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Subnet: ManagedControlPlaneSubnet{ + Name: "test-cluster-template", + CIDRBlock: "10.0.0.16/24", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, c := range cases { + tc := c + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.controlPlaneTemplate.setDefaultSubnet() + if !reflect.DeepEqual(tc.controlPlaneTemplate, tc.outputTemplate) { + expected, _ := json.MarshalIndent(tc.outputTemplate, "", "\t") + actual, _ := json.MarshalIndent(tc.controlPlaneTemplate, "", "\t") + t.Errorf("Expected %s, got %s", string(expected), string(actual)) + } + }) + } +} + +func TestSetDefault(t *testing.T) { + g := NewGomegaWithT(t) + + type Struct struct{ name string } + + var s *Struct + setDefault(&s, &Struct{"hello"}) + g.Expect(s.name).To(Equal("hello")) + setDefault(&s, &Struct{"world"}) + g.Expect(s.name).To(Equal("hello")) + + r := &Struct{} + setDefault(&r, &Struct{"a name"}) + g.Expect(r.name).To(BeEmpty()) + setDefault(&r.name, "hello") + g.Expect(r.name).To(Equal("hello")) + setDefault(&r.name, "world") + g.Expect(r.name).To(Equal("hello")) + + str := "" + setDefault(&str, "a string") + g.Expect(str).To(Equal("a string")) + setDefault(&str, "another string") + g.Expect(str).To(Equal("a string")) +} diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_types.go b/api/v1beta1/azuremanagedcontrolplanetemplate_types.go new file mode 100644 index 00000000000..21ec160b42a --- /dev/null +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_types.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// AzureManagedControlPlaneTemplateSpec defines the desired state of AzureManagedControlPlaneTemplate. +type AzureManagedControlPlaneTemplateSpec struct { + Template AzureManagedControlPlaneTemplateResource `json:"template"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=azuremanagedcontrolplanetemplates,scope=Namespaced,categories=cluster-api,shortName=amcpt +// +kubebuilder:storageversion + +// AzureManagedControlPlaneTemplate is the Schema for the AzureManagedControlPlaneTemplates API. +type AzureManagedControlPlaneTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AzureManagedControlPlaneTemplateSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// AzureManagedControlPlaneTemplateList contains a list of AzureManagedControlPlaneTemplates. +type AzureManagedControlPlaneTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AzureManagedControlPlaneTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&AzureManagedControlPlaneTemplate{}, &AzureManagedControlPlaneTemplateList{}) +} + +// AzureManagedControlPlaneTemplateResource describes the data needed to create an AzureManagedCluster from a template. +type AzureManagedControlPlaneTemplateResource struct { + Spec AzureManagedControlPlaneTemplateResourceSpec `json:"spec"` +} diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go new file mode 100644 index 00000000000..f31ef754ba7 --- /dev/null +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go @@ -0,0 +1,289 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "reflect" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/cluster-api-provider-azure/feature" + webhookutils "sigs.k8s.io/cluster-api-provider-azure/util/webhook" + capifeature "sigs.k8s.io/cluster-api/feature" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// SetupAzureManagedControlPlaneTemplateWebhookWithManager will set up the webhook to be managed by the specified manager. +func SetupAzureManagedControlPlaneTemplateWebhookWithManager(mgr ctrl.Manager) error { + mcpw := &azureManagedControlPlaneTemplateWebhook{Client: mgr.GetClient()} + return ctrl.NewWebhookManagedBy(mgr). + For(&AzureManagedControlPlaneTemplate{}). + WithDefaulter(mcpw). + WithValidator(mcpw). + Complete() +} + +// +kubebuilder:webhook:verbs=create;update,path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplanetemplate,mutating=false,failurePolicy=fail,groups=infrastructure.cluster.x-k8s.io,resources=azuremanagedcontrolplanetemplates,versions=v1beta1,name=validation.azuremanagedcontrolplanetemplates.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// +kubebuilder:webhook:verbs=create;update,path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplanetemplate,mutating=true,failurePolicy=fail,groups=infrastructure.cluster.x-k8s.io,resources=azuremanagedcontrolplanetemplates,versions=v1beta1,name=default.azuremanagedcontrolplanetemplates.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 + +type azureManagedControlPlaneTemplateWebhook struct { + Client client.Client +} + +// Default implements webhook.Defaulter so a webhook will be registered for the type. +func (mcpw *azureManagedControlPlaneTemplateWebhook) Default(ctx context.Context, obj runtime.Object) error { + mcp, ok := obj.(*AzureManagedControlPlaneTemplate) + if !ok { + return apierrors.NewBadRequest("expected an AzureManagedControlPlaneTemplate") + } + mcp.setDefaults() + return nil +} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (mcpw *azureManagedControlPlaneTemplateWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + mcp, ok := obj.(*AzureManagedControlPlaneTemplate) + if !ok { + return nil, apierrors.NewBadRequest("expected an AzureManagedControlPlaneTemplate") + } + // NOTE: AzureManagedControlPlaneTemplate relies upon MachinePools, which is behind a feature gate flag. + // The webhook must prevent creating new objects in case the feature flag is disabled. + if !feature.Gates.Enabled(capifeature.MachinePool) { + return nil, field.Forbidden( + field.NewPath("spec"), + "can be set only if the Cluster API 'MachinePool' feature flag is enabled", + ) + } + + return nil, mcp.validateManagedControlPlaneTemplate(mcpw.Client) +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (mcpw *azureManagedControlPlaneTemplateWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + var allErrs field.ErrorList + old, ok := oldObj.(*AzureManagedControlPlaneTemplate) + if !ok { + return nil, apierrors.NewBadRequest("expected an AzureManagedControlPlaneTemplate") + } + mcp, ok := newObj.(*AzureManagedControlPlaneTemplate) + if !ok { + return nil, apierrors.NewBadRequest("expected an AzureManagedControlPlaneTemplate") + } + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "SubscriptionID"), + old.Spec.Template.Spec.SubscriptionID, + mcp.Spec.Template.Spec.SubscriptionID); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "Location"), + old.Spec.Template.Spec.Location, + mcp.Spec.Template.Spec.Location); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "DNSServiceIP"), + old.Spec.Template.Spec.DNSServiceIP, + mcp.Spec.Template.Spec.DNSServiceIP); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "NetworkPlugin"), + old.Spec.Template.Spec.NetworkPlugin, + mcp.Spec.Template.Spec.NetworkPlugin); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "NetworkPolicy"), + old.Spec.Template.Spec.NetworkPolicy, + mcp.Spec.Template.Spec.NetworkPolicy); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "LoadBalancerSKU"), + old.Spec.Template.Spec.LoadBalancerSKU, + mcp.Spec.Template.Spec.LoadBalancerSKU); err != nil { + allErrs = append(allErrs, err) + } + + if old.Spec.Template.Spec.AADProfile != nil { + if mcp.Spec.Template.Spec.AADProfile == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Template", "Spec", "AADProfile"), + mcp.Spec.Template.Spec.AADProfile, + "field cannot be nil, cannot disable AADProfile")) + } else { + if !mcp.Spec.Template.Spec.AADProfile.Managed && old.Spec.Template.Spec.AADProfile.Managed { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Template", "Spec", "AADProfile.Managed"), + mcp.Spec.Template.Spec.AADProfile.Managed, + "cannot set AADProfile.Managed to false")) + } + if len(mcp.Spec.Template.Spec.AADProfile.AdminGroupObjectIDs) == 0 { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Template", "Spec", "AADProfile.AdminGroupObjectIDs"), + mcp.Spec.Template.Spec.AADProfile.AdminGroupObjectIDs, + "length of AADProfile.AdminGroupObjectIDs cannot be zero")) + } + } + } + + // Consider removing this once moves out of preview + // Updating outboundType after cluster creation (PREVIEW) + // https://learn.microsoft.com/en-us/azure/aks/egress-outboundtype#updating-outboundtype-after-cluster-creation-preview + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "OutboundType"), + old.Spec.Template.Spec.OutboundType, + mcp.Spec.Template.Spec.OutboundType); err != nil { + allErrs = append(allErrs, err) + } + + if errs := mcp.validateVirtualNetworkTemplateUpdate(old); len(errs) > 0 { + allErrs = append(allErrs, errs...) + } + + if errs := mcp.validateAPIServerAccessProfileTemplateUpdate(old); len(errs) > 0 { + allErrs = append(allErrs, errs...) + } + + if len(allErrs) == 0 { + return nil, mcp.validateManagedControlPlaneTemplate(mcpw.Client) + } + + return nil, apierrors.NewInvalid(GroupVersion.WithKind("AzureManagedControlPlaneTemplate").GroupKind(), mcp.Name, allErrs) +} + +// Validate the Azure Managed Control Plane Template and return an aggregate error. +func (mcp *AzureManagedControlPlaneTemplate) validateManagedControlPlaneTemplate(cli client.Client) error { + var allErrs field.ErrorList + + allErrs = append(allErrs, validateVersion( + mcp.Spec.Template.Spec.Version, + field.NewPath("spec").Child("template").Child("spec").Child("Version"))...) + + allErrs = append(allErrs, validateLoadBalancerProfile( + mcp.Spec.Template.Spec.LoadBalancerProfile, + field.NewPath("spec").Child("template").Child("spec").Child("LoadBalancerProfile"))...) + + allErrs = append(allErrs, validateManagedClusterNetwork( + cli, + mcp.Labels, + mcp.Namespace, + mcp.Spec.Template.Spec.DNSServiceIP, + mcp.Spec.Template.Spec.VirtualNetwork.Subnet, + field.NewPath("spec").Child("template").Child("spec"))...) + + allErrs = append(allErrs, validateName(mcp.Name, field.NewPath("Name"))...) + + allErrs = append(allErrs, validateAutoScalerProfile(mcp.Spec.Template.Spec.AutoScalerProfile, field.NewPath("spec").Child("template").Child("spec").Child("AutoScalerProfile"))...) + + return allErrs.ToAggregate() +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (mcpw *azureManagedControlPlaneTemplateWebhook) ValidateDelete(ctx context.Context, _ runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +// validateVirtualNetworkTemplateUpdate validates update to VirtualNetworkTemplate. +func (mcp *AzureManagedControlPlaneTemplate) validateVirtualNetworkTemplateUpdate(old *AzureManagedControlPlaneTemplate) field.ErrorList { + var allErrs field.ErrorList + if old.Spec.Template.Spec.VirtualNetwork.Name != mcp.Spec.Template.Spec.VirtualNetwork.Name { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Template", "Spec", "VirtualNetwork.Name"), + mcp.Spec.Template.Spec.VirtualNetwork.Name, + "Virtual Network Name is immutable")) + } + + if old.Spec.Template.Spec.VirtualNetwork.CIDRBlock != mcp.Spec.Template.Spec.VirtualNetwork.CIDRBlock { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Template", "Spec", "VirtualNetwork.CIDRBlock"), + mcp.Spec.Template.Spec.VirtualNetwork.CIDRBlock, + "Virtual Network CIDRBlock is immutable")) + } + + if old.Spec.Template.Spec.VirtualNetwork.Subnet.Name != mcp.Spec.Template.Spec.VirtualNetwork.Subnet.Name { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Template", "Spec", "VirtualNetwork.Subnet.Name"), + mcp.Spec.Template.Spec.VirtualNetwork.Subnet.Name, + "Subnet Name is immutable")) + } + + // NOTE: This only works because we force the user to set the CIDRBlock for both the + // managed and unmanaged Vnets. If we ever update the subnet cidr based on what's + // actually set in the subnet, and it is different from what's in the Spec, for + // unmanaged Vnets like we do with the AzureCluster this logic will break. + if old.Spec.Template.Spec.VirtualNetwork.Subnet.CIDRBlock != mcp.Spec.Template.Spec.VirtualNetwork.Subnet.CIDRBlock { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Template", "Spec", "VirtualNetwork.Subnet.CIDRBlock"), + mcp.Spec.Template.Spec.VirtualNetwork.Subnet.CIDRBlock, + "Subnet CIDRBlock is immutable")) + } + + return allErrs +} + +// validateAPIServerAccessProfileTemplateUpdate validates update to APIServerAccessProfileTemplate. +func (mcp *AzureManagedControlPlaneTemplate) validateAPIServerAccessProfileTemplateUpdate(old *AzureManagedControlPlaneTemplate) field.ErrorList { + var allErrs field.ErrorList + + newAPIServerAccessProfileNormalized := &APIServerAccessProfile{} + oldAPIServerAccessProfileNormalized := &APIServerAccessProfile{} + if mcp.Spec.Template.Spec.APIServerAccessProfile != nil { + newAPIServerAccessProfileNormalized = &APIServerAccessProfile{ + APIServerAccessProfileClassSpec: APIServerAccessProfileClassSpec{ + EnablePrivateCluster: mcp.Spec.Template.Spec.APIServerAccessProfile.EnablePrivateCluster, + PrivateDNSZone: mcp.Spec.Template.Spec.APIServerAccessProfile.PrivateDNSZone, + EnablePrivateClusterPublicFQDN: mcp.Spec.Template.Spec.APIServerAccessProfile.EnablePrivateClusterPublicFQDN, + }, + } + } + if old.Spec.Template.Spec.APIServerAccessProfile != nil { + oldAPIServerAccessProfileNormalized = &APIServerAccessProfile{ + APIServerAccessProfileClassSpec: APIServerAccessProfileClassSpec{ + EnablePrivateCluster: old.Spec.Template.Spec.APIServerAccessProfile.EnablePrivateCluster, + PrivateDNSZone: old.Spec.Template.Spec.APIServerAccessProfile.PrivateDNSZone, + EnablePrivateClusterPublicFQDN: old.Spec.Template.Spec.APIServerAccessProfile.EnablePrivateClusterPublicFQDN, + }, + } + } + + if !reflect.DeepEqual(newAPIServerAccessProfileNormalized, oldAPIServerAccessProfileNormalized) { + allErrs = append(allErrs, + field.Invalid(field.NewPath("Spec", "Template", "Spec", "APIServerAccessProfile"), + mcp.Spec.Template.Spec.APIServerAccessProfile, "fields are immutable"), + ) + } + + return allErrs +} diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go new file mode 100644 index 00000000000..326d9c40eeb --- /dev/null +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go @@ -0,0 +1,370 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +func TestControlPlaneTemplateDefaultingWebhook(t *testing.T) { + g := NewWithT(t) + + t.Logf("Testing amcp defaulting webhook with no baseline") + amcpt := getAzureManagedControlPlaneTemplate() + mcptw := &azureManagedControlPlaneTemplateWebhook{} + err := mcptw.Default(context.Background(), amcpt) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(*amcpt.Spec.Template.Spec.NetworkPlugin).To(Equal("azure")) + g.Expect(*amcpt.Spec.Template.Spec.LoadBalancerSKU).To(Equal("Standard")) + g.Expect(amcpt.Spec.Template.Spec.Version).To(Equal("v1.17.5")) + g.Expect(amcpt.Spec.Template.Spec.VirtualNetwork.Name).To(Equal("fooName")) + g.Expect(amcpt.Spec.Template.Spec.VirtualNetwork.CIDRBlock).To(Equal(defaultAKSVnetCIDR)) + g.Expect(amcpt.Spec.Template.Spec.VirtualNetwork.Subnet.Name).To(Equal("fooName")) + g.Expect(amcpt.Spec.Template.Spec.VirtualNetwork.Subnet.CIDRBlock).To(Equal(defaultAKSNodeSubnetCIDR)) + g.Expect(amcpt.Spec.Template.Spec.SKU.Tier).To(Equal(FreeManagedControlPlaneTier)) + + t.Logf("Testing amcp defaulting webhook with baseline") + netPlug := "kubenet" + lbSKU := "Basic" + netPol := "azure" + amcpt.Spec.Template.Spec.NetworkPlugin = &netPlug + amcpt.Spec.Template.Spec.LoadBalancerSKU = &lbSKU + amcpt.Spec.Template.Spec.NetworkPolicy = &netPol + amcpt.Spec.Template.Spec.Version = "9.99.99" + amcpt.Spec.Template.Spec.VirtualNetwork.Name = "fooVnetName" + amcpt.Spec.Template.Spec.VirtualNetwork.Subnet.Name = "fooSubnetName" + amcpt.Spec.Template.Spec.SKU.Tier = PaidManagedControlPlaneTier + + err = mcptw.Default(context.Background(), amcpt) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(*amcpt.Spec.Template.Spec.NetworkPlugin).To(Equal(netPlug)) + g.Expect(*amcpt.Spec.Template.Spec.LoadBalancerSKU).To(Equal(lbSKU)) + g.Expect(*amcpt.Spec.Template.Spec.NetworkPolicy).To(Equal(netPol)) + g.Expect(amcpt.Spec.Template.Spec.Version).To(Equal("v9.99.99")) + g.Expect(amcpt.Spec.Template.Spec.VirtualNetwork.Name).To(Equal("fooVnetName")) + g.Expect(amcpt.Spec.Template.Spec.VirtualNetwork.Subnet.Name).To(Equal("fooSubnetName")) + g.Expect(amcpt.Spec.Template.Spec.SKU.Tier).To(Equal(StandardManagedControlPlaneTier)) +} + +func TestControlPlaneTemplateUpdateWebhook(t *testing.T) { + tests := []struct { + name string + oldControlPlaneTemplate *AzureManagedControlPlaneTemplate + controlPlaneTemplate *AzureManagedControlPlaneTemplate + wantErr bool + }{ + { + name: "azuremanagedcontrolplanetemplate no changes - valid spec", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(), + wantErr: false, + }, + { + name: "azuremanagedcontrolplanetemplate subscriptionID is immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.SubscriptionID = "foo" + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.SubscriptionID = "bar" + }), + wantErr: true, + }, + { + name: "azuremanagedcontrolplanetemplate location is immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.Location = "foo" + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.Location = "bar" + }), + wantErr: true, + }, + { + name: "azuremanagedcontrolplanetemplate DNSServiceIP is immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.DNSServiceIP = ptr.To("foo") + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.DNSServiceIP = ptr.To("bar") + }), + wantErr: true, + }, + { + name: "azuremanagedcontrolplanetemplate NetworkPlugin is immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.NetworkPlugin = ptr.To("foo") + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.NetworkPlugin = ptr.To("bar") + }), + wantErr: true, + }, + { + name: "azuremanagedcontrolplanetemplate NetworkPolicy is immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.NetworkPolicy = ptr.To("foo") + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.NetworkPolicy = ptr.To("bar") + }), + wantErr: true, + }, + { + name: "azuremanagedcontrolplanetemplate LoadBalancerSKU is immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.LoadBalancerSKU = ptr.To("foo") + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.LoadBalancerSKU = ptr.To("bar") + }), + wantErr: true, + }, + { + name: "cannot disable AADProfile", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.AADProfile = &AADProfile{ + Managed: true, + } + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(), + wantErr: true, + }, + { + name: "cannot set AADProfile.Managed to false", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.AADProfile = &AADProfile{ + Managed: true, + } + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.AADProfile = &AADProfile{ + Managed: false, + } + }), + wantErr: true, + }, + { + name: "length of AADProfile.AdminGroupObjectIDs cannot be zero", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.AADProfile = &AADProfile{ + AdminGroupObjectIDs: []string{"foo"}, + } + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.AADProfile = &AADProfile{ + AdminGroupObjectIDs: []string{}, + } + }), + wantErr: true, + }, + { + name: "azuremanagedcontrolplanetemplate OutboundType is immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.OutboundType = (*ManagedControlPlaneOutboundType)(ptr.To(string(ManagedControlPlaneOutboundTypeLoadBalancer))) + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.OutboundType = (*ManagedControlPlaneOutboundType)(ptr.To(string(ManagedControlPlaneOutboundTypeManagedNATGateway))) + }), + wantErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + cpw := &azureManagedControlPlaneTemplateWebhook{} + _, err := cpw.ValidateUpdate(context.Background(), tc.oldControlPlaneTemplate, tc.controlPlaneTemplate) + if tc.wantErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} + +func TestValidateVirtualNetworkTemplateUpdate(t *testing.T) { + tests := []struct { + name string + oldControlPlaneTemplate *AzureManagedControlPlaneTemplate + controlPlaneTemplate *AzureManagedControlPlaneTemplate + wantErr bool + }{ + { + name: "azuremanagedcontrolplanetemplate no changes - valid spec", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(), + wantErr: false, + }, + { + name: "azuremanagedcontrolplanetemplate name is immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.VirtualNetwork = ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Name: "fooName", + }, + } + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.VirtualNetwork = ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + Name: "barName", + }, + } + }), + wantErr: true, + }, + { + name: "azuremanagedcontrolplanetemplate networkCIDRBlock is immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.VirtualNetwork = ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + CIDRBlock: "fooCIDRBlock", + }, + } + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.VirtualNetwork = ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: ManagedControlPlaneVirtualNetworkClassSpec{ + CIDRBlock: "barCIDRBlock", + }, + } + }), + wantErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + allErrs := tc.controlPlaneTemplate.validateVirtualNetworkTemplateUpdate(tc.oldControlPlaneTemplate) + if tc.wantErr { + g.Expect(allErrs).NotTo(BeEmpty()) + } else { + g.Expect(allErrs).To(BeEmpty()) + } + }) + } +} + +func TestValidateAPIServerAccessProfileUpdate(t *testing.T) { + tests := []struct { + name string + oldControlPlaneTemplate *AzureManagedControlPlaneTemplate + controlPlaneTemplate *AzureManagedControlPlaneTemplate + wantErr bool + }{ + { + name: "azuremanagedcontrolplanetemplate no changes - valid spec", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(), + wantErr: false, + }, + { + name: "azuremanagedcontrolplanetemplate enablePrivateCluster is immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.APIServerAccessProfile = &APIServerAccessProfile{ + APIServerAccessProfileClassSpec: APIServerAccessProfileClassSpec{ + EnablePrivateCluster: ptr.To(true), + }, + } + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.APIServerAccessProfile = &APIServerAccessProfile{ + APIServerAccessProfileClassSpec: APIServerAccessProfileClassSpec{ + EnablePrivateCluster: ptr.To(false), + }, + } + }), + wantErr: true, + }, + { + name: "azuremanagedcontrolplanetemplate privateDNSZone is immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.APIServerAccessProfile = &APIServerAccessProfile{ + APIServerAccessProfileClassSpec: APIServerAccessProfileClassSpec{ + PrivateDNSZone: ptr.To("foo"), + }, + } + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.APIServerAccessProfile = &APIServerAccessProfile{ + APIServerAccessProfileClassSpec: APIServerAccessProfileClassSpec{ + PrivateDNSZone: ptr.To("bar"), + }, + } + }), + wantErr: true, + }, + { + name: "azuremanagedcontrolplanetemplate enablePrivateClusterPublicFQDN is immutable", + oldControlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.APIServerAccessProfile = &APIServerAccessProfile{ + APIServerAccessProfileClassSpec: APIServerAccessProfileClassSpec{ + EnablePrivateClusterPublicFQDN: ptr.To(true), + }, + } + }), + controlPlaneTemplate: getAzureManagedControlPlaneTemplate(func(cpt *AzureManagedControlPlaneTemplate) { + cpt.Spec.Template.Spec.APIServerAccessProfile = &APIServerAccessProfile{ + APIServerAccessProfileClassSpec: APIServerAccessProfileClassSpec{ + EnablePrivateClusterPublicFQDN: ptr.To(false), + }, + } + }), + wantErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + allErrs := tc.controlPlaneTemplate.validateAPIServerAccessProfileTemplateUpdate(tc.oldControlPlaneTemplate) + if tc.wantErr { + g.Expect(allErrs).NotTo(BeEmpty()) + } else { + g.Expect(allErrs).To(BeEmpty()) + } + }) + } +} + +func getAzureManagedControlPlaneTemplate(changes ...func(*AzureManagedControlPlaneTemplate)) *AzureManagedControlPlaneTemplate { + input := &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fooName", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ + Location: "fooLocation", + Version: "v1.17.5", + }, + }, + }, + }, + } + + for _, change := range changes { + change(input) + } + + return input +} diff --git a/api/v1beta1/azuremanagedmachinepool_types.go b/api/v1beta1/azuremanagedmachinepool_types.go index 0a59c29ced9..eea54107af0 100644 --- a/api/v1beta1/azuremanagedmachinepool_types.go +++ b/api/v1beta1/azuremanagedmachinepool_types.go @@ -17,7 +17,6 @@ limitations under the License. package v1beta1 import ( - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" capierrors "sigs.k8s.io/cluster-api/errors" @@ -419,156 +418,11 @@ type LinuxOSConfig struct { // AzureManagedMachinePoolSpec defines the desired state of AzureManagedMachinePool. type AzureManagedMachinePoolSpec struct { - - // AdditionalTags is an optional set of tags to add to Azure resources managed by the - // Azure provider, in addition to the ones added by default. - // +optional - AdditionalTags Tags `json:"additionalTags,omitempty"` - - // Name - name of the agent pool. If not specified, CAPZ uses the name of the CR as the agent pool name. - // Immutable. - // +optional - Name *string `json:"name,omitempty"` - - // Mode - represents mode of an agent pool. Possible values include: System, User. - // +kubebuilder:validation:Enum=System;User - Mode string `json:"mode"` - - // SKU is the size of the VMs in the node pool. - // Immutable. - SKU string `json:"sku"` - - // OSDiskSizeGB is the disk size for every machine in this agent pool. - // If you specify 0, it will apply the default osDisk size according to the vmSize specified. - // Immutable. - // +optional - OSDiskSizeGB *int `json:"osDiskSizeGB,omitempty"` - - // AvailabilityZones - Availability zones for nodes. Must use VirtualMachineScaleSets AgentPoolType. - // Immutable. - // +optional - AvailabilityZones []string `json:"availabilityZones,omitempty"` - - // Node labels - labels for all of the nodes present in node pool. - // See also [AKS doc]. - // - // [AKS doc]: https://learn.microsoft.com/azure/aks/use-labels - // +optional - NodeLabels map[string]string `json:"nodeLabels,omitempty"` - - // Taints specifies the taints for nodes present in this agent pool. - // See also [AKS doc]. - // - // [AKS doc]: https://learn.microsoft.com/azure/aks/use-multiple-node-pools#setting-node-pool-taints - // +optional - Taints Taints `json:"taints,omitempty"` + AzureManagedMachinePoolClassSpec `json:",inline"` // ProviderIDList is the unique identifier as specified by the cloud provider. // +optional ProviderIDList []string `json:"providerIDList,omitempty"` - - // Scaling specifies the autoscaling parameters for the node pool. - // +optional - Scaling *ManagedMachinePoolScaling `json:"scaling,omitempty"` - - // MaxPods specifies the kubelet `--max-pods` configuration for the node pool. - // Immutable. - // See also [AKS doc], [K8s doc]. - // - // [AKS doc]: https://learn.microsoft.com/azure/aks/configure-azure-cni#configure-maximum---new-clusters - // [K8s doc]: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/ - // +optional - MaxPods *int `json:"maxPods,omitempty"` - - // OsDiskType specifies the OS disk type for each node in the pool. Allowed values are 'Ephemeral' and 'Managed' (default). - // Immutable. - // See also [AKS doc]. - // - // [AKS doc]: https://learn.microsoft.com/azure/aks/cluster-configuration#ephemeral-os - // +kubebuilder:validation:Enum=Ephemeral;Managed - // +kubebuilder:default=Managed - // +optional - OsDiskType *string `json:"osDiskType,omitempty"` - - // EnableUltraSSD enables the storage type UltraSSD_LRS for the agent pool. - // Immutable. - // +optional - EnableUltraSSD *bool `json:"enableUltraSSD,omitempty"` - - // OSType specifies the virtual machine operating system. Default to Linux. Possible values include: 'Linux', 'Windows'. - // 'Windows' requires the AzureManagedControlPlane's `spec.networkPlugin` to be `azure`. - // Immutable. - // See also [AKS doc]. - // - // [AKS doc]: https://learn.microsoft.com/rest/api/aks/agent-pools/create-or-update?tabs=HTTP#ostype - // +kubebuilder:validation:Enum=Linux;Windows - // +optional - OSType *string `json:"osType,omitempty"` - - // EnableNodePublicIP controls whether or not nodes in the pool each have a public IP address. - // Immutable. - // +optional - EnableNodePublicIP *bool `json:"enableNodePublicIP,omitempty"` - - // NodePublicIPPrefixID specifies the public IP prefix resource ID which VM nodes should use IPs from. - // Immutable. - // +optional - NodePublicIPPrefixID *string `json:"nodePublicIPPrefixID,omitempty"` - - // ScaleSetPriority specifies the ScaleSetPriority value. Default to Regular. Possible values include: 'Regular', 'Spot' - // Immutable. - // +kubebuilder:validation:Enum=Regular;Spot - // +optional - ScaleSetPriority *string `json:"scaleSetPriority,omitempty"` - - // ScaleDownMode affects the cluster autoscaler behavior. Default to Delete. Possible values include: 'Deallocate', 'Delete' - // +kubebuilder:validation:Enum=Deallocate;Delete - // +kubebuilder:default=Delete - // +optional - ScaleDownMode *string `json:"scaleDownMode,omitempty"` - - // SpotMaxPrice defines max price to pay for spot instance. Possible values are any decimal value greater than zero or -1. - // If you set the max price to be -1, the VM won't be evicted based on price. The price for the VM will be the current price - // for spot or the price for a standard VM, which ever is less, as long as there's capacity and quota available. - // +optional - SpotMaxPrice *resource.Quantity `json:"spotMaxPrice,omitempty"` - - // KubeletConfig specifies the kubelet configurations for nodes. - // Immutable. - // +optional - KubeletConfig *KubeletConfig `json:"kubeletConfig,omitempty"` - - // KubeletDiskType specifies the kubelet disk type. Default to OS. Possible values include: 'OS', 'Temporary'. - // Requires Microsoft.ContainerService/KubeletDisk preview feature to be set. - // Immutable. - // See also [AKS doc]. - // - // [AKS doc]: https://learn.microsoft.com/rest/api/aks/agent-pools/create-or-update?tabs=HTTP#kubeletdisktype - // +kubebuilder:validation:Enum=OS;Temporary - // +optional - KubeletDiskType *KubeletDiskType `json:"kubeletDiskType,omitempty"` - - // LinuxOSConfig specifies the custom Linux OS settings and configurations. - // Immutable. - // +optional - LinuxOSConfig *LinuxOSConfig `json:"linuxOSConfig,omitempty"` - // SubnetName specifies the Subnet where the MachinePool will be placed - // Immutable. - // +optional - SubnetName *string `json:"subnetName,omitempty"` - - // EnableFIPS indicates whether FIPS is enabled on the node pool. - // Immutable. - // +optional - EnableFIPS *bool `json:"enableFIPS,omitempty"` - - // EnableEncryptionAtHost indicates whether host encryption is enabled on the node pool. - // Immutable. - // See also [AKS doc]. - // - // [AKS doc]: https://learn.microsoft.com/en-us/azure/aks/enable-host-encryption - // +optional - EnableEncryptionAtHost *bool `json:"enableEncryptionAtHost,omitempty"` } // ManagedMachinePoolScaling specifies scaling options. diff --git a/api/v1beta1/azuremanagedmachinepool_webhook.go b/api/v1beta1/azuremanagedmachinepool_webhook.go index 56b8e6eae21..fd06dec4986 100644 --- a/api/v1beta1/azuremanagedmachinepool_webhook.go +++ b/api/v1beta1/azuremanagedmachinepool_webhook.go @@ -98,24 +98,49 @@ func (mw *azureManagedMachinePoolWebhook) ValidateCreate(ctx context.Context, ob "can be set only if the Cluster API 'MachinePool' feature flag is enabled", ) } - validators := []func() error{ - m.validateMaxPods, - m.validateOSType, - m.validateName, - m.validateNodeLabels, - m.validateNodePublicIPPrefixID, - m.validateEnableNodePublicIP, - m.validateKubeletConfig, - m.validateLinuxOSConfig, - m.validateSubnetName, - } var errs []error - for _, validator := range validators { - if err := validator(); err != nil { - errs = append(errs, err) - } - } + + errs = append(errs, validateMaxPods( + m.Spec.MaxPods, + field.NewPath("Spec", "MaxPods"))) + + errs = append(errs, validateOSType( + m.Spec.Mode, + m.Spec.OSType, + field.NewPath("Spec", "OSType"))) + + errs = append(errs, validateMPName( + m.Name, + m.Spec.Name, + m.Spec.OSType, + field.NewPath("Spec", "Name"))) + + errs = append(errs, validateNodeLabels( + m.Spec.NodeLabels, + field.NewPath("Spec", "NodeLabels"))) + + errs = append(errs, validateNodePublicIPPrefixID( + m.Spec.NodePublicIPPrefixID, + field.NewPath("Spec", "NodePublicIPPrefixID"))) + + errs = append(errs, validateEnableNodePublicIP( + m.Spec.EnableNodePublicIP, + m.Spec.NodePublicIPPrefixID, + field.NewPath("Spec", "EnableNodePublicIP"))) + + errs = append(errs, validateKubeletConfig( + m.Spec.KubeletConfig, + field.NewPath("Spec", "KubeletConfig"))) + + errs = append(errs, validateLinuxOSConfig( + m.Spec.LinuxOSConfig, + m.Spec.KubeletConfig, + field.NewPath("Spec", "LinuxOSConfig"))) + + errs = append(errs, validateMPSubnetName( + m.Spec.SubnetName, + field.NewPath("Spec", "SubnetName"))) return nil, kerrors.NewAggregate(errs) } @@ -139,7 +164,7 @@ func (mw *azureManagedMachinePoolWebhook) ValidateUpdate(ctx context.Context, ol allErrs = append(allErrs, err) } - if err := m.validateNodeLabels(); err != nil { + if err := validateNodeLabels(m.Spec.NodeLabels, field.NewPath("Spec", "NodeLabels")); err != nil { allErrs = append(allErrs, field.Invalid( field.NewPath("Spec", "NodeLabels"), @@ -199,7 +224,7 @@ func (mw *azureManagedMachinePoolWebhook) ValidateUpdate(ctx context.Context, ol if m.Spec.Mode != string(NodePoolModeSystem) && old.Spec.Mode == string(NodePoolModeSystem) { // validate for last system node pool - if err := m.validateLastSystemNodePool(mw.Client); err != nil { + if err := validateLastSystemNodePool(mw.Client, m.Labels, m.Namespace); err != nil { allErrs = append(allErrs, field.Forbidden( field.NewPath("Spec", "Mode"), "Cannot change node pool mode to User, you must have at least one System node pool in your cluster")) @@ -284,23 +309,23 @@ func (mw *azureManagedMachinePoolWebhook) ValidateDelete(ctx context.Context, ob return nil, nil } - return nil, errors.Wrapf(m.validateLastSystemNodePool(mw.Client), "if the delete is triggered via owner MachinePool please refer to trouble shooting section in https://capz.sigs.k8s.io/topics/managedcluster.html") + return nil, errors.Wrapf(validateLastSystemNodePool(mw.Client, m.Labels, m.Namespace), "if the delete is triggered via owner MachinePool please refer to trouble shooting section in https://capz.sigs.k8s.io/topics/managedcluster.html") } // validateLastSystemNodePool is used to check if the existing system node pool is the last system node pool. // If it is a last system node pool it cannot be deleted or mutated to user node pool as AKS expects min 1 system node pool. -func (m *AzureManagedMachinePool) validateLastSystemNodePool(cli client.Client) error { +func validateLastSystemNodePool(cli client.Client, labels map[string]string, namespace string) error { ctx := context.Background() // Fetch the Cluster. - clusterName, ok := m.Labels[clusterv1.ClusterNameLabel] + clusterName, ok := labels[clusterv1.ClusterNameLabel] if !ok { return nil } ownerCluster := &clusterv1.Cluster{} key := client.ObjectKey{ - Namespace: m.Namespace, + Namespace: namespace, Name: clusterName, } @@ -317,7 +342,7 @@ func (m *AzureManagedMachinePool) validateLastSystemNodePool(cli client.Client) return nil } - opt1 := client.InNamespace(m.Namespace) + opt1 := client.InNamespace(namespace) opt2 := client.MatchingLabels(map[string]string{ clusterv1.ClusterNameLabel: clusterName, LabelAgentPoolMode: string(NodePoolModeSystem), @@ -334,12 +359,12 @@ func (m *AzureManagedMachinePool) validateLastSystemNodePool(cli client.Client) return nil } -func (m *AzureManagedMachinePool) validateMaxPods() error { - if m.Spec.MaxPods != nil { - if ptr.Deref(m.Spec.MaxPods, 0) < 10 || ptr.Deref(m.Spec.MaxPods, 0) > 250 { +func validateMaxPods(maxPods *int, fldPath *field.Path) error { + if maxPods != nil { + if ptr.Deref(maxPods, 0) < 10 || ptr.Deref(maxPods, 0) > 250 { return field.Invalid( - field.NewPath("Spec", "MaxPods"), - m.Spec.MaxPods, + fldPath, + maxPods, "MaxPods must be between 10 and 250") } } @@ -347,11 +372,11 @@ func (m *AzureManagedMachinePool) validateMaxPods() error { return nil } -func (m *AzureManagedMachinePool) validateOSType() error { - if m.Spec.Mode == string(NodePoolModeSystem) { - if m.Spec.OSType != nil && *m.Spec.OSType != LinuxOS { +func validateOSType(mode string, osType *string, fldPath *field.Path) error { + if mode == string(NodePoolModeSystem) { + if osType != nil && *osType != LinuxOS { return field.Forbidden( - field.NewPath("Spec", "OSType"), + fldPath, "System node pooll must have OSType 'Linux'") } } @@ -359,48 +384,48 @@ func (m *AzureManagedMachinePool) validateOSType() error { return nil } -func (m *AzureManagedMachinePool) validateName() error { +func validateMPName(mpName string, specName *string, osType *string, fldPath *field.Path) error { var name *string var fieldNameMessage string - if m.Spec.Name == nil || *m.Spec.Name == "" { - name = &m.Name + if specName == nil || *specName == "" { + name = &mpName fieldNameMessage = "when spec.name is empty, metadata.name" } else { - name = m.Spec.Name + name = specName fieldNameMessage = "spec.name" } - if err := validateNameLength(m.Spec.OSType, name, fieldNameMessage); err != nil { + if err := validateNameLength(osType, name, fieldNameMessage, fldPath); err != nil { return err } - return validateNamePattern(name, fieldNameMessage) + return validateNamePattern(name, fieldNameMessage, fldPath) } -func validateNameLength(osType *string, name *string, fieldNameMessage string) error { +func validateNameLength(osType *string, name *string, fieldNameMessage string, fldPath *field.Path) error { if osType != nil && *osType == WindowsOS && name != nil && len(*name) > 6 { return field.Invalid( - field.NewPath("Spec", "Name"), + fldPath, name, fmt.Sprintf("For OSType Windows, %s can not be longer than 6 characters.", fieldNameMessage)) } else if (osType == nil || *osType == LinuxOS) && (name != nil && len(*name) > 12) { return field.Invalid( - field.NewPath("Spec", "Name"), + fldPath, osType, fmt.Sprintf("For OSType Linux, %s can not be longer than 12 characters.", fieldNameMessage)) } return nil } -func validateNamePattern(name *string, fieldNameMessage string) error { +func validateNamePattern(name *string, fieldNameMessage string, fldPath *field.Path) error { if name == nil || *name == "" { return nil } if !unicode.IsLower(rune((*name)[0])) { return field.Invalid( - field.NewPath("Spec", "Name"), + fldPath, name, fmt.Sprintf("%s must begin with a lowercase letter.", fieldNameMessage)) } @@ -408,7 +433,7 @@ func validateNamePattern(name *string, fieldNameMessage string) error { for _, char := range *name { if !(unicode.IsLower(char) || unicode.IsNumber(char)) { return field.Invalid( - field.NewPath("Spec", "Name"), + fldPath, name, fmt.Sprintf("%s may only contain lowercase alphanumeric characters.", fieldNameMessage)) } @@ -416,11 +441,11 @@ func validateNamePattern(name *string, fieldNameMessage string) error { return nil } -func (m *AzureManagedMachinePool) validateNodeLabels() error { - for key := range m.Spec.NodeLabels { +func validateNodeLabels(nodeLabels map[string]string, fldPath *field.Path) error { + for key := range nodeLabels { if azureutil.IsAzureSystemNodeLabelKey(key) { return field.Invalid( - field.NewPath("Spec", "NodeLabels"), + fldPath, key, fmt.Sprintf("Node pool label key must not start with %s", azureutil.AzureSystemNodeLabelPrefix)) } @@ -429,33 +454,33 @@ func (m *AzureManagedMachinePool) validateNodeLabels() error { return nil } -func (m *AzureManagedMachinePool) validateNodePublicIPPrefixID() error { - if m.Spec.NodePublicIPPrefixID != nil && !validNodePublicPrefixID.MatchString(*m.Spec.NodePublicIPPrefixID) { +func validateNodePublicIPPrefixID(nodePublicIPPrefixID *string, fldPath *field.Path) error { + if nodePublicIPPrefixID != nil && !validNodePublicPrefixID.MatchString(*nodePublicIPPrefixID) { return field.Invalid( - field.NewPath("Spec", "NodePublicIPPrefixID"), - m.Spec.NodePublicIPPrefixID, + fldPath, + nodePublicIPPrefixID, fmt.Sprintf("resource ID must match %q", validNodePublicPrefixID.String())) } return nil } -func (m *AzureManagedMachinePool) validateEnableNodePublicIP() error { - if (m.Spec.EnableNodePublicIP == nil || !*m.Spec.EnableNodePublicIP) && - m.Spec.NodePublicIPPrefixID != nil { +func validateEnableNodePublicIP(enableNodePublicIP *bool, nodePublicIPPrefixID *string, fldPath *field.Path) error { + if (enableNodePublicIP == nil || !*enableNodePublicIP) && + nodePublicIPPrefixID != nil { return field.Invalid( - field.NewPath("Spec", "EnableNodePublicIP"), - m.Spec.EnableNodePublicIP, + fldPath, + enableNodePublicIP, "must be set to true when NodePublicIPPrefixID is set") } return nil } -func (m *AzureManagedMachinePool) validateSubnetName() error { - if m.Spec.SubnetName != nil { +func validateMPSubnetName(subnetName *string, fldPath *field.Path) error { + if subnetName != nil { subnetRegex := "^[a-zA-Z0-9][a-zA-Z0-9-]{0,78}[a-zA-Z0-9]$" regex := regexp.MustCompile(subnetRegex) - if success := regex.MatchString(ptr.Deref(m.Spec.SubnetName, "")); !success { - return field.Invalid(field.NewPath("Spec", "SubnetName"), m.Spec.SubnetName, + if success := regex.MatchString(ptr.Deref(subnetName, "")); !success { + return field.Invalid(fldPath, subnetName, fmt.Sprintf("name of subnet doesn't match regex %s", subnetRegex)) } } @@ -464,7 +489,7 @@ func (m *AzureManagedMachinePool) validateSubnetName() error { // validateKubeletConfig enforces the AKS API configuration for KubeletConfig. // See: https://learn.microsoft.com/en-us/azure/aks/custom-node-configuration. -func (m *AzureManagedMachinePool) validateKubeletConfig() error { +func validateKubeletConfig(kubeletConfig *KubeletConfig, fldPath *field.Path) error { var allowedUnsafeSysctlsPatterns = []string{ `^kernel\.shm.+$`, `^kernel\.msg.+$`, @@ -472,25 +497,25 @@ func (m *AzureManagedMachinePool) validateKubeletConfig() error { `^fs\.mqueue\..+$`, `^net\..+$`, } - if m.Spec.KubeletConfig != nil { - if m.Spec.KubeletConfig.CPUCfsQuotaPeriod != nil { - if !strings.HasSuffix(ptr.Deref(m.Spec.KubeletConfig.CPUCfsQuotaPeriod, ""), "ms") { + if kubeletConfig != nil { + if kubeletConfig.CPUCfsQuotaPeriod != nil { + if !strings.HasSuffix(ptr.Deref(kubeletConfig.CPUCfsQuotaPeriod, ""), "ms") { return field.Invalid( - field.NewPath("Spec", "KubeletConfig", "CPUCfsQuotaPeriod"), - m.Spec.KubeletConfig.CPUCfsQuotaPeriod, + fldPath.Child("CPUfsQuotaPeriod"), + kubeletConfig.CPUCfsQuotaPeriod, "must be a string value in milliseconds with a 'ms' suffix, e.g., '100ms'") } } - if m.Spec.KubeletConfig.ImageGcHighThreshold != nil && m.Spec.KubeletConfig.ImageGcLowThreshold != nil { - if ptr.Deref(m.Spec.KubeletConfig.ImageGcLowThreshold, 0) > ptr.Deref(m.Spec.KubeletConfig.ImageGcHighThreshold, 0) { + if kubeletConfig.ImageGcHighThreshold != nil && kubeletConfig.ImageGcLowThreshold != nil { + if ptr.Deref(kubeletConfig.ImageGcLowThreshold, 0) > ptr.Deref(kubeletConfig.ImageGcHighThreshold, 0) { return field.Invalid( - field.NewPath("Spec", "KubeletConfig", "ImageGcLowThreshold"), - m.Spec.KubeletConfig.ImageGcLowThreshold, + fldPath.Child("ImageGcLowThreshold"), + kubeletConfig.ImageGcLowThreshold, fmt.Sprintf("must not be greater than ImageGcHighThreshold, ImageGcLowThreshold=%d, ImageGcHighThreshold=%d", - ptr.Deref(m.Spec.KubeletConfig.ImageGcLowThreshold, 0), ptr.Deref(m.Spec.KubeletConfig.ImageGcHighThreshold, 0))) + ptr.Deref(kubeletConfig.ImageGcLowThreshold, 0), ptr.Deref(kubeletConfig.ImageGcHighThreshold, 0))) } } - for _, val := range m.Spec.KubeletConfig.AllowedUnsafeSysctls { + for _, val := range kubeletConfig.AllowedUnsafeSysctls { var hasMatch bool for _, p := range allowedUnsafeSysctlsPatterns { if m, _ := regexp.MatchString(p, val); m { @@ -500,8 +525,8 @@ func (m *AzureManagedMachinePool) validateKubeletConfig() error { } if !hasMatch { return field.Invalid( - field.NewPath("Spec", "KubeletConfig", "AllowedUnsafeSysctls"), - m.Spec.KubeletConfig.AllowedUnsafeSysctls, + fldPath.Child("AllowedUnsafeSysctls"), + kubeletConfig.AllowedUnsafeSysctls, fmt.Sprintf("%s is not a supported AllowedUnsafeSysctls configuration", val)) } } @@ -511,25 +536,25 @@ func (m *AzureManagedMachinePool) validateKubeletConfig() error { // validateLinuxOSConfig enforces AKS API configuration for Linux OS custom configuration // See: https://learn.microsoft.com/en-us/azure/aks/custom-node-configuration#linux-os-custom-configuration for detailed information. -func (m *AzureManagedMachinePool) validateLinuxOSConfig() error { +func validateLinuxOSConfig(linuxOSConfig *LinuxOSConfig, kubeletConfig *KubeletConfig, fldPath *field.Path) error { var errs []error - if m.Spec.LinuxOSConfig == nil { + if linuxOSConfig == nil { return nil } - if m.Spec.LinuxOSConfig.SwapFileSizeMB != nil { - if m.Spec.KubeletConfig == nil || ptr.Deref(m.Spec.KubeletConfig.FailSwapOn, true) { + if linuxOSConfig.SwapFileSizeMB != nil { + if kubeletConfig == nil || ptr.Deref(kubeletConfig.FailSwapOn, true) { errs = append(errs, field.Invalid( - field.NewPath("Spec", "LinuxOSConfig", "SwapFileSizeMB"), - m.Spec.LinuxOSConfig.SwapFileSizeMB, + fldPath.Child("SwapFileSizeMB"), + linuxOSConfig.SwapFileSizeMB, "KubeletConfig.FailSwapOn must be set to false to enable swap file on nodes")) } } - if m.Spec.LinuxOSConfig.Sysctls != nil && m.Spec.LinuxOSConfig.Sysctls.NetIpv4IPLocalPortRange != nil { + if linuxOSConfig.Sysctls != nil && linuxOSConfig.Sysctls.NetIpv4IPLocalPortRange != nil { // match numbers separated by a space portRangeRegex := `^[0-9]+ [0-9]+$` - portRange := *m.Spec.LinuxOSConfig.Sysctls.NetIpv4IPLocalPortRange + portRange := *linuxOSConfig.Sysctls.NetIpv4IPLocalPortRange match, matchErr := regexp.MatchString(portRangeRegex, portRange) if matchErr != nil { @@ -537,8 +562,8 @@ func (m *AzureManagedMachinePool) validateLinuxOSConfig() error { } if !match { errs = append(errs, field.Invalid( - field.NewPath("Spec", "LinuxOSConfig", "Sysctls", "NetIpv4IpLocalPortRange"), - m.Spec.LinuxOSConfig.Sysctls.NetIpv4IPLocalPortRange, + fldPath.Child("NetIpv4IpLocalPortRange"), + linuxOSConfig.Sysctls.NetIpv4IPLocalPortRange, "LinuxOSConfig.Sysctls.NetIpv4IpLocalPortRange must be of the format \" \"")) } else { ports := strings.Split(portRange, " ") @@ -547,22 +572,22 @@ func (m *AzureManagedMachinePool) validateLinuxOSConfig() error { if firstPort < 1024 || firstPort > 60999 { errs = append(errs, field.Invalid( - field.NewPath("Spec", "LinuxOSConfig", "Sysctls", "NetIpv4IpLocalPortRange", "First"), - m.Spec.LinuxOSConfig.Sysctls.NetIpv4IPLocalPortRange, + fldPath.Child("NetIpv4IpLocalPortRange", "First"), + linuxOSConfig.Sysctls.NetIpv4IPLocalPortRange, fmt.Sprintf("first port of NetIpv4IpLocalPortRange=%d must be in between [1024 - 60999]", firstPort))) } if lastPort < 32768 || lastPort > 65000 { errs = append(errs, field.Invalid( - field.NewPath("Spec", "LinuxOSConfig", "Sysctls", "NetIpv4IpLocalPortRange", "Last"), - m.Spec.LinuxOSConfig.Sysctls.NetIpv4IPLocalPortRange, + fldPath.Child("NetIpv4IpLocalPortRange", "Last"), + linuxOSConfig.Sysctls.NetIpv4IPLocalPortRange, fmt.Sprintf("last port of NetIpv4IpLocalPortRange=%d must be in between [32768 -65000]", lastPort))) } if firstPort > lastPort { errs = append(errs, field.Invalid( - field.NewPath("Spec", "LinuxOSConfig", "Sysctls", "NetIpv4IpLocalPortRange", "First"), - m.Spec.LinuxOSConfig.Sysctls.NetIpv4IPLocalPortRange, + fldPath.Child("NetIpv4IpLocalPortRange", "First"), + linuxOSConfig.Sysctls.NetIpv4IPLocalPortRange, fmt.Sprintf("first port of NetIpv4IpLocalPortRange=%d cannot be greater than last port of NetIpv4IpLocalPortRange=%d", firstPort, lastPort))) } } diff --git a/api/v1beta1/azuremanagedmachinepool_webhook_test.go b/api/v1beta1/azuremanagedmachinepool_webhook_test.go index 529062e8f39..5aa197d876d 100644 --- a/api/v1beta1/azuremanagedmachinepool_webhook_test.go +++ b/api/v1beta1/azuremanagedmachinepool_webhook_test.go @@ -42,9 +42,11 @@ func TestAzureManagedMachinePoolDefaultingWebhook(t *testing.T) { Name: "fooname", }, Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + }, }, } var client client.Client @@ -95,12 +97,16 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Cannot change Name of the agentpool", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Name: ptr.To("pool-new"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Name: ptr.To("pool-new"), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Name: ptr.To("pool-old"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Name: ptr.To("pool-old"), + }, }, }, wantErr: true, @@ -109,16 +115,20 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Cannot change SKU of the agentpool", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V4", - OSDiskSizeGB: ptr.To(512), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V4", + OSDiskSizeGB: ptr.To(512), + }, }, }, wantErr: true, @@ -127,18 +137,22 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Cannot change OSType of the agentpool", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - OSType: ptr.To(LinuxOS), - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + OSType: ptr.To(LinuxOS), + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - OSType: ptr.To(WindowsOS), - Mode: "System", - SKU: "StandardD2S_V4", - OSDiskSizeGB: ptr.To(512), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + OSType: ptr.To(WindowsOS), + Mode: "System", + SKU: "StandardD2S_V4", + OSDiskSizeGB: ptr.To(512), + }, }, }, wantErr: true, @@ -147,16 +161,20 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Cannot change OSDiskSizeGB of the agentpool", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(1024), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(1024), + }, }, }, wantErr: true, @@ -165,17 +183,21 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Cannot add AvailabilityZones after creating agentpool", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - AvailabilityZones: []string{"1", "2", "3"}, + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + AvailabilityZones: []string{"1", "2", "3"}, + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + }, }, }, wantErr: true, @@ -184,17 +206,21 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Cannot remove AvailabilityZones after creating agentpool", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - AvailabilityZones: []string{"1", "2", "3"}, + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + AvailabilityZones: []string{"1", "2", "3"}, + }, }, }, wantErr: true, @@ -203,18 +229,22 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Cannot change AvailabilityZones of the agentpool", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - AvailabilityZones: []string{"1", "2"}, + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + AvailabilityZones: []string{"1", "2"}, + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - AvailabilityZones: []string{"1", "2", "3"}, + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + AvailabilityZones: []string{"1", "2", "3"}, + }, }, }, wantErr: true, @@ -223,18 +253,22 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "AvailabilityZones order can be different", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - AvailabilityZones: []string{"1", "3", "2"}, + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + AvailabilityZones: []string{"1", "3", "2"}, + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - AvailabilityZones: []string{"1", "2", "3"}, + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + AvailabilityZones: []string{"1", "2", "3"}, + }, }, }, wantErr: false, @@ -243,18 +277,22 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Cannot change MaxPods of the agentpool", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - MaxPods: ptr.To(24), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + MaxPods: ptr.To(24), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - MaxPods: ptr.To(25), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + MaxPods: ptr.To(25), + }, }, }, wantErr: true, @@ -263,18 +301,22 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Unchanged MaxPods in an agentpool should not result in an error", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - MaxPods: ptr.To(30), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + MaxPods: ptr.To(30), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - MaxPods: ptr.To(30), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + MaxPods: ptr.To(30), + }, }, }, wantErr: false, @@ -283,20 +325,24 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Cannot change OSDiskType of the agentpool", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - MaxPods: ptr.To(24), - OsDiskType: ptr.To(string(asocontainerservicev1.OSDiskType_Ephemeral)), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + MaxPods: ptr.To(24), + OsDiskType: ptr.To(string(asocontainerservicev1.OSDiskType_Ephemeral)), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - MaxPods: ptr.To(24), - OsDiskType: ptr.To(string(asocontainerservicev1.OSDiskType_Managed)), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + MaxPods: ptr.To(24), + OsDiskType: ptr.To(string(asocontainerservicev1.OSDiskType_Managed)), + }, }, }, wantErr: true, @@ -305,20 +351,24 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Unchanged OSDiskType in an agentpool should not result in an error", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - MaxPods: ptr.To(30), - OsDiskType: ptr.To(string(asocontainerservicev1.OSDiskType_Managed)), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + MaxPods: ptr.To(30), + OsDiskType: ptr.To(string(asocontainerservicev1.OSDiskType_Managed)), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - SKU: "StandardD2S_V3", - OSDiskSizeGB: ptr.To(512), - MaxPods: ptr.To(30), - OsDiskType: ptr.To(string(asocontainerservicev1.OSDiskType_Managed)), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: ptr.To(512), + MaxPods: ptr.To(30), + OsDiskType: ptr.To(string(asocontainerservicev1.OSDiskType_Managed)), + }, }, }, wantErr: false, @@ -327,12 +377,16 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Unexpected error, value EnableUltraSSD is unchanged", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableUltraSSD: ptr.To(true), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableUltraSSD: ptr.To(true), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableUltraSSD: ptr.To(true), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableUltraSSD: ptr.To(true), + }, }, }, wantErr: false, @@ -341,12 +395,16 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "EnableUltraSSD feature is immutable and currently enabled on this agentpool", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableUltraSSD: ptr.To(false), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableUltraSSD: ptr.To(false), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableUltraSSD: ptr.To(true), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableUltraSSD: ptr.To(true), + }, }, }, wantErr: true, @@ -355,12 +413,16 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Unexpected error, value EnableNodePublicIP is unchanged", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableNodePublicIP: ptr.To(true), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableNodePublicIP: ptr.To(true), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableNodePublicIP: ptr.To(true), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableNodePublicIP: ptr.To(true), + }, }, }, wantErr: false, @@ -369,12 +431,16 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "EnableNodePublicIP feature is immutable and currently enabled on this agentpool", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableNodePublicIP: ptr.To(false), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableNodePublicIP: ptr.To(false), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableNodePublicIP: ptr.To(true), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableNodePublicIP: ptr.To(true), + }, }, }, wantErr: true, @@ -383,22 +449,26 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "NodeTaints are mutable", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Taints: []Taint{ - { - Effect: TaintEffect("NoSchedule"), - Key: "foo", - Value: "baz", + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Taints: []Taint{ + { + Effect: TaintEffect("NoSchedule"), + Key: "foo", + Value: "baz", + }, }, }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Taints: []Taint{ - { - Effect: TaintEffect("NoSchedule"), - Key: "foo", - Value: "bar", + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Taints: []Taint{ + { + Effect: TaintEffect("NoSchedule"), + Key: "foo", + Value: "bar", + }, }, }, }, @@ -409,16 +479,20 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Can't add a node label that begins with kubernetes.azure.com", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - NodeLabels: map[string]string{ - "foo": "bar", - "kubernetes.azure.com/scalesetpriority": "spot", + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + NodeLabels: map[string]string{ + "foo": "bar", + "kubernetes.azure.com/scalesetpriority": "spot", + }, }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - NodeLabels: map[string]string{ - "foo": "bar", + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + NodeLabels: map[string]string{ + "foo": "bar", + }, }, }, }, @@ -428,15 +502,19 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Can't update kubeletconfig", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - KubeletConfig: &KubeletConfig{ - CPUCfsQuota: ptr.To(true), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + KubeletConfig: &KubeletConfig{ + CPUCfsQuota: ptr.To(true), + }, }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - KubeletConfig: &KubeletConfig{ - CPUCfsQuota: ptr.To(false), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + KubeletConfig: &KubeletConfig{ + CPUCfsQuota: ptr.To(false), + }, }, }, }, @@ -446,15 +524,19 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Can't update LinuxOSConfig", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - LinuxOSConfig: &LinuxOSConfig{ - SwapFileSizeMB: ptr.To(10), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + LinuxOSConfig: &LinuxOSConfig{ + SwapFileSizeMB: ptr.To(10), + }, }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - LinuxOSConfig: &LinuxOSConfig{ - SwapFileSizeMB: ptr.To(5), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + LinuxOSConfig: &LinuxOSConfig{ + SwapFileSizeMB: ptr.To(5), + }, }, }, }, @@ -464,12 +546,16 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Can't update SubnetName with error", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("my-subnet"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("my-subnet"), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("my-subnet-1"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("my-subnet-1"), + }, }, }, wantErr: true, @@ -478,12 +564,16 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Can update SubnetName if subnetName is empty", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("my-subnet"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("my-subnet"), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: nil, + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: nil, + }, }, }, wantErr: false, @@ -492,12 +582,16 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Can't update SubnetName without error", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("my-subnet"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("my-subnet"), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("my-subnet"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("my-subnet"), + }, }, }, wantErr: false, @@ -506,12 +600,16 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Cannot update enableFIPS", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableFIPS: ptr.To(true), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableFIPS: ptr.To(true), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableFIPS: ptr.To(false), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableFIPS: ptr.To(false), + }, }, }, wantErr: true, @@ -520,12 +618,16 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { name: "Cannot update enableEncryptionAtHost", new: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableEncryptionAtHost: ptr.To(true), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableEncryptionAtHost: ptr.To(true), + }, }, }, old: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableEncryptionAtHost: ptr.To(false), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableEncryptionAtHost: ptr.To(false), + }, }, }, wantErr: true, @@ -569,8 +671,10 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "another valid permutation", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - MaxPods: ptr.To(249), - OsDiskType: ptr.To(string(asocontainerservicev1.OSDiskType_Managed)), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + MaxPods: ptr.To(249), + OsDiskType: ptr.To(string(asocontainerservicev1.OSDiskType_Managed)), + }, }, }, wantErr: false, @@ -586,7 +690,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "too many MaxPods", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - MaxPods: ptr.To(251), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + MaxPods: ptr.To(251), + }, }, }, wantErr: true, @@ -596,7 +702,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "invalid subnetname", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("1+subnet"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("1+subnet"), + }, }, }, wantErr: true, @@ -606,7 +714,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "invalid subnetname", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("1"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("1"), + }, }, }, wantErr: true, @@ -616,7 +726,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "invalid subnetname", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("-a_b-c"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("-a_b-c"), + }, }, }, wantErr: true, @@ -626,7 +738,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "invalid subnetname", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("E-a_b-c"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("E-a_b-c"), + }, }, }, wantErr: true, @@ -636,7 +750,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "invalid subnetname", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("-_-_"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("-_-_"), + }, }, }, wantErr: true, @@ -646,7 +762,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "invalid subnetname", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("abc@#$"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("abc@#$"), + }, }, }, wantErr: true, @@ -656,7 +774,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "invalid subnetname with character length 81", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("3DgIb8EZMkLs0KlyPaTcNxoJU9ufmW6jvXrweqz1hVp5nS4RtH2QY7AFOiC5nS4RtH2QY7AFOiC3DgIb8"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("3DgIb8EZMkLs0KlyPaTcNxoJU9ufmW6jvXrweqz1hVp5nS4RtH2QY7AFOiC5nS4RtH2QY7AFOiC3DgIb8"), + }, }, }, wantErr: true, @@ -666,7 +786,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "valid subnetname with character length 80", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("3DgIb8EZMkLs0KlyPaTcNxoJU9ufmW6jvXrweqz1hVp5nS4RtH2QY7AFOiC5nS4RtH2QY7AFOiC3DgIb"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("3DgIb8EZMkLs0KlyPaTcNxoJU9ufmW6jvXrweqz1hVp5nS4RtH2QY7AFOiC5nS4RtH2QY7AFOiC3DgIb"), + }, }, }, wantErr: false, @@ -675,7 +797,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "valid subnetname", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("1abc"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("1abc"), + }, }, }, wantErr: false, @@ -684,7 +808,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "valid subnetname", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("1-a-b-c"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("1-a-b-c"), + }, }, }, wantErr: false, @@ -693,7 +819,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "valid subnetname", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - SubnetName: ptr.To("my-subnet"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + SubnetName: ptr.To("my-subnet"), + }, }, }, wantErr: false, @@ -702,7 +830,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "too few MaxPods", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - MaxPods: ptr.To(9), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + MaxPods: ptr.To(9), + }, }, }, wantErr: true, @@ -712,8 +842,10 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "ostype Windows with System mode not allowed", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "System", - OSType: ptr.To(WindowsOS), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + OSType: ptr.To(WindowsOS), + }, }, }, wantErr: true, @@ -723,8 +855,10 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "ostype windows with User mode", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "User", - OSType: ptr.To(WindowsOS), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "User", + OSType: ptr.To(WindowsOS), + }, }, }, wantErr: false, @@ -736,8 +870,10 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { Name: "pool0", }, Spec: AzureManagedMachinePoolSpec{ - Mode: "User", - OSType: ptr.To(WindowsOS), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "User", + OSType: ptr.To(WindowsOS), + }, }, }, wantErr: false, @@ -746,9 +882,11 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "Windows clusters with more than 6char names are not allowed", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Name: ptr.To("pool0-name-too-long"), - Mode: "User", - OSType: ptr.To(WindowsOS), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Name: ptr.To("pool0-name-too-long"), + Mode: "User", + OSType: ptr.To(WindowsOS), + }, }, }, wantErr: true, @@ -758,10 +896,12 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "valid label", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "User", - OSType: ptr.To(LinuxOS), - NodeLabels: map[string]string{ - "foo": "bar", + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "User", + OSType: ptr.To(LinuxOS), + NodeLabels: map[string]string{ + "foo": "bar", + }, }, }, }, @@ -771,10 +911,12 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "kubernetes.azure.com label", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - Mode: "User", - OSType: ptr.To(LinuxOS), - NodeLabels: map[string]string{ - "kubernetes.azure.com/scalesetpriority": "spot", + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "User", + OSType: ptr.To(LinuxOS), + NodeLabels: map[string]string{ + "kubernetes.azure.com/scalesetpriority": "spot", + }, }, }, }, @@ -785,8 +927,10 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "pool with invalid public ip prefix", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableNodePublicIP: ptr.To(true), - NodePublicIPPrefixID: ptr.To("not a valid resource ID"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableNodePublicIP: ptr.To(true), + NodePublicIPPrefixID: ptr.To("not a valid resource ID"), + }, }, }, wantErr: true, @@ -796,8 +940,10 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "pool with public ip prefix cannot omit node public IP", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableNodePublicIP: nil, - NodePublicIPPrefixID: ptr.To("subscriptions/11111111-2222-aaaa-bbbb-cccccccccccc/resourceGroups/public-ip-test/providers/Microsoft.Network/publicipprefixes/public-ip-prefix"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableNodePublicIP: nil, + NodePublicIPPrefixID: ptr.To("subscriptions/11111111-2222-aaaa-bbbb-cccccccccccc/resourceGroups/public-ip-test/providers/Microsoft.Network/publicipprefixes/public-ip-prefix"), + }, }, }, wantErr: true, @@ -807,8 +953,10 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "pool with public ip prefix cannot disable node public IP", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableNodePublicIP: ptr.To(false), - NodePublicIPPrefixID: ptr.To("subscriptions/11111111-2222-aaaa-bbbb-cccccccccccc/resourceGroups/public-ip-test/providers/Microsoft.Network/publicipprefixes/public-ip-prefix"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableNodePublicIP: ptr.To(false), + NodePublicIPPrefixID: ptr.To("subscriptions/11111111-2222-aaaa-bbbb-cccccccccccc/resourceGroups/public-ip-test/providers/Microsoft.Network/publicipprefixes/public-ip-prefix"), + }, }, }, wantErr: true, @@ -818,8 +966,10 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "pool with public ip prefix with node public IP enabled ok", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableNodePublicIP: ptr.To(true), - NodePublicIPPrefixID: ptr.To("subscriptions/11111111-2222-aaaa-bbbb-cccccccccccc/resourceGroups/public-ip-test/providers/Microsoft.Network/publicipprefixes/public-ip-prefix"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableNodePublicIP: ptr.To(true), + NodePublicIPPrefixID: ptr.To("subscriptions/11111111-2222-aaaa-bbbb-cccccccccccc/resourceGroups/public-ip-test/providers/Microsoft.Network/publicipprefixes/public-ip-prefix"), + }, }, }, wantErr: false, @@ -828,8 +978,10 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "pool with public ip prefix with leading slash with node public IP enabled ok", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableNodePublicIP: ptr.To(true), - NodePublicIPPrefixID: ptr.To("/subscriptions/11111111-2222-aaaa-bbbb-cccccccccccc/resourceGroups/public-ip-test/providers/Microsoft.Network/publicipprefixes/public-ip-prefix"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableNodePublicIP: ptr.To(true), + NodePublicIPPrefixID: ptr.To("/subscriptions/11111111-2222-aaaa-bbbb-cccccccccccc/resourceGroups/public-ip-test/providers/Microsoft.Network/publicipprefixes/public-ip-prefix"), + }, }, }, wantErr: false, @@ -838,7 +990,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "pool without public ip prefix with node public IP unset ok", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableNodePublicIP: nil, + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableNodePublicIP: nil, + }, }, }, wantErr: false, @@ -847,7 +1001,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "pool without public ip prefix with node public IP enabled ok", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableNodePublicIP: ptr.To(true), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableNodePublicIP: ptr.To(true), + }, }, }, wantErr: false, @@ -856,7 +1012,9 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "pool without public ip prefix with node public IP disabled ok", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - EnableNodePublicIP: ptr.To(false), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + EnableNodePublicIP: ptr.To(false), + }, }, }, wantErr: false, @@ -865,8 +1023,10 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "KubeletConfig CPUCfsQuotaPeriod needs 'ms' suffix", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - KubeletConfig: &KubeletConfig{ - CPUCfsQuotaPeriod: ptr.To("100"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + KubeletConfig: &KubeletConfig{ + CPUCfsQuotaPeriod: ptr.To("100"), + }, }, }, }, @@ -877,8 +1037,10 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "KubeletConfig CPUCfsQuotaPeriod has valid 'ms' suffix", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - KubeletConfig: &KubeletConfig{ - CPUCfsQuotaPeriod: ptr.To("100ms"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + KubeletConfig: &KubeletConfig{ + CPUCfsQuotaPeriod: ptr.To("100ms"), + }, }, }, }, @@ -888,9 +1050,11 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "KubeletConfig ImageGcLowThreshold can't be more than ImageGcHighThreshold", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - KubeletConfig: &KubeletConfig{ - ImageGcLowThreshold: ptr.To(100), - ImageGcHighThreshold: ptr.To(99), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + KubeletConfig: &KubeletConfig{ + ImageGcLowThreshold: ptr.To(100), + ImageGcHighThreshold: ptr.To(99), + }, }, }, }, @@ -901,9 +1065,11 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "KubeletConfig ImageGcLowThreshold is lower than ImageGcHighThreshold", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - KubeletConfig: &KubeletConfig{ - ImageGcLowThreshold: ptr.To(99), - ImageGcHighThreshold: ptr.To(100), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + KubeletConfig: &KubeletConfig{ + ImageGcLowThreshold: ptr.To(99), + ImageGcHighThreshold: ptr.To(100), + }, }, }, }, @@ -913,13 +1079,15 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "valid KubeletConfig AllowedUnsafeSysctls values", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - KubeletConfig: &KubeletConfig{ - AllowedUnsafeSysctls: []string{ - "kernel.shm*", - "kernel.msg*", - "kernel.sem", - "fs.mqueue.*", - "net.*", + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + KubeletConfig: &KubeletConfig{ + AllowedUnsafeSysctls: []string{ + "kernel.shm*", + "kernel.msg*", + "kernel.sem", + "fs.mqueue.*", + "net.*", + }, }, }, }, @@ -930,13 +1098,15 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "more valid KubeletConfig AllowedUnsafeSysctls values", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - KubeletConfig: &KubeletConfig{ - AllowedUnsafeSysctls: []string{ - "kernel.shm.something", - "kernel.msg.foo.bar", - "kernel.sem", - "fs.mqueue.baz", - "net.my.configuration.path", + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + KubeletConfig: &KubeletConfig{ + AllowedUnsafeSysctls: []string{ + "kernel.shm.something", + "kernel.msg.foo.bar", + "kernel.sem", + "fs.mqueue.baz", + "net.my.configuration.path", + }, }, }, }, @@ -947,14 +1117,16 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "an invalid KubeletConfig AllowedUnsafeSysctls value in a set", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - KubeletConfig: &KubeletConfig{ - AllowedUnsafeSysctls: []string{ - "kernel.shm.something", - "kernel.msg.foo.bar", - "kernel.sem", - "fs.mqueue.baz", - "net.my.configuration.path", - "kernel.not.allowed", + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + KubeletConfig: &KubeletConfig{ + AllowedUnsafeSysctls: []string{ + "kernel.shm.something", + "kernel.msg.foo.bar", + "kernel.sem", + "fs.mqueue.baz", + "net.my.configuration.path", + "kernel.not.allowed", + }, }, }, }, @@ -966,9 +1138,11 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "validLinuxOSConfig Sysctls NetIpv4IpLocalPortRange.First is less than NetIpv4IpLocalPortRange.Last", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - LinuxOSConfig: &LinuxOSConfig{ - Sysctls: &SysctlConfig{ - NetIpv4IPLocalPortRange: ptr.To("2000 33000"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + LinuxOSConfig: &LinuxOSConfig{ + Sysctls: &SysctlConfig{ + NetIpv4IPLocalPortRange: ptr.To("2000 33000"), + }, }, }, }, @@ -979,9 +1153,11 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "an invalid LinuxOSConfig Sysctls NetIpv4IpLocalPortRange.First string is ill-formed", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - LinuxOSConfig: &LinuxOSConfig{ - Sysctls: &SysctlConfig{ - NetIpv4IPLocalPortRange: ptr.To("wrong 33000"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + LinuxOSConfig: &LinuxOSConfig{ + Sysctls: &SysctlConfig{ + NetIpv4IPLocalPortRange: ptr.To("wrong 33000"), + }, }, }, }, @@ -993,9 +1169,11 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "an invalid LinuxOSConfig Sysctls NetIpv4IpLocalPortRange.Last string is ill-formed", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - LinuxOSConfig: &LinuxOSConfig{ - Sysctls: &SysctlConfig{ - NetIpv4IPLocalPortRange: ptr.To("2000 wrong"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + LinuxOSConfig: &LinuxOSConfig{ + Sysctls: &SysctlConfig{ + NetIpv4IPLocalPortRange: ptr.To("2000 wrong"), + }, }, }, }, @@ -1007,9 +1185,11 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "an invalid LinuxOSConfig Sysctls NetIpv4IpLocalPortRange.First less than allowed value", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - LinuxOSConfig: &LinuxOSConfig{ - Sysctls: &SysctlConfig{ - NetIpv4IPLocalPortRange: ptr.To("1020 32999"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + LinuxOSConfig: &LinuxOSConfig{ + Sysctls: &SysctlConfig{ + NetIpv4IPLocalPortRange: ptr.To("1020 32999"), + }, }, }, }, @@ -1021,9 +1201,11 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "an invalid LinuxOSConfig Sysctls NetIpv4IpLocalPortRange.Last less than allowed value", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - LinuxOSConfig: &LinuxOSConfig{ - Sysctls: &SysctlConfig{ - NetIpv4IPLocalPortRange: ptr.To("1024 32000"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + LinuxOSConfig: &LinuxOSConfig{ + Sysctls: &SysctlConfig{ + NetIpv4IPLocalPortRange: ptr.To("1024 32000"), + }, }, }, }, @@ -1035,9 +1217,11 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "an invalid LinuxOSConfig Sysctls NetIpv4IpLocalPortRange.First is greater than NetIpv4IpLocalPortRange.Last", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - LinuxOSConfig: &LinuxOSConfig{ - Sysctls: &SysctlConfig{ - NetIpv4IPLocalPortRange: ptr.To("33000 32999"), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + LinuxOSConfig: &LinuxOSConfig{ + Sysctls: &SysctlConfig{ + NetIpv4IPLocalPortRange: ptr.To("33000 32999"), + }, }, }, }, @@ -1049,11 +1233,13 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "valid LinuxOSConfig Sysctls is set by disabling FailSwapOn", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - KubeletConfig: &KubeletConfig{ - FailSwapOn: ptr.To(false), - }, - LinuxOSConfig: &LinuxOSConfig{ - SwapFileSizeMB: ptr.To(1500), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + KubeletConfig: &KubeletConfig{ + FailSwapOn: ptr.To(false), + }, + LinuxOSConfig: &LinuxOSConfig{ + SwapFileSizeMB: ptr.To(1500), + }, }, }, }, @@ -1063,11 +1249,13 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "an invalid LinuxOSConfig Sysctls is set with FailSwapOn set to true", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - KubeletConfig: &KubeletConfig{ - FailSwapOn: ptr.To(true), - }, - LinuxOSConfig: &LinuxOSConfig{ - SwapFileSizeMB: ptr.To(1500), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + KubeletConfig: &KubeletConfig{ + FailSwapOn: ptr.To(true), + }, + LinuxOSConfig: &LinuxOSConfig{ + SwapFileSizeMB: ptr.To(1500), + }, }, }, }, @@ -1078,8 +1266,10 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { name: "an invalid LinuxOSConfig Sysctls is set without disabling FailSwapOn", ammp: &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - LinuxOSConfig: &LinuxOSConfig{ - SwapFileSizeMB: ptr.To(1500), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + LinuxOSConfig: &LinuxOSConfig{ + SwapFileSizeMB: ptr.To(1500), + }, }, }, }, @@ -1209,7 +1399,7 @@ func TestAzureManagedMachinePool_validateLastSystemNodePool(t *testing.T) { _ = AddToScheme(scheme) _ = clusterv1.AddToScheme(scheme) fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(tc.cluster, tc.ammp).Build() - err := tc.ammp.validateLastSystemNodePool(fakeClient) + err := validateLastSystemNodePool(fakeClient, tc.ammp.Spec.NodeLabels, tc.ammp.Namespace) if tc.wantErr { g.Expect(err).To(HaveOccurred()) } else { @@ -1222,8 +1412,10 @@ func TestAzureManagedMachinePool_validateLastSystemNodePool(t *testing.T) { func getKnownValidAzureManagedMachinePool() *AzureManagedMachinePool { return &AzureManagedMachinePool{ Spec: AzureManagedMachinePoolSpec{ - MaxPods: ptr.To(30), - OsDiskType: ptr.To(string(asocontainerservicev1.OSDiskType_Ephemeral)), + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + MaxPods: ptr.To(30), + OsDiskType: ptr.To(string(asocontainerservicev1.OSDiskType_Ephemeral)), + }, }, } } @@ -1237,5 +1429,12 @@ func getManagedMachinePoolWithSystemMode() *AzureManagedMachinePool { LabelAgentPoolMode: string(NodePoolModeSystem), }, }, + Spec: AzureManagedMachinePoolSpec{ + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + NodeLabels: map[string]string{ + clusterv1.ClusterNameLabel: "test-cluster", + }, + }, + }, } } diff --git a/api/v1beta1/azuremanagedmachinepooltemplate_types.go b/api/v1beta1/azuremanagedmachinepooltemplate_types.go new file mode 100644 index 00000000000..7cf3f3e66b0 --- /dev/null +++ b/api/v1beta1/azuremanagedmachinepooltemplate_types.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// AzureManagedMachinePoolTemplateSpec defines the desired state of AzureManagedMachinePoolTemplate. +type AzureManagedMachinePoolTemplateSpec struct { + Template AzureManagedMachinePoolTemplateResource `json:"template"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=azuremanagedmachinepooltemplates,scope=Namespaced,categories=cluster-api,shortName=ammpt +// +kubebuilder:storageversion + +// AzureManagedMachinePoolTemplate is the Schema for the AzureManagedMachinePoolTemplates API. +type AzureManagedMachinePoolTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AzureManagedMachinePoolTemplateSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// AzureManagedMachinePoolTemplateList contains a list of AzureManagedMachinePoolTemplates. +type AzureManagedMachinePoolTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AzureManagedMachinePoolTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&AzureManagedMachinePoolTemplate{}, &AzureManagedMachinePoolTemplateList{}) +} + +// AzureManagedMachinePoolTemplateResource describes the data needed to create an AzureManagedCluster from a template. +type AzureManagedMachinePoolTemplateResource struct { + Spec AzureManagedMachinePoolTemplateResourceSpec `json:"spec"` +} diff --git a/api/v1beta1/azuremanagedmachinepooltemplate_webhook.go b/api/v1beta1/azuremanagedmachinepooltemplate_webhook.go new file mode 100644 index 00000000000..7789511f33d --- /dev/null +++ b/api/v1beta1/azuremanagedmachinepooltemplate_webhook.go @@ -0,0 +1,287 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api-provider-azure/feature" + webhookutils "sigs.k8s.io/cluster-api-provider-azure/util/webhook" + capifeature "sigs.k8s.io/cluster-api/feature" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// SetupAzureManagedMachinePoolTemplateWebhookWithManager will set up the webhook to be managed by the specified manager. +func SetupAzureManagedMachinePoolTemplateWebhookWithManager(mgr ctrl.Manager) error { + mpw := &azureManagedMachinePoolTemplateWebhook{Client: mgr.GetClient()} + return ctrl.NewWebhookManagedBy(mgr). + For(&AzureManagedMachinePoolTemplate{}). + WithDefaulter(mpw). + WithValidator(mpw). + Complete() +} + +//+kubebuilder:webhook:path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedmachinepooltemplate,mutating=true,failurePolicy=fail,groups=infrastructure.cluster.x-k8s.io,resources=azuremanagedmachinepooltemplates,verbs=create;update,versions=v1beta1,name=default.azuremanagedmachinepooltemplates.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 + +type azureManagedMachinePoolTemplateWebhook struct { + Client client.Client +} + +// Default implements webhook.Defaulter so a webhook will be registered for the type. +func (mpw *azureManagedMachinePoolTemplateWebhook) Default(ctx context.Context, obj runtime.Object) error { + mp, ok := obj.(*AzureManagedMachinePoolTemplate) + if !ok { + return apierrors.NewBadRequest("expected an AzureManagedMachinePoolTemplate") + } + if mp.Labels == nil { + mp.Labels = make(map[string]string) + } + mp.Labels[LabelAgentPoolMode] = mp.Spec.Template.Spec.Mode + + if mp.Spec.Template.Spec.Name == nil || *mp.Spec.Template.Spec.Name == "" { + mp.Spec.Template.Spec.Name = &mp.Name + } + + setDefault[*string](&mp.Spec.Template.Spec.OSType, ptr.To(DefaultOSType)) + + return nil +} + +//+kubebuilder:webhook:verbs=create;update;delete,path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedmachinepooltemplate,mutating=false,failurePolicy=fail,groups=infrastructure.cluster.x-k8s.io,resources=azuremanagedmachinepooltemplates,versions=v1beta1,name=validation.azuremanagedmachinepooltemplates.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (mpw *azureManagedMachinePoolTemplateWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + mp, ok := obj.(*AzureManagedMachinePoolTemplate) + if !ok { + return nil, apierrors.NewBadRequest("expected an AzureManagedMachinePoolTemplate") + } + + if !feature.Gates.Enabled(capifeature.MachinePool) { + return nil, field.Forbidden( + field.NewPath("spec"), + "can be set only if the Cluster API 'MachinePool' feature flag is enabled", + ) + } + + var errs []error + + errs = append(errs, validateMaxPods( + mp.Spec.Template.Spec.MaxPods, + field.NewPath("Spec", "Template", "Spec", "MaxPods"))) + + errs = append(errs, validateOSType( + mp.Spec.Template.Spec.Mode, + mp.Spec.Template.Spec.OSType, + field.NewPath("Spec", "Template", "Spec", "OSType"))) + + errs = append(errs, validateMPName( + mp.Name, + mp.Spec.Template.Spec.Name, + mp.Spec.Template.Spec.OSType, + field.NewPath("Spec", "Template", "Spec", "Name"))) + + errs = append(errs, validateNodeLabels( + mp.Spec.Template.Spec.NodeLabels, + field.NewPath("Spec", "Template", "Spec", "NodeLabels"))) + + errs = append(errs, validateNodePublicIPPrefixID( + mp.Spec.Template.Spec.NodePublicIPPrefixID, + field.NewPath("Spec", "Template", "Spec", "NodePublicIPPrefixID"))) + + errs = append(errs, validateEnableNodePublicIP( + mp.Spec.Template.Spec.EnableNodePublicIP, + mp.Spec.Template.Spec.NodePublicIPPrefixID, + field.NewPath("Spec", "Template", "Spec", "EnableNodePublicIP"))) + + errs = append(errs, validateKubeletConfig( + mp.Spec.Template.Spec.KubeletConfig, + field.NewPath("Spec", "Template", "Spec", "KubeletConfig"))) + + errs = append(errs, validateLinuxOSConfig( + mp.Spec.Template.Spec.LinuxOSConfig, + mp.Spec.Template.Spec.KubeletConfig, + field.NewPath("Spec", "Template", "Spec", "LinuxOSConfig"))) + + return nil, kerrors.NewAggregate(errs) +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (mpw *azureManagedMachinePoolTemplateWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + var allErrs field.ErrorList + old, ok := oldObj.(*AzureManagedMachinePoolTemplate) + if !ok { + return nil, apierrors.NewBadRequest("expected an AzureManagedMachinePoolTemplate") + } + mp, ok := newObj.(*AzureManagedMachinePoolTemplate) + if !ok { + return nil, apierrors.NewBadRequest("expected an AzureManagedMachinePoolTemplate") + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "Name"), + old.Spec.Template.Spec.Name, + mp.Spec.Template.Spec.Name); err != nil { + allErrs = append(allErrs, err) + } + + if err := validateNodeLabels(mp.Spec.Template.Spec.NodeLabels, field.NewPath("Spec", "Template", "Spec", "NodeLabels")); err != nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Template", "Spec", "NodeLabels"), + mp.Spec.Template.Spec.NodeLabels, + err.Error())) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "OSType"), + old.Spec.Template.Spec.OSType, + mp.Spec.Template.Spec.OSType); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "SKU"), + old.Spec.Template.Spec.SKU, + mp.Spec.Template.Spec.SKU); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "OSDiskSizeGB"), + old.Spec.Template.Spec.OSDiskSizeGB, + mp.Spec.Template.Spec.OSDiskSizeGB); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "SubnetName"), + old.Spec.Template.Spec.SubnetName, + mp.Spec.Template.Spec.SubnetName); err != nil && old.Spec.Template.Spec.SubnetName != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "EnableFIPS"), + old.Spec.Template.Spec.EnableFIPS, + mp.Spec.Template.Spec.EnableFIPS); err != nil && old.Spec.Template.Spec.EnableFIPS != nil { + allErrs = append(allErrs, err) + } + + if !webhookutils.EnsureStringSlicesAreEquivalent(mp.Spec.Template.Spec.AvailabilityZones, old.Spec.Template.Spec.AvailabilityZones) { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "Template", "Spec", "AvailabilityZones"), + mp.Spec.Template.Spec.AvailabilityZones, + "field is immutable")) + } + + if mp.Spec.Template.Spec.Mode != string(NodePoolModeSystem) && old.Spec.Template.Spec.Mode == string(NodePoolModeSystem) { + // validate for last system node pool + if err := validateLastSystemNodePool(mpw.Client, mp.Spec.Template.Spec.NodeLabels, mp.Namespace); err != nil { + allErrs = append(allErrs, field.Forbidden( + field.NewPath("Spec", "Template", "Spec", "Mode"), + "Cannot change node pool mode to User, you must have at least one System node pool in your cluster")) + } + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "MaxPods"), + old.Spec.Template.Spec.MaxPods, + mp.Spec.Template.Spec.MaxPods); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "OsDiskType"), + old.Spec.Template.Spec.OsDiskType, + mp.Spec.Template.Spec.OsDiskType); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "ScaleSetPriority"), + old.Spec.Template.Spec.ScaleSetPriority, + mp.Spec.Template.Spec.ScaleSetPriority); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "EnableUltraSSD"), + old.Spec.Template.Spec.EnableUltraSSD, + mp.Spec.Template.Spec.EnableUltraSSD); err != nil { + allErrs = append(allErrs, err) + } + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "EnableNodePublicIP"), + old.Spec.Template.Spec.EnableNodePublicIP, + mp.Spec.Template.Spec.EnableNodePublicIP); err != nil { + allErrs = append(allErrs, err) + } + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "NodePublicIPPrefixID"), + old.Spec.Template.Spec.NodePublicIPPrefixID, + mp.Spec.Template.Spec.NodePublicIPPrefixID); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "KubeletConfig"), + old.Spec.Template.Spec.KubeletConfig, + mp.Spec.Template.Spec.KubeletConfig); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "KubeletDiskType"), + old.Spec.Template.Spec.KubeletDiskType, + mp.Spec.Template.Spec.KubeletDiskType); err != nil { + allErrs = append(allErrs, err) + } + + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Template", "Spec", "LinuxOSConfig"), + old.Spec.Template.Spec.LinuxOSConfig, + mp.Spec.Template.Spec.LinuxOSConfig); err != nil { + allErrs = append(allErrs, err) + } + + if len(allErrs) == 0 { + return nil, nil + } + return nil, apierrors.NewInvalid(GroupVersion.WithKind("AzureManagedMachinePoolTemplate").GroupKind(), mp.Name, allErrs) +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (mpw *azureManagedMachinePoolTemplateWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + mp, ok := obj.(*AzureManagedMachinePoolTemplate) + if !ok { + return nil, apierrors.NewBadRequest("expected an AzureManagedMachinePoolTemplate") + } + if mp.Spec.Template.Spec.Mode != string(NodePoolModeSystem) { + return nil, nil + } + + return nil, errors.Wrapf(validateLastSystemNodePool(mpw.Client, mp.Spec.Template.Spec.NodeLabels, mp.Namespace), "if the delete is triggered via owner MachinePool please refer to trouble shooting section in https://capz.sigs.k8s.io/topics/managedcluster.html") +} diff --git a/api/v1beta1/azuremanagedmachinepooltemplate_webhook_test.go b/api/v1beta1/azuremanagedmachinepooltemplate_webhook_test.go new file mode 100644 index 00000000000..5f2c1bdf714 --- /dev/null +++ b/api/v1beta1/azuremanagedmachinepooltemplate_webhook_test.go @@ -0,0 +1,269 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" +) + +func TestManagedMachinePoolTemplateDefaultingWebhook(t *testing.T) { + g := NewWithT(t) + + t.Logf("Testing ammpt defaulting webhook with no baseline") + ammpt := getAzureManagedMachinePoolTemplate() + mmptw := &azureManagedMachinePoolTemplateWebhook{} + err := mmptw.Default(context.Background(), ammpt) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(ammpt.Labels).To(Equal(map[string]string{ + LabelAgentPoolMode: "System", + })) + g.Expect(ammpt.Spec.Template.Spec.Name).To(Equal(ptr.To("fooName"))) + g.Expect(ammpt.Spec.Template.Spec.OSType).To(Equal(ptr.To("Linux"))) + + t.Logf("Testing ammpt defaulting webhook with baseline") + ammpt = getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.Mode = "User" + ammpt.Spec.Template.Spec.Name = ptr.To("barName") + ammpt.Spec.Template.Spec.OSType = ptr.To("Windows") + }) + err = mmptw.Default(context.Background(), ammpt) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(ammpt.Labels).To(Equal(map[string]string{ + LabelAgentPoolMode: "User", + })) + g.Expect(ammpt.Spec.Template.Spec.Name).To(Equal(ptr.To("barName"))) + g.Expect(ammpt.Spec.Template.Spec.OSType).To(Equal(ptr.To("Windows"))) +} + +func TestManagedMachinePoolTemplateUpdateWebhook(t *testing.T) { + tests := []struct { + name string + oldMachinePoolTemplate *AzureManagedMachinePoolTemplate + machinePoolTemplate *AzureManagedMachinePoolTemplate + wantErr bool + }{ + { + name: "azuremanagedmachinepooltemplate no changes - valid spec", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(), + wantErr: false, + }, + { + name: "azuremanagedmachinepooltemplate name is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.Name = ptr.To("barName") + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate invalid nodeLabel", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.NodeLabels = map[string]string{ + azureutil.AzureSystemNodeLabelPrefix: "foo", + } + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate osType is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.OSType = ptr.To("Windows") + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.OSType = ptr.To("Linux") + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate SKU is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.SKU = "Standard_D2s_v3" + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.SKU = "Standard_D4s_v3" + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate OSDiskSizeGB is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.OSDiskSizeGB = ptr.To(128) + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.OSDiskSizeGB = ptr.To(256) + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate SubnetName is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.SubnetName = ptr.To("fooSubnet") + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.SubnetName = ptr.To("barSubnet") + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate enableFIPS is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.EnableFIPS = ptr.To(true) + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.EnableFIPS = ptr.To(false) + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate MaxPods is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.MaxPods = ptr.To(128) + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.MaxPods = ptr.To(256) + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate OSDiskType is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.OsDiskType = ptr.To("Standard_LRS") + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.OsDiskType = ptr.To("Premium_LRS") + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate scaleSetPriority is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.ScaleSetPriority = ptr.To("Regular") + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.ScaleSetPriority = ptr.To("Spot") + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate enableUltraSSD is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.EnableUltraSSD = ptr.To(true) + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.EnableUltraSSD = ptr.To(false) + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate enableNodePublicIP is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.EnableNodePublicIP = ptr.To(true) + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.EnableNodePublicIP = ptr.To(false) + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate nodePublicIPPrefixID is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.NodePublicIPPrefixID = ptr.To("fooPublicIPPrefixID") + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.NodePublicIPPrefixID = ptr.To("barPublicIPPrefixID") + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate kubeletConfig is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.KubeletConfig = &KubeletConfig{} + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.KubeletConfig = &KubeletConfig{ + FailSwapOn: ptr.To(true), + } + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate kubeletDiskType is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.KubeletDiskType = ptr.To(KubeletDiskTypeOS) + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.KubeletDiskType = ptr.To(KubeletDiskTypeTemporary) + }), + wantErr: true, + }, + { + name: "azuremanagedmachinepooltemplate linuxOSConfig is immutable", + oldMachinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.LinuxOSConfig = &LinuxOSConfig{} + }), + machinePoolTemplate: getAzureManagedMachinePoolTemplate(func(ammpt *AzureManagedMachinePoolTemplate) { + ammpt.Spec.Template.Spec.LinuxOSConfig = &LinuxOSConfig{ + SwapFileSizeMB: ptr.To(128), + } + }), + wantErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + mpw := &azureManagedMachinePoolTemplateWebhook{} + _, err := mpw.ValidateUpdate(context.Background(), tc.oldMachinePoolTemplate, tc.machinePoolTemplate) + if tc.wantErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} + +func getAzureManagedMachinePoolTemplate(changes ...func(*AzureManagedMachinePoolTemplate)) *AzureManagedMachinePoolTemplate { + input := &AzureManagedMachinePoolTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fooName", + }, + Spec: AzureManagedMachinePoolTemplateSpec{ + Template: AzureManagedMachinePoolTemplateResource{ + Spec: AzureManagedMachinePoolTemplateResourceSpec{ + AzureManagedMachinePoolClassSpec: AzureManagedMachinePoolClassSpec{ + Mode: "System", + }, + }, + }, + }, + } + + for _, change := range changes { + change(input) + } + + return input +} diff --git a/api/v1beta1/consts.go b/api/v1beta1/consts.go index fa1920b0774..07aeb89f5a3 100644 --- a/api/v1beta1/consts.go +++ b/api/v1beta1/consts.go @@ -170,4 +170,6 @@ const ( AzureManagedControlPlaneKind = "AzureManagedControlPlane" // AzureClusterIdentityKind indicates the kind of an AzureClusterIdentity. AzureClusterIdentityKind = "AzureClusterIdentity" + // AzureNetworkPluginName is the name of the Azure network plugin. + AzureNetworkPluginName = "azure" ) diff --git a/api/v1beta1/types_class.go b/api/v1beta1/types_class.go index cfec0c603e1..3a7b6be1cde 100644 --- a/api/v1beta1/types_class.go +++ b/api/v1beta1/types_class.go @@ -18,6 +18,7 @@ package v1beta1 import ( corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) @@ -66,6 +67,302 @@ type AzureClusterClassSpec struct { FailureDomains clusterv1.FailureDomains `json:"failureDomains,omitempty"` } +// AzureManagedControlPlaneClassSpec defines the AzureManagedControlPlane properties that may be shared across several azure managed control planes. +type AzureManagedControlPlaneClassSpec struct { + // MachineTemplate contains information about how machines + // should be shaped when creating or updating a control plane. + // For the AzureManagedControlPlaneTemplate, this field is used + // only to fulfill the CAPI contract. + // +optional + MachineTemplate *AzureManagedControlPlaneTemplateMachineTemplate `json:"machineTemplate,omitempty"` + + // Version defines the desired Kubernetes version. + // +kubebuilder:validation:MinLength:=2 + Version string `json:"version"` + + // VirtualNetwork describes the virtual network for the AKS cluster. It will be created if it does not already exist. + // +optional + VirtualNetwork ManagedControlPlaneVirtualNetwork `json:"virtualNetwork,omitempty"` + + // SubscriptionID is the GUID of the Azure subscription that owns this cluster. + // +optional + SubscriptionID string `json:"subscriptionID,omitempty"` + + // Location is a string matching one of the canonical Azure region names. Examples: "westus2", "eastus". + Location string `json:"location"` + + // AdditionalTags is an optional set of tags to add to Azure resources managed by the Azure provider, in addition to the + // ones added by default. + // +optional + AdditionalTags Tags `json:"additionalTags,omitempty"` + + // NetworkPlugin used for building Kubernetes network. + // +kubebuilder:validation:Enum=azure;kubenet + // +optional + NetworkPlugin *string `json:"networkPlugin,omitempty"` + + // NetworkPluginMode is the mode the network plugin should use. + // Allowed value is "overlay". + // +kubebuilder:validation:Enum=overlay + // +optional + NetworkPluginMode *NetworkPluginMode `json:"networkPluginMode,omitempty"` + + // NetworkPolicy used for building Kubernetes network. + // +kubebuilder:validation:Enum=azure;calico + // +optional + NetworkPolicy *string `json:"networkPolicy,omitempty"` + + // Outbound configuration used by Nodes. + // +kubebuilder:validation:Enum=loadBalancer;managedNATGateway;userAssignedNATGateway;userDefinedRouting + // +optional + OutboundType *ManagedControlPlaneOutboundType `json:"outboundType,omitempty"` + + // DNSServiceIP is an IP address assigned to the Kubernetes DNS service. + // It must be within the Kubernetes service address range specified in serviceCidr. + // Immutable. + // +optional + DNSServiceIP *string `json:"dnsServiceIP,omitempty"` + + // LoadBalancerSKU is the SKU of the loadBalancer to be provisioned. + // Immutable. + // +kubebuilder:validation:Enum=Basic;Standard + // +kubebuilder:default:=Standard + // +optional + LoadBalancerSKU *string `json:"loadBalancerSKU,omitempty"` + + // IdentityRef is a reference to a AzureClusterIdentity to be used when reconciling this cluster + IdentityRef *corev1.ObjectReference `json:"identityRef"` + + // AadProfile is Azure Active Directory configuration to integrate with AKS for aad authentication. + // +optional + AADProfile *AADProfile `json:"aadProfile,omitempty"` + + // AddonProfiles are the profiles of managed cluster add-on. + // +optional + AddonProfiles []AddonProfile `json:"addonProfiles,omitempty"` + + // SKU is the SKU of the AKS to be provisioned. + // +optional + SKU *AKSSku `json:"sku,omitempty"` + + // LoadBalancerProfile is the profile of the cluster load balancer. + // +optional + LoadBalancerProfile *LoadBalancerProfile `json:"loadBalancerProfile,omitempty"` + + // APIServerAccessProfile is the access profile for AKS API server. + // Immutable except for `authorizedIPRanges`. + // +optional + APIServerAccessProfile *APIServerAccessProfile `json:"apiServerAccessProfile,omitempty"` + + // AutoscalerProfile is the parameters to be applied to the cluster-autoscaler when enabled + // +optional + AutoScalerProfile *AutoScalerProfile `json:"autoscalerProfile,omitempty"` + + // AzureEnvironment is the name of the AzureCloud to be used. + // The default value that would be used by most users is "AzurePublicCloud", other values are: + // - ChinaCloud: "AzureChinaCloud" + // - PublicCloud: "AzurePublicCloud" + // - USGovernmentCloud: "AzureUSGovernmentCloud" + // +optional + AzureEnvironment string `json:"azureEnvironment,omitempty"` + + // Identity configuration used by the AKS control plane. + // +optional + Identity *Identity `json:"identity,omitempty"` + + // KubeletUserAssignedIdentity is the user-assigned identity for kubelet. + // For authentication with Azure Container Registry. + // +optional + KubeletUserAssignedIdentity string `json:"kubeletUserAssignedIdentity,omitempty"` + + // HTTPProxyConfig is the HTTP proxy configuration for the cluster. + // Immutable. + // +optional + HTTPProxyConfig *HTTPProxyConfig `json:"httpProxyConfig,omitempty"` + + // OIDCIssuerProfile is the OIDC issuer profile of the Managed Cluster. + // +optional + OIDCIssuerProfile *OIDCIssuerProfile `json:"oidcIssuerProfile,omitempty"` + + // DisableLocalAccounts disables getting static credentials for this cluster when set. Expected to only be used for AAD clusters. + // +optional + DisableLocalAccounts *bool `json:"disableLocalAccounts,omitempty"` +} + +// AzureManagedMachinePoolClassSpec defines the AzureManagedMachinePool properties that may be shared across several Azure managed machinepools. +type AzureManagedMachinePoolClassSpec struct { + // AdditionalTags is an optional set of tags to add to Azure resources managed by the + // Azure provider, in addition to the ones added by default. + // +optional + AdditionalTags Tags `json:"additionalTags,omitempty"` + + // Name is the name of the agent pool. If not specified, CAPZ uses the name of the CR as the agent pool name. + // Immutable. + // +optional + Name *string `json:"name,omitempty"` + + // Mode represents the mode of an agent pool. Possible values include: System, User. + // +kubebuilder:validation:Enum=System;User + Mode string `json:"mode"` + + // SKU is the size of the VMs in the node pool. + // Immutable. + SKU string `json:"sku"` + + // OSDiskSizeGB is the disk size for every machine in this agent pool. + // If you specify 0, it will apply the default osDisk size according to the vmSize specified. + // Immutable. + // +optional + OSDiskSizeGB *int `json:"osDiskSizeGB,omitempty"` + + // AvailabilityZones - Availability zones for nodes. Must use VirtualMachineScaleSets AgentPoolType. + // Immutable. + // +optional + AvailabilityZones []string `json:"availabilityZones,omitempty"` + + // Node labels represent the labels for all of the nodes present in node pool. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/azure/aks/use-labels + // +optional + NodeLabels map[string]string `json:"nodeLabels,omitempty"` + + // Taints specifies the taints for nodes present in this agent pool. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/azure/aks/use-multiple-node-pools#setting-node-pool-taints + // +optional + Taints Taints `json:"taints,omitempty"` + + // Scaling specifies the autoscaling parameters for the node pool. + // +optional + Scaling *ManagedMachinePoolScaling `json:"scaling,omitempty"` + + // MaxPods specifies the kubelet `--max-pods` configuration for the node pool. + // Immutable. + // See also [AKS doc], [K8s doc]. + // + // [AKS doc]: https://learn.microsoft.com/azure/aks/configure-azure-cni#configure-maximum---new-clusters + // [K8s doc]: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/ + // +optional + MaxPods *int `json:"maxPods,omitempty"` + + // OsDiskType specifies the OS disk type for each node in the pool. Allowed values are 'Ephemeral' and 'Managed' (default). + // Immutable. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/azure/aks/cluster-configuration#ephemeral-os + // +kubebuilder:validation:Enum=Ephemeral;Managed + // +kubebuilder:default=Managed + // +optional + OsDiskType *string `json:"osDiskType,omitempty"` + + // EnableUltraSSD enables the storage type UltraSSD_LRS for the agent pool. + // Immutable. + // +optional + EnableUltraSSD *bool `json:"enableUltraSSD,omitempty"` + + // OSType specifies the virtual machine operating system. Default to Linux. Possible values include: 'Linux', 'Windows'. + // 'Windows' requires the AzureManagedControlPlane's `spec.networkPlugin` to be `azure`. + // Immutable. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/rest/api/aks/agent-pools/create-or-update?tabs=HTTP#ostype + // +kubebuilder:validation:Enum=Linux;Windows + // +optional + OSType *string `json:"osType,omitempty"` + + // EnableNodePublicIP controls whether or not nodes in the pool each have a public IP address. + // Immutable. + // +optional + EnableNodePublicIP *bool `json:"enableNodePublicIP,omitempty"` + + // NodePublicIPPrefixID specifies the public IP prefix resource ID which VM nodes should use IPs from. + // Immutable. + // +optional + NodePublicIPPrefixID *string `json:"nodePublicIPPrefixID,omitempty"` + + // ScaleSetPriority specifies the ScaleSetPriority value. Default to Regular. Possible values include: 'Regular', 'Spot' + // Immutable. + // +kubebuilder:validation:Enum=Regular;Spot + // +optional + ScaleSetPriority *string `json:"scaleSetPriority,omitempty"` + + // ScaleDownMode affects the cluster autoscaler behavior. Default to Delete. Possible values include: 'Deallocate', 'Delete' + // +kubebuilder:validation:Enum=Deallocate;Delete + // +kubebuilder:default=Delete + // +optional + ScaleDownMode *string `json:"scaleDownMode,omitempty"` + + // SpotMaxPrice defines max price to pay for spot instance. Possible values are any decimal value greater than zero or -1. + // If you set the max price to be -1, the VM won't be evicted based on price. The price for the VM will be the current price + // for spot or the price for a standard VM, which ever is less, as long as there's capacity and quota available. + // +optional + SpotMaxPrice *resource.Quantity `json:"spotMaxPrice,omitempty"` + + // KubeletConfig specifies the kubelet configurations for nodes. + // Immutable. + // +optional + KubeletConfig *KubeletConfig `json:"kubeletConfig,omitempty"` + + // KubeletDiskType specifies the kubelet disk type. Default to OS. Possible values include: 'OS', 'Temporary'. + // Requires Microsoft.ContainerService/KubeletDisk preview feature to be set. + // Immutable. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/rest/api/aks/agent-pools/create-or-update?tabs=HTTP#kubeletdisktype + // +kubebuilder:validation:Enum=OS;Temporary + // +optional + KubeletDiskType *KubeletDiskType `json:"kubeletDiskType,omitempty"` + + // LinuxOSConfig specifies the custom Linux OS settings and configurations. + // Immutable. + // +optional + LinuxOSConfig *LinuxOSConfig `json:"linuxOSConfig,omitempty"` + + // SubnetName specifies the Subnet where the MachinePool will be placed + // Immutable. + // +optional + SubnetName *string `json:"subnetName,omitempty"` + + // EnableFIPS indicates whether FIPS is enabled on the node pool. + // Immutable. + // +optional + EnableFIPS *bool `json:"enableFIPS,omitempty"` + + // EnableEncryptionAtHost indicates whether host encryption is enabled on the node pool. + // Immutable. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/en-us/azure/aks/enable-host-encryption + // +optional + EnableEncryptionAtHost *bool `json:"enableEncryptionAtHost,omitempty"` +} + +// ManagedControlPlaneVirtualNetworkClassSpec defines the ManagedControlPlaneVirtualNetwork properties that may be shared across several managed control plane vnets. +type ManagedControlPlaneVirtualNetworkClassSpec struct { + Name string `json:"name"` + CIDRBlock string `json:"cidrBlock"` + // +optional + Subnet ManagedControlPlaneSubnet `json:"subnet,omitempty"` +} + +// APIServerAccessProfileClassSpec defines the APIServerAccessProfile properties that may be shared across several API server access profiles. +type APIServerAccessProfileClassSpec struct { + // EnablePrivateCluster indicates whether to create the cluster as a private cluster or not. + // +optional + EnablePrivateCluster *bool `json:"enablePrivateCluster,omitempty"` + + // PrivateDNSZone enables private dns zone mode for private cluster. + // +kubebuilder:validation:Enum=System;None + // +optional + PrivateDNSZone *string `json:"privateDNSZone,omitempty"` + + // EnablePrivateClusterPublicFQDN indicates whether to create additional public FQDN for private cluster or not. + // +optional + EnablePrivateClusterPublicFQDN *bool `json:"enablePrivateClusterPublicFQDN,omitempty"` +} + // ExtendedLocationSpec defines the ExtendedLocation properties to enable CAPZ for Azure public MEC. type ExtendedLocationSpec struct { // Name defines the name for the extended location. diff --git a/api/v1beta1/types_template.go b/api/v1beta1/types_template.go index baed624f26d..fb4f30f1a41 100644 --- a/api/v1beta1/types_template.go +++ b/api/v1beta1/types_template.go @@ -21,6 +21,23 @@ import ( "k8s.io/utils/net" ) +// AzureManagedControlPlaneTemplateResourceSpec specifies an Azure managed control plane template resource. +type AzureManagedControlPlaneTemplateResourceSpec struct { + AzureManagedControlPlaneClassSpec `json:",inline"` +} + +// AzureManagedControlPlaneTemplateMachineTemplate is only used to fulfill the CAPI contract which expects a +// MachineTemplate field for any controlplane ref in a topology. +type AzureManagedControlPlaneTemplateMachineTemplate struct{} + +// AzureManagedMachinePoolTemplateResourceSpec specifies an Azure managed control plane template resource. +type AzureManagedMachinePoolTemplateResourceSpec struct { + AzureManagedMachinePoolClassSpec `json:",inline"` +} + +// AzureManagedClusterTemplateResourceSpec specifies an Azure managed cluster template resource. +type AzureManagedClusterTemplateResourceSpec struct{} + // AzureClusterTemplateResourceSpec specifies an Azure cluster template resource. type AzureClusterTemplateResourceSpec struct { AzureClusterClassSpec `json:",inline"` diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 96873192539..fa8744f1404 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -72,6 +72,22 @@ func (in *APIServerAccessProfile) DeepCopyInto(out *APIServerAccessProfile) { *out = make([]string, len(*in)) copy(*out, *in) } + in.APIServerAccessProfileClassSpec.DeepCopyInto(&out.APIServerAccessProfileClassSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIServerAccessProfile. +func (in *APIServerAccessProfile) DeepCopy() *APIServerAccessProfile { + if in == nil { + return nil + } + out := new(APIServerAccessProfile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIServerAccessProfileClassSpec) DeepCopyInto(out *APIServerAccessProfileClassSpec) { + *out = *in if in.EnablePrivateCluster != nil { in, out := &in.EnablePrivateCluster, &out.EnablePrivateCluster *out = new(bool) @@ -89,12 +105,12 @@ func (in *APIServerAccessProfile) DeepCopyInto(out *APIServerAccessProfile) { } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIServerAccessProfile. -func (in *APIServerAccessProfile) DeepCopy() *APIServerAccessProfile { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIServerAccessProfileClassSpec. +func (in *APIServerAccessProfileClassSpec) DeepCopy() *APIServerAccessProfileClassSpec { if in == nil { return nil } - out := new(APIServerAccessProfile) + out := new(APIServerAccessProfileClassSpec) in.DeepCopyInto(out) return out } @@ -1097,26 +1113,25 @@ func (in *AzureManagedClusterStatus) DeepCopy() *AzureManagedClusterStatus { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AzureManagedControlPlane) DeepCopyInto(out *AzureManagedControlPlane) { +func (in *AzureManagedClusterTemplate) DeepCopyInto(out *AzureManagedClusterTemplate) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) + out.Spec = in.Spec } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlane. -func (in *AzureManagedControlPlane) DeepCopy() *AzureManagedControlPlane { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedClusterTemplate. +func (in *AzureManagedClusterTemplate) DeepCopy() *AzureManagedClusterTemplate { if in == nil { return nil } - out := new(AzureManagedControlPlane) + out := new(AzureManagedClusterTemplate) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *AzureManagedControlPlane) DeepCopyObject() runtime.Object { +func (in *AzureManagedClusterTemplate) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -1124,31 +1139,31 @@ func (in *AzureManagedControlPlane) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AzureManagedControlPlaneList) DeepCopyInto(out *AzureManagedControlPlaneList) { +func (in *AzureManagedClusterTemplateList) DeepCopyInto(out *AzureManagedClusterTemplateList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items - *out = make([]AzureManagedControlPlane, len(*in)) + *out = make([]AzureManagedClusterTemplate, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneList. -func (in *AzureManagedControlPlaneList) DeepCopy() *AzureManagedControlPlaneList { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedClusterTemplateList. +func (in *AzureManagedClusterTemplateList) DeepCopy() *AzureManagedClusterTemplateList { if in == nil { return nil } - out := new(AzureManagedControlPlaneList) + out := new(AzureManagedClusterTemplateList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *AzureManagedControlPlaneList) DeepCopyObject() runtime.Object { +func (in *AzureManagedClusterTemplateList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -1156,10 +1171,88 @@ func (in *AzureManagedControlPlaneList) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPlaneSpec) { +func (in *AzureManagedClusterTemplateResource) DeepCopyInto(out *AzureManagedClusterTemplateResource) { *out = *in + out.Spec = in.Spec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedClusterTemplateResource. +func (in *AzureManagedClusterTemplateResource) DeepCopy() *AzureManagedClusterTemplateResource { + if in == nil { + return nil + } + out := new(AzureManagedClusterTemplateResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedClusterTemplateResourceSpec) DeepCopyInto(out *AzureManagedClusterTemplateResourceSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedClusterTemplateResourceSpec. +func (in *AzureManagedClusterTemplateResourceSpec) DeepCopy() *AzureManagedClusterTemplateResourceSpec { + if in == nil { + return nil + } + out := new(AzureManagedClusterTemplateResourceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedClusterTemplateSpec) DeepCopyInto(out *AzureManagedClusterTemplateSpec) { + *out = *in + out.Template = in.Template +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedClusterTemplateSpec. +func (in *AzureManagedClusterTemplateSpec) DeepCopy() *AzureManagedClusterTemplateSpec { + if in == nil { + return nil + } + out := new(AzureManagedClusterTemplateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedControlPlane) DeepCopyInto(out *AzureManagedControlPlane) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlane. +func (in *AzureManagedControlPlane) DeepCopy() *AzureManagedControlPlane { + if in == nil { + return nil + } + out := new(AzureManagedControlPlane) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AzureManagedControlPlane) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedControlPlaneClassSpec) DeepCopyInto(out *AzureManagedControlPlaneClassSpec) { + *out = *in + if in.MachineTemplate != nil { + in, out := &in.MachineTemplate, &out.MachineTemplate + *out = new(AzureManagedControlPlaneTemplateMachineTemplate) + **out = **in + } in.VirtualNetwork.DeepCopyInto(&out.VirtualNetwork) - out.ControlPlaneEndpoint = in.ControlPlaneEndpoint if in.AdditionalTags != nil { in, out := &in.AdditionalTags, &out.AdditionalTags *out = make(Tags, len(*in)) @@ -1187,11 +1280,6 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = new(ManagedControlPlaneOutboundType) **out = **in } - if in.SSHPublicKey != nil { - in, out := &in.SSHPublicKey, &out.SSHPublicKey - *out = new(string) - **out = **in - } if in.DNSServiceIP != nil { in, out := &in.DNSServiceIP, &out.DNSServiceIP *out = new(string) @@ -1254,11 +1342,6 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = new(OIDCIssuerProfile) (*in).DeepCopyInto(*out) } - if in.DNSPrefix != nil { - in, out := &in.DNSPrefix, &out.DNSPrefix - *out = new(string) - **out = **in - } if in.DisableLocalAccounts != nil { in, out := &in.DisableLocalAccounts, &out.DisableLocalAccounts *out = new(bool) @@ -1266,6 +1349,65 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla } } +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneClassSpec. +func (in *AzureManagedControlPlaneClassSpec) DeepCopy() *AzureManagedControlPlaneClassSpec { + if in == nil { + return nil + } + out := new(AzureManagedControlPlaneClassSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedControlPlaneList) DeepCopyInto(out *AzureManagedControlPlaneList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AzureManagedControlPlane, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneList. +func (in *AzureManagedControlPlaneList) DeepCopy() *AzureManagedControlPlaneList { + if in == nil { + return nil + } + out := new(AzureManagedControlPlaneList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AzureManagedControlPlaneList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPlaneSpec) { + *out = *in + in.AzureManagedControlPlaneClassSpec.DeepCopyInto(&out.AzureManagedControlPlaneClassSpec) + out.ControlPlaneEndpoint = in.ControlPlaneEndpoint + if in.SSHPublicKey != nil { + in, out := &in.SSHPublicKey, &out.SSHPublicKey + *out = new(string) + **out = **in + } + if in.DNSPrefix != nil { + in, out := &in.DNSPrefix, &out.DNSPrefix + *out = new(string) + **out = **in + } +} + // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneSpec. func (in *AzureManagedControlPlaneSpec) DeepCopy() *AzureManagedControlPlaneSpec { if in == nil { @@ -1309,26 +1451,25 @@ func (in *AzureManagedControlPlaneStatus) DeepCopy() *AzureManagedControlPlaneSt } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AzureManagedMachinePool) DeepCopyInto(out *AzureManagedMachinePool) { +func (in *AzureManagedControlPlaneTemplate) DeepCopyInto(out *AzureManagedControlPlaneTemplate) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePool. -func (in *AzureManagedMachinePool) DeepCopy() *AzureManagedMachinePool { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneTemplate. +func (in *AzureManagedControlPlaneTemplate) DeepCopy() *AzureManagedControlPlaneTemplate { if in == nil { return nil } - out := new(AzureManagedMachinePool) + out := new(AzureManagedControlPlaneTemplate) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *AzureManagedMachinePool) DeepCopyObject() runtime.Object { +func (in *AzureManagedControlPlaneTemplate) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -1336,31 +1477,31 @@ func (in *AzureManagedMachinePool) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AzureManagedMachinePoolList) DeepCopyInto(out *AzureManagedMachinePoolList) { +func (in *AzureManagedControlPlaneTemplateList) DeepCopyInto(out *AzureManagedControlPlaneTemplateList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items - *out = make([]AzureManagedMachinePool, len(*in)) + *out = make([]AzureManagedControlPlaneTemplate, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolList. -func (in *AzureManagedMachinePoolList) DeepCopy() *AzureManagedMachinePoolList { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneTemplateList. +func (in *AzureManagedControlPlaneTemplateList) DeepCopy() *AzureManagedControlPlaneTemplateList { if in == nil { return nil } - out := new(AzureManagedMachinePoolList) + out := new(AzureManagedControlPlaneTemplateList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *AzureManagedMachinePoolList) DeepCopyObject() runtime.Object { +func (in *AzureManagedControlPlaneTemplateList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -1368,7 +1509,97 @@ func (in *AzureManagedMachinePoolList) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AzureManagedMachinePoolSpec) DeepCopyInto(out *AzureManagedMachinePoolSpec) { +func (in *AzureManagedControlPlaneTemplateMachineTemplate) DeepCopyInto(out *AzureManagedControlPlaneTemplateMachineTemplate) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneTemplateMachineTemplate. +func (in *AzureManagedControlPlaneTemplateMachineTemplate) DeepCopy() *AzureManagedControlPlaneTemplateMachineTemplate { + if in == nil { + return nil + } + out := new(AzureManagedControlPlaneTemplateMachineTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedControlPlaneTemplateResource) DeepCopyInto(out *AzureManagedControlPlaneTemplateResource) { + *out = *in + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneTemplateResource. +func (in *AzureManagedControlPlaneTemplateResource) DeepCopy() *AzureManagedControlPlaneTemplateResource { + if in == nil { + return nil + } + out := new(AzureManagedControlPlaneTemplateResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedControlPlaneTemplateResourceSpec) DeepCopyInto(out *AzureManagedControlPlaneTemplateResourceSpec) { + *out = *in + in.AzureManagedControlPlaneClassSpec.DeepCopyInto(&out.AzureManagedControlPlaneClassSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneTemplateResourceSpec. +func (in *AzureManagedControlPlaneTemplateResourceSpec) DeepCopy() *AzureManagedControlPlaneTemplateResourceSpec { + if in == nil { + return nil + } + out := new(AzureManagedControlPlaneTemplateResourceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedControlPlaneTemplateSpec) DeepCopyInto(out *AzureManagedControlPlaneTemplateSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneTemplateSpec. +func (in *AzureManagedControlPlaneTemplateSpec) DeepCopy() *AzureManagedControlPlaneTemplateSpec { + if in == nil { + return nil + } + out := new(AzureManagedControlPlaneTemplateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedMachinePool) DeepCopyInto(out *AzureManagedMachinePool) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePool. +func (in *AzureManagedMachinePool) DeepCopy() *AzureManagedMachinePool { + if in == nil { + return nil + } + out := new(AzureManagedMachinePool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AzureManagedMachinePool) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedMachinePoolClassSpec) DeepCopyInto(out *AzureManagedMachinePoolClassSpec) { *out = *in if in.AdditionalTags != nil { in, out := &in.AdditionalTags, &out.AdditionalTags @@ -1404,11 +1635,6 @@ func (in *AzureManagedMachinePoolSpec) DeepCopyInto(out *AzureManagedMachinePool *out = make(Taints, len(*in)) copy(*out, *in) } - if in.ProviderIDList != nil { - in, out := &in.ProviderIDList, &out.ProviderIDList - *out = make([]string, len(*in)) - copy(*out, *in) - } if in.Scaling != nil { in, out := &in.Scaling, &out.Scaling *out = new(ManagedMachinePoolScaling) @@ -1491,6 +1717,59 @@ func (in *AzureManagedMachinePoolSpec) DeepCopyInto(out *AzureManagedMachinePool } } +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolClassSpec. +func (in *AzureManagedMachinePoolClassSpec) DeepCopy() *AzureManagedMachinePoolClassSpec { + if in == nil { + return nil + } + out := new(AzureManagedMachinePoolClassSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedMachinePoolList) DeepCopyInto(out *AzureManagedMachinePoolList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AzureManagedMachinePool, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolList. +func (in *AzureManagedMachinePoolList) DeepCopy() *AzureManagedMachinePoolList { + if in == nil { + return nil + } + out := new(AzureManagedMachinePoolList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AzureManagedMachinePoolList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedMachinePoolSpec) DeepCopyInto(out *AzureManagedMachinePoolSpec) { + *out = *in + in.AzureManagedMachinePoolClassSpec.DeepCopyInto(&out.AzureManagedMachinePoolClassSpec) + if in.ProviderIDList != nil { + in, out := &in.ProviderIDList, &out.ProviderIDList + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolSpec. func (in *AzureManagedMachinePoolSpec) DeepCopy() *AzureManagedMachinePoolSpec { if in == nil { @@ -1538,6 +1817,112 @@ func (in *AzureManagedMachinePoolStatus) DeepCopy() *AzureManagedMachinePoolStat return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedMachinePoolTemplate) DeepCopyInto(out *AzureManagedMachinePoolTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolTemplate. +func (in *AzureManagedMachinePoolTemplate) DeepCopy() *AzureManagedMachinePoolTemplate { + if in == nil { + return nil + } + out := new(AzureManagedMachinePoolTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AzureManagedMachinePoolTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedMachinePoolTemplateList) DeepCopyInto(out *AzureManagedMachinePoolTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AzureManagedMachinePoolTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolTemplateList. +func (in *AzureManagedMachinePoolTemplateList) DeepCopy() *AzureManagedMachinePoolTemplateList { + if in == nil { + return nil + } + out := new(AzureManagedMachinePoolTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AzureManagedMachinePoolTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedMachinePoolTemplateResource) DeepCopyInto(out *AzureManagedMachinePoolTemplateResource) { + *out = *in + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolTemplateResource. +func (in *AzureManagedMachinePoolTemplateResource) DeepCopy() *AzureManagedMachinePoolTemplateResource { + if in == nil { + return nil + } + out := new(AzureManagedMachinePoolTemplateResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedMachinePoolTemplateResourceSpec) DeepCopyInto(out *AzureManagedMachinePoolTemplateResourceSpec) { + *out = *in + in.AzureManagedMachinePoolClassSpec.DeepCopyInto(&out.AzureManagedMachinePoolClassSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolTemplateResourceSpec. +func (in *AzureManagedMachinePoolTemplateResourceSpec) DeepCopy() *AzureManagedMachinePoolTemplateResourceSpec { + if in == nil { + return nil + } + out := new(AzureManagedMachinePoolTemplateResourceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedMachinePoolTemplateSpec) DeepCopyInto(out *AzureManagedMachinePoolTemplateSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolTemplateSpec. +func (in *AzureManagedMachinePoolTemplateSpec) DeepCopy() *AzureManagedMachinePoolTemplateSpec { + if in == nil { + return nil + } + out := new(AzureManagedMachinePoolTemplateSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureMarketplaceImage) DeepCopyInto(out *AzureMarketplaceImage) { *out = *in @@ -2240,7 +2625,7 @@ func (in *ManagedControlPlaneSubnet) DeepCopy() *ManagedControlPlaneSubnet { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ManagedControlPlaneVirtualNetwork) DeepCopyInto(out *ManagedControlPlaneVirtualNetwork) { *out = *in - in.Subnet.DeepCopyInto(&out.Subnet) + in.ManagedControlPlaneVirtualNetworkClassSpec.DeepCopyInto(&out.ManagedControlPlaneVirtualNetworkClassSpec) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagedControlPlaneVirtualNetwork. @@ -2253,6 +2638,22 @@ func (in *ManagedControlPlaneVirtualNetwork) DeepCopy() *ManagedControlPlaneVirt return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ManagedControlPlaneVirtualNetworkClassSpec) DeepCopyInto(out *ManagedControlPlaneVirtualNetworkClassSpec) { + *out = *in + in.Subnet.DeepCopyInto(&out.Subnet) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagedControlPlaneVirtualNetworkClassSpec. +func (in *ManagedControlPlaneVirtualNetworkClassSpec) DeepCopy() *ManagedControlPlaneVirtualNetworkClassSpec { + if in == nil { + return nil + } + out := new(ManagedControlPlaneVirtualNetworkClassSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ManagedDiskParameters) DeepCopyInto(out *ManagedDiskParameters) { *out = *in diff --git a/azure/defaults.go b/azure/defaults.go index 90928538d0d..c91495c2e4a 100644 --- a/azure/defaults.go +++ b/azure/defaults.go @@ -230,6 +230,11 @@ func PublicIPID(subscriptionID, resourceGroup, ipName string) string { return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/publicIPAddresses/%s", subscriptionID, resourceGroup, ipName) } +// PublicIPPrefixID returns the azure resource ID for a given public IP prefix. +func PublicIPPrefixID(subscriptionID, resourceGroup, ipName string) string { + return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/publicipprefixes/%s", subscriptionID, resourceGroup, ipName) +} + // RouteTableID returns the azure resource ID for a given route table. func RouteTableID(subscriptionID, resourceGroup, routeTableName string) string { return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/routeTables/%s", subscriptionID, resourceGroup, routeTableName) diff --git a/azure/scope/managedcontrolplane_test.go b/azure/scope/managedcontrolplane_test.go index 75abd004bd6..014f0f06e5a 100644 --- a/azure/scope/managedcontrolplane_test.go +++ b/azure/scope/managedcontrolplane_test.go @@ -55,8 +55,10 @@ func TestManagedControlPlaneScope_OutboundType(t *testing.T) { }, ControlPlane: &infrav1.AzureManagedControlPlane{ Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", - OutboundType: &explicitOutboundType, + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + OutboundType: &explicitOutboundType, + }, }, }, }, @@ -73,7 +75,9 @@ func TestManagedControlPlaneScope_OutboundType(t *testing.T) { }, ControlPlane: &infrav1.AzureManagedControlPlane{ Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, }, @@ -121,7 +125,9 @@ func TestManagedControlPlaneScope_PoolVersion(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -159,8 +165,10 @@ func TestManagedControlPlaneScope_PoolVersion(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - Version: "v1.22.0", - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + Version: "v1.22.0", + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -199,8 +207,10 @@ func TestManagedControlPlaneScope_PoolVersion(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - Version: "v1.20.1", - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + Version: "v1.20.1", + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -257,7 +267,9 @@ func TestManagedControlPlaneScope_AddonProfiles(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -284,10 +296,12 @@ func TestManagedControlPlaneScope_AddonProfiles(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", - AddonProfiles: []infrav1.AddonProfile{ - {Name: "addon1", Config: nil, Enabled: false}, - {Name: "addon2", Config: map[string]string{"k1": "v1", "k2": "v2"}, Enabled: true}, + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AddonProfiles: []infrav1.AddonProfile{ + {Name: "addon1", Config: nil, Enabled: false}, + {Name: "addon2", Config: map[string]string{"k1": "v1", "k2": "v2"}, Enabled: true}, + }, }, }, }, @@ -345,8 +359,10 @@ func TestManagedControlPlaneScope_OSType(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - Version: "v1.20.1", - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + Version: "v1.20.1", + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -414,8 +430,10 @@ func TestManagedControlPlaneScope_OSType(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - Version: "v1.20.1", - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + Version: "v1.20.1", + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -482,8 +500,10 @@ func TestManagedControlPlaneScope_IsVnetManagedCache(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - Version: "v1.20.1", - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + Version: "v1.20.1", + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -515,8 +535,10 @@ func TestManagedControlPlaneScope_IsVnetManagedCache(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - Version: "v1.20.1", - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + Version: "v1.20.1", + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -550,8 +572,10 @@ func TestManagedControlPlaneScope_IsVnetManagedCache(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - Version: "v1.20.1", - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + Version: "v1.20.1", + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -610,7 +634,9 @@ func TestManagedControlPlaneScope_AADProfile(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -637,10 +663,12 @@ func TestManagedControlPlaneScope_AADProfile(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", - AADProfile: &infrav1.AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, }, }, }, @@ -698,7 +726,9 @@ func TestManagedControlPlaneScope_DisableLocalAccounts(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -725,8 +755,10 @@ func TestManagedControlPlaneScope_DisableLocalAccounts(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", - DisableLocalAccounts: ptr.To[bool](true), + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + DisableLocalAccounts: ptr.To[bool](true), + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -753,12 +785,14 @@ func TestManagedControlPlaneScope_DisableLocalAccounts(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", - AADProfile: &infrav1.AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To[bool](true), }, - DisableLocalAccounts: ptr.To[bool](true), }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -811,7 +845,9 @@ func TestIsAADEnabled(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -838,12 +874,14 @@ func TestIsAADEnabled(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", - AADProfile: &infrav1.AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To[bool](true), }, - DisableLocalAccounts: ptr.To[bool](true), }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -894,7 +932,9 @@ func TestAreLocalAccountsDisabled(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -921,10 +961,12 @@ func TestAreLocalAccountsDisabled(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", - AADProfile: &infrav1.AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, }, }, }, @@ -952,12 +994,14 @@ func TestAreLocalAccountsDisabled(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", - AADProfile: &infrav1.AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To[bool](true), }, - DisableLocalAccounts: ptr.To[bool](true), }, }, ManagedMachinePools: []ManagedMachinePool{ diff --git a/azure/scope/managedmachinepool_test.go b/azure/scope/managedmachinepool_test.go index 76a1a02f824..ff692ee0c5a 100644 --- a/azure/scope/managedmachinepool_test.go +++ b/azure/scope/managedmachinepool_test.go @@ -60,7 +60,9 @@ func TestManagedMachinePoolScope_Autoscaling(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -95,7 +97,9 @@ func TestManagedMachinePoolScope_Autoscaling(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -160,7 +164,9 @@ func TestManagedMachinePoolScope_NodeLabels(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -194,7 +200,9 @@ func TestManagedMachinePoolScope_NodeLabels(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -261,7 +269,9 @@ func TestManagedMachinePoolScope_AdditionalTags(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -295,7 +305,9 @@ func TestManagedMachinePoolScope_AdditionalTags(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -362,7 +374,9 @@ func TestManagedMachinePoolScope_MaxPods(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -396,7 +410,9 @@ func TestManagedMachinePoolScope_MaxPods(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -459,7 +475,9 @@ func TestManagedMachinePoolScope_Taints(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -494,7 +512,9 @@ func TestManagedMachinePoolScope_Taints(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -563,7 +583,9 @@ func TestManagedMachinePoolScope_OSDiskType(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -597,7 +619,9 @@ func TestManagedMachinePoolScope_OSDiskType(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -660,7 +684,9 @@ func TestManagedMachinePoolScope_SubnetName(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -694,13 +720,17 @@ func TestManagedMachinePoolScope_SubnetName(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", - VirtualNetwork: infrav1.ManagedControlPlaneVirtualNetwork{ - Name: "my-vnet", - Subnet: infrav1.ManagedControlPlaneSubnet{ - Name: "my-vnet-subnet", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + VirtualNetwork: infrav1.ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: infrav1.ManagedControlPlaneVirtualNetworkClassSpec{ + Name: "my-vnet", + Subnet: infrav1.ManagedControlPlaneSubnet{ + Name: "my-vnet-subnet", + }, + }, + ResourceGroup: "my-resource-group", }, - ResourceGroup: "my-resource-group", }, }, }, @@ -735,13 +765,17 @@ func TestManagedMachinePoolScope_SubnetName(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", - VirtualNetwork: infrav1.ManagedControlPlaneVirtualNetwork{ - Name: "my-vnet", - Subnet: infrav1.ManagedControlPlaneSubnet{ - Name: "my-vnet-subnet", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + VirtualNetwork: infrav1.ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: infrav1.ManagedControlPlaneVirtualNetworkClassSpec{ + Name: "my-vnet", + Subnet: infrav1.ManagedControlPlaneSubnet{ + Name: "my-vnet-subnet", + }, + }, + ResourceGroup: "my-resource-group", }, - ResourceGroup: "my-resource-group", }, }, }, @@ -805,7 +839,9 @@ func TestManagedMachinePoolScope_KubeletDiskType(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -839,7 +875,9 @@ func TestManagedMachinePoolScope_KubeletDiskType(t *testing.T) { Namespace: "default", }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "00000000-0000-0000-0000-000000000000", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, }, }, ManagedMachinePool: ManagedMachinePool{ @@ -894,9 +932,11 @@ func getAzureMachinePool(name string, mode infrav1.NodePoolMode) *infrav1.AzureM }, }, Spec: infrav1.AzureManagedMachinePoolSpec{ - Mode: string(mode), - SKU: "Standard_D2s_v3", - Name: ptr.To(name), + AzureManagedMachinePoolClassSpec: infrav1.AzureManagedMachinePoolClassSpec{ + Mode: string(mode), + SKU: "Standard_D2s_v3", + Name: ptr.To(name), + }, }, } } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedclustertemplates.yaml new file mode 100644 index 00000000000..8f129c2f2de --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedclustertemplates.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: azuremanagedclustertemplates.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureManagedClusterTemplate + listKind: AzureManagedClusterTemplateList + plural: azuremanagedclustertemplates + shortNames: + - amct + singular: azuremanagedclustertemplate + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureManagedClusterTemplate is the Schema for the AzureManagedClusterTemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedClusterTemplateSpec defines the desired state + of AzureManagedClusterTemplate. + properties: + template: + description: AzureManagedClusterTemplateResource describes the data + needed to create an AzureManagedCluster from a template. + properties: + spec: + description: AzureManagedClusterTemplateResourceSpec specifies + an Azure managed cluster template resource. + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index 3a2bba126d4..25297762f30 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -97,16 +97,16 @@ spec: type: string type: array enablePrivateCluster: - description: EnablePrivateCluster - Whether to create the cluster - as a private cluster or not. + description: EnablePrivateCluster indicates whether to create + the cluster as a private cluster or not. type: boolean enablePrivateClusterPublicFQDN: - description: EnablePrivateClusterPublicFQDN - Whether to create - additional public FQDN for private cluster or not. + description: EnablePrivateClusterPublicFQDN indicates whether + to create additional public FQDN for private cluster or not. type: boolean privateDNSZone: - description: PrivateDNSZone - Private dns zone mode for private - cluster. + description: PrivateDNSZone enables private dns zone mode for + private cluster. enum: - System - None @@ -371,15 +371,19 @@ spec: type: string location: description: 'Location is a string matching one of the canonical Azure - region names. Examples: "westus2", "eastus". Immutable.' + region names. Examples: "westus2", "eastus".' type: string + machineTemplate: + description: MachineTemplate contains information about how machines + should be shaped when creating or updating a control plane. For + the AzureManagedControlPlaneTemplate, this field is used only to + fulfill the CAPI contract. + type: object networkPlugin: - description: NetworkPlugin used for building Kubernetes network. Allowed - values are "azure", "kubenet". Immutable. + description: NetworkPlugin used for building Kubernetes network. enum: - azure - kubenet - - none type: string networkPluginMode: description: NetworkPluginMode is the mode the network plugin should @@ -388,8 +392,7 @@ spec: - overlay type: string networkPolicy: - description: NetworkPolicy used for building Kubernetes network. Allowed - values are "azure", "calico". Immutable. + description: NetworkPolicy used for building Kubernetes network. enum: - azure - calico @@ -408,7 +411,7 @@ spec: type: boolean type: object outboundType: - description: Outbound configuration used by Nodes. Immutable. + description: Outbound configuration used by Nodes. enum: - loadBalancer - managedNATGateway @@ -439,15 +442,15 @@ spec: type: string subscriptionID: description: SubscriptionID is the GUID of the Azure subscription - to hold this cluster. Immutable. + that owns this cluster. type: string version: description: Version defines the desired Kubernetes version. minLength: 2 type: string virtualNetwork: - description: VirtualNetwork describes the vnet for the AKS cluster. - Will be created if it does not exist. Immutable except for `subnet`. + description: VirtualNetwork describes the virtual network for the + AKS cluster. It will be created if it does not already exist. properties: cidrBlock: type: string @@ -458,7 +461,8 @@ spec: for the VNet and Subnet. type: string subnet: - description: Immutable except for `serviceEndpoints`. + description: ManagedControlPlaneSubnet describes a subnet for + an AKS cluster. properties: cidrBlock: type: string @@ -682,6 +686,10 @@ spec: ready: description: Ready is true when the provider resource is ready. type: boolean + version: + description: Version defines the Kubernetes version for the control + plane instance. + type: string type: object type: object served: true diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml new file mode 100644 index 00000000000..e906cc5546d --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml @@ -0,0 +1,594 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: azuremanagedcontrolplanetemplates.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureManagedControlPlaneTemplate + listKind: AzureManagedControlPlaneTemplateList + plural: azuremanagedcontrolplanetemplates + shortNames: + - amcpt + singular: azuremanagedcontrolplanetemplate + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureManagedControlPlaneTemplate is the Schema for the AzureManagedControlPlaneTemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedControlPlaneTemplateSpec defines the desired + state of AzureManagedControlPlaneTemplate. + properties: + template: + description: AzureManagedControlPlaneTemplateResource describes the + data needed to create an AzureManagedCluster from a template. + properties: + spec: + description: AzureManagedControlPlaneTemplateResourceSpec specifies + an Azure managed control plane template resource. + properties: + aadProfile: + description: AadProfile is Azure Active Directory configuration + to integrate with AKS for aad authentication. + properties: + adminGroupObjectIDs: + description: AdminGroupObjectIDs - AAD group object IDs + that will have admin role of the cluster. + items: + type: string + type: array + managed: + description: Managed - Whether to enable managed AAD. + type: boolean + required: + - adminGroupObjectIDs + - managed + type: object + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to Azure resources managed by the Azure provider, in + addition to the ones added by default. + type: object + addonProfiles: + description: AddonProfiles are the profiles of managed cluster + add-on. + items: + description: AddonProfile represents a managed cluster add-on. + properties: + config: + additionalProperties: + type: string + description: Config - Key-value pairs for configuring + the add-on. + type: object + enabled: + description: Enabled - Whether the add-on is enabled + or not. + type: boolean + name: + description: Name - The name of the managed cluster + add-on. + type: string + required: + - enabled + - name + type: object + type: array + apiServerAccessProfile: + description: APIServerAccessProfile is the access profile + for AKS API server. Immutable except for `authorizedIPRanges`. + properties: + authorizedIPRanges: + description: AuthorizedIPRanges - Authorized IP Ranges + to kubernetes API server. + items: + type: string + type: array + enablePrivateCluster: + description: EnablePrivateCluster indicates whether to + create the cluster as a private cluster or not. + type: boolean + enablePrivateClusterPublicFQDN: + description: EnablePrivateClusterPublicFQDN indicates + whether to create additional public FQDN for private + cluster or not. + type: boolean + privateDNSZone: + description: PrivateDNSZone enables private dns zone mode + for private cluster. + enum: + - System + - None + type: string + type: object + autoscalerProfile: + description: AutoscalerProfile is the parameters to be applied + to the cluster-autoscaler when enabled + properties: + balanceSimilarNodeGroups: + description: BalanceSimilarNodeGroups - Valid values are + 'true' and 'false'. The default is false. + enum: + - "true" + - "false" + type: string + expander: + description: Expander - If not specified, the default + is 'random'. See [expanders](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-expanders) + for more information. + enum: + - least-waste + - most-pods + - priority + - random + type: string + maxEmptyBulkDelete: + description: MaxEmptyBulkDelete - The default is 10. + type: string + maxGracefulTerminationSec: + description: MaxGracefulTerminationSec - The default is + 600. + pattern: ^(\d+)$ + type: string + maxNodeProvisionTime: + description: MaxNodeProvisionTime - The default is '15m'. + Values must be an integer followed by an 'm'. No unit + of time other than minutes (m) is supported. + pattern: ^(\d+)m$ + type: string + maxTotalUnreadyPercentage: + description: MaxTotalUnreadyPercentage - The default is + 45. The maximum is 100 and the minimum is 0. + maxLength: 3 + minLength: 1 + pattern: ^(\d+)$ + type: string + newPodScaleUpDelay: + description: NewPodScaleUpDelay - For scenarios like burst/batch + scale where you don't want CA to act before the kubernetes + scheduler could schedule all the pods, you can tell + CA to ignore unscheduled pods before they're a certain + age. The default is '0s'. Values must be an integer + followed by a unit ('s' for seconds, 'm' for minutes, + 'h' for hours, etc). + type: string + okTotalUnreadyCount: + description: OkTotalUnreadyCount - This must be an integer. + The default is 3. + pattern: ^(\d+)$ + type: string + scaleDownDelayAfterAdd: + description: ScaleDownDelayAfterAdd - The default is '10m'. + Values must be an integer followed by an 'm'. No unit + of time other than minutes (m) is supported. + pattern: ^(\d+)m$ + type: string + scaleDownDelayAfterDelete: + description: ScaleDownDelayAfterDelete - The default is + the scan-interval. Values must be an integer followed + by an 's'. No unit of time other than seconds (s) is + supported. + pattern: ^(\d+)s$ + type: string + scaleDownDelayAfterFailure: + description: ScaleDownDelayAfterFailure - The default + is '3m'. Values must be an integer followed by an 'm'. + No unit of time other than minutes (m) is supported. + pattern: ^(\d+)m$ + type: string + scaleDownUnneededTime: + description: ScaleDownUnneededTime - The default is '10m'. + Values must be an integer followed by an 'm'. No unit + of time other than minutes (m) is supported. + pattern: ^(\d+)m$ + type: string + scaleDownUnreadyTime: + description: ScaleDownUnreadyTime - The default is '20m'. + Values must be an integer followed by an 'm'. No unit + of time other than minutes (m) is supported. + pattern: ^(\d+)m$ + type: string + scaleDownUtilizationThreshold: + description: ScaleDownUtilizationThreshold - The default + is '0.5'. + type: string + scanInterval: + description: ScanInterval - How often cluster is reevaluated + for scale up or down. The default is '10s'. + pattern: ^(\d+)s$ + type: string + skipNodesWithLocalStorage: + description: SkipNodesWithLocalStorage - The default is + false. + enum: + - "true" + - "false" + type: string + skipNodesWithSystemPods: + description: SkipNodesWithSystemPods - The default is + true. + enum: + - "true" + - "false" + type: string + type: object + azureEnvironment: + description: 'AzureEnvironment is the name of the AzureCloud + to be used. The default value that would be used by most + users is "AzurePublicCloud", other values are: - ChinaCloud: + "AzureChinaCloud" - PublicCloud: "AzurePublicCloud" - USGovernmentCloud: + "AzureUSGovernmentCloud"' + type: string + disableLocalAccounts: + description: DisableLocalAccounts disables getting static + credentials for this cluster when set. Expected to only + be used for AAD clusters. + type: boolean + dnsServiceIP: + description: DNSServiceIP is an IP address assigned to the + Kubernetes DNS service. It must be within the Kubernetes + service address range specified in serviceCidr. Immutable. + type: string + httpProxyConfig: + description: HTTPProxyConfig is the HTTP proxy configuration + for the cluster. Immutable. + properties: + httpProxy: + description: HTTPProxy is the HTTP proxy server endpoint + to use. + type: string + httpsProxy: + description: HTTPSProxy is the HTTPS proxy server endpoint + to use. + type: string + noProxy: + description: NoProxy indicates the endpoints that should + not go through proxy. + items: + type: string + type: array + trustedCa: + description: TrustedCA is the alternative CA cert to use + for connecting to proxy servers. + type: string + type: object + identity: + description: Identity configuration used by the AKS control + plane. + properties: + type: + description: Type - The Identity type to use. + enum: + - SystemAssigned + - UserAssigned + type: string + userAssignedIdentityResourceID: + description: UserAssignedIdentityResourceID - Identity + ARM resource ID when using user-assigned identity. + type: string + type: object + identityRef: + description: IdentityRef is a reference to a AzureClusterIdentity + to be used when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this + pod). This syntax is chosen only to have some well-defined + way of referencing a part of an object. TODO: this design + is not final and this field is subject to change in + the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + kubeletUserAssignedIdentity: + description: KubeletUserAssignedIdentity is the user-assigned + identity for kubelet. For authentication with Azure Container + Registry. + type: string + loadBalancerProfile: + description: LoadBalancerProfile is the profile of the cluster + load balancer. + properties: + allocatedOutboundPorts: + description: AllocatedOutboundPorts - Desired number of + allocated SNAT ports per VM. Allowed values must be + in the range of 0 to 64000 (inclusive). The default + value is 0 which results in Azure dynamically allocating + ports. + type: integer + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes - Desired outbound flow + idle timeout in minutes. Allowed values must be in the + range of 4 to 120 (inclusive). The default value is + 30 minutes. + type: integer + managedOutboundIPs: + description: ManagedOutboundIPs - Desired managed outbound + IPs for the cluster load balancer. + type: integer + outboundIPPrefixes: + description: OutboundIPPrefixes - Desired outbound IP + Prefix resources for the cluster load balancer. + items: + type: string + type: array + outboundIPs: + description: OutboundIPs - Desired outbound IP resources + for the cluster load balancer. + items: + type: string + type: array + type: object + loadBalancerSKU: + default: Standard + description: LoadBalancerSKU is the SKU of the loadBalancer + to be provisioned. Immutable. + enum: + - Basic + - Standard + type: string + location: + description: 'Location is a string matching one of the canonical + Azure region names. Examples: "westus2", "eastus".' + type: string + machineTemplate: + description: MachineTemplate contains information about how + machines should be shaped when creating or updating a control + plane. For the AzureManagedControlPlaneTemplate, this field + is used only to fulfill the CAPI contract. + type: object + networkPlugin: + description: NetworkPlugin used for building Kubernetes network. + enum: + - azure + - kubenet + type: string + networkPluginMode: + description: NetworkPluginMode is the mode the network plugin + should use. Allowed value is "overlay". + enum: + - overlay + type: string + networkPolicy: + description: NetworkPolicy used for building Kubernetes network. + enum: + - azure + - calico + type: string + oidcIssuerProfile: + description: OIDCIssuerProfile is the OIDC issuer profile + of the Managed Cluster. + properties: + enabled: + description: Enabled is whether the OIDC issuer is enabled. + type: boolean + type: object + outboundType: + description: Outbound configuration used by Nodes. + enum: + - loadBalancer + - managedNATGateway + - userAssignedNATGateway + - userDefinedRouting + type: string + sku: + description: SKU is the SKU of the AKS to be provisioned. + properties: + tier: + description: Tier - Tier of an AKS cluster. + enum: + - Free + - Paid + - Standard + type: string + required: + - tier + type: object + subscriptionID: + description: SubscriptionID is the GUID of the Azure subscription + that owns this cluster. + type: string + version: + description: Version defines the desired Kubernetes version. + minLength: 2 + type: string + virtualNetwork: + description: VirtualNetwork describes the virtual network + for the AKS cluster. It will be created if it does not already + exist. + properties: + cidrBlock: + type: string + name: + type: string + resourceGroup: + description: ResourceGroup is the name of the Azure resource + group for the VNet and Subnet. + type: string + subnet: + description: ManagedControlPlaneSubnet describes a subnet + for an AKS cluster. + properties: + cidrBlock: + type: string + name: + type: string + privateEndpoints: + description: PrivateEndpoints is a slice of Virtual + Network private endpoints to create for the subnets. + items: + description: PrivateEndpointSpec configures an Azure + Private Endpoint. + properties: + applicationSecurityGroups: + description: ApplicationSecurityGroups specifies + the Application security group in which the + private endpoint IP configuration is included. + items: + type: string + type: array + customNetworkInterfaceName: + description: CustomNetworkInterfaceName specifies + the network interface name associated with + the private endpoint. + type: string + location: + description: Location specifies the region to + create the private endpoint. + type: string + manualApproval: + description: ManualApproval specifies if the + connection approval needs to be done manually + or not. Set it true when the network admin + does not have access to approve connections + to the remote resource. Defaults to false. + type: boolean + name: + description: Name specifies the name of the + private endpoint. + type: string + privateIPAddresses: + description: PrivateIPAddresses specifies the + IP addresses for the network interface associated + with the private endpoint. They have to be + part of the subnet where the private endpoint + is linked. + items: + type: string + type: array + privateLinkServiceConnections: + description: PrivateLinkServiceConnections specifies + Private Link Service Connections of the private + endpoint. + items: + description: PrivateLinkServiceConnection + defines the specification for a private + link service connection associated with + a private endpoint. + properties: + groupIDs: + description: GroupIDs specifies the ID(s) + of the group(s) obtained from the remote + resource that this private endpoint + should connect to. + items: + type: string + type: array + name: + description: Name specifies the name of + the private link service. + type: string + privateLinkServiceID: + description: PrivateLinkServiceID specifies + the resource ID of the private link + service. + type: string + requestMessage: + description: RequestMessage specifies + a message passed to the owner of the + remote resource with the private endpoint + connection request. + maxLength: 140 + type: string + type: object + type: array + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + serviceEndpoints: + description: ServiceEndpoints is a slice of Virtual + Network service endpoints to enable for the subnets. + items: + description: ServiceEndpointSpec configures an Azure + Service Endpoint. + properties: + locations: + items: + type: string + type: array + service: + type: string + required: + - locations + - service + type: object + type: array + x-kubernetes-list-map-keys: + - service + x-kubernetes-list-type: map + required: + - cidrBlock + - name + type: object + required: + - cidrBlock + - name + type: object + required: + - identityRef + - location + - version + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml index 6f9abbdaf3f..faef2fd9414 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml @@ -421,21 +421,21 @@ spec: [K8s doc]: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/" type: integer mode: - description: 'Mode - represents mode of an agent pool. Possible values - include: System, User.' + description: 'Mode represents the mode of an agent pool. Possible + values include: System, User.' enum: - System - User type: string name: - description: Name - name of the agent pool. If not specified, CAPZ - uses the name of the CR as the agent pool name. Immutable. + description: Name is the name of the agent pool. If not specified, + CAPZ uses the name of the CR as the agent pool name. Immutable. type: string nodeLabels: additionalProperties: type: string - description: "Node labels - labels for all of the nodes present in - node pool. See also [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/azure/aks/use-labels" + description: "Node labels represent the labels for all of the nodes + present in node pool. See also [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/azure/aks/use-labels" type: object nodePublicIPPrefixID: description: NodePublicIPPrefixID specifies the public IP prefix resource diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepooltemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepooltemplates.yaml new file mode 100644 index 00000000000..4495624a7fe --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepooltemplates.yaml @@ -0,0 +1,584 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: azuremanagedmachinepooltemplates.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureManagedMachinePoolTemplate + listKind: AzureManagedMachinePoolTemplateList + plural: azuremanagedmachinepooltemplates + shortNames: + - ammpt + singular: azuremanagedmachinepooltemplate + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureManagedMachinePoolTemplate is the Schema for the AzureManagedMachinePoolTemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedMachinePoolTemplateSpec defines the desired state + of AzureManagedMachinePoolTemplate. + properties: + template: + description: AzureManagedMachinePoolTemplateResource describes the + data needed to create an AzureManagedCluster from a template. + properties: + spec: + description: AzureManagedMachinePoolTemplateResourceSpec specifies + an Azure managed control plane template resource. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to Azure resources managed by the Azure provider, in + addition to the ones added by default. + type: object + availabilityZones: + description: AvailabilityZones - Availability zones for nodes. + Must use VirtualMachineScaleSets AgentPoolType. Immutable. + items: + type: string + type: array + enableEncryptionAtHost: + description: "EnableEncryptionAtHost indicates whether host + encryption is enabled on the node pool. Immutable. See also + [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/en-us/azure/aks/enable-host-encryption" + type: boolean + enableFIPS: + description: EnableFIPS indicates whether FIPS is enabled + on the node pool. Immutable. + type: boolean + enableNodePublicIP: + description: EnableNodePublicIP controls whether or not nodes + in the pool each have a public IP address. Immutable. + type: boolean + enableUltraSSD: + description: EnableUltraSSD enables the storage type UltraSSD_LRS + for the agent pool. Immutable. + type: boolean + kubeletConfig: + description: KubeletConfig specifies the kubelet configurations + for nodes. Immutable. + properties: + allowedUnsafeSysctls: + description: AllowedUnsafeSysctls - Allowlist of unsafe + sysctls or unsafe sysctl patterns (ending in `*`). Valid + values match `kernel.shm*`, `kernel.msg*`, `kernel.sem`, + `fs.mqueue.*`, or `net.*`. + items: + type: string + type: array + containerLogMaxFiles: + description: ContainerLogMaxFiles - The maximum number + of container log files that can be present for a container. + The number must be ≥ 2. + minimum: 2 + type: integer + containerLogMaxSizeMB: + description: ContainerLogMaxSizeMB - The maximum size + in MB of a container log file before it is rotated. + type: integer + cpuCfsQuota: + description: CPUCfsQuota - Enable CPU CFS quota enforcement + for containers that specify CPU limits. + type: boolean + cpuCfsQuotaPeriod: + description: CPUCfsQuotaPeriod - Sets CPU CFS quota period + value. Must end in "ms", e.g. "100ms" + type: string + cpuManagerPolicy: + description: CPUManagerPolicy - CPU Manager policy to + use. + enum: + - none + - static + type: string + failSwapOn: + description: FailSwapOn - If set to true it will make + the Kubelet fail to start if swap is enabled on the + node. + type: boolean + imageGcHighThreshold: + description: ImageGcHighThreshold - The percent of disk + usage after which image garbage collection is always + run. Valid values are 0-100 (inclusive). + maximum: 100 + minimum: 0 + type: integer + imageGcLowThreshold: + description: ImageGcLowThreshold - The percent of disk + usage before which image garbage collection is never + run. Valid values are 0-100 (inclusive) and must be + less than `imageGcHighThreshold`. + maximum: 100 + minimum: 0 + type: integer + podMaxPids: + description: PodMaxPids - The maximum number of processes + per pod. Must not exceed kernel PID limit. -1 disables + the limit. + minimum: -1 + type: integer + topologyManagerPolicy: + description: TopologyManagerPolicy - Topology Manager + policy to use. + enum: + - none + - best-effort + - restricted + - single-numa-node + type: string + type: object + kubeletDiskType: + description: "KubeletDiskType specifies the kubelet disk type. + Default to OS. Possible values include: 'OS', 'Temporary'. + Requires Microsoft.ContainerService/KubeletDisk preview + feature to be set. Immutable. See also [AKS doc]. \n [AKS + doc]: https://learn.microsoft.com/rest/api/aks/agent-pools/create-or-update?tabs=HTTP#kubeletdisktype" + enum: + - OS + - Temporary + type: string + linuxOSConfig: + description: LinuxOSConfig specifies the custom Linux OS settings + and configurations. Immutable. + properties: + swapFileSizeMB: + description: "SwapFileSizeMB specifies size in MB of a + swap file will be created on the agent nodes from this + node pool. Max value of SwapFileSizeMB should be the + size of temporary disk(/dev/sdb). Must be at least 1. + See also [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/azure/virtual-machines/managed-disks-overview#temporary-disk" + minimum: 1 + type: integer + sysctls: + description: Sysctl specifies the settings for Linux agent + nodes. + properties: + fsAioMaxNr: + description: FsAioMaxNr specifies the maximum number + of system-wide asynchronous io requests. Valid values + are 65536-6553500 (inclusive). Maps to fs.aio-max-nr. + maximum: 6553500 + minimum: 65536 + type: integer + fsFileMax: + description: FsFileMax specifies the max number of + file-handles that the Linux kernel will allocate, + by increasing increases the maximum number of open + files permitted. Valid values are 8192-12000500 + (inclusive). Maps to fs.file-max. + maximum: 12000500 + minimum: 8192 + type: integer + fsInotifyMaxUserWatches: + description: FsInotifyMaxUserWatches specifies the + number of file watches allowed by the system. Each + watch is roughly 90 bytes on a 32-bit kernel, and + roughly 160 bytes on a 64-bit kernel. Valid values + are 781250-2097152 (inclusive). Maps to fs.inotify.max_user_watches. + maximum: 2097152 + minimum: 781250 + type: integer + fsNrOpen: + description: FsNrOpen specifies the maximum number + of file-handles a process can allocate. Valid values + are 8192-20000500 (inclusive). Maps to fs.nr_open. + maximum: 20000500 + minimum: 8192 + type: integer + kernelThreadsMax: + description: KernelThreadsMax specifies the maximum + number of all threads that can be created. Valid + values are 20-513785 (inclusive). Maps to kernel.threads-max. + maximum: 513785 + minimum: 20 + type: integer + netCoreNetdevMaxBacklog: + description: NetCoreNetdevMaxBacklog specifies maximum + number of packets, queued on the INPUT side, when + the interface receives packets faster than kernel + can process them. Valid values are 1000-3240000 + (inclusive). Maps to net.core.netdev_max_backlog. + maximum: 3240000 + minimum: 1000 + type: integer + netCoreOptmemMax: + description: NetCoreOptmemMax specifies the maximum + ancillary buffer size (option memory buffer) allowed + per socket. Socket option memory is used in a few + cases to store extra structures relating to usage + of the socket. Valid values are 20480-4194304 (inclusive). + Maps to net.core.optmem_max. + maximum: 4194304 + minimum: 20480 + type: integer + netCoreRmemDefault: + description: NetCoreRmemDefault specifies the default + receive socket buffer size in bytes. Valid values + are 212992-134217728 (inclusive). Maps to net.core.rmem_default. + maximum: 134217728 + minimum: 212992 + type: integer + netCoreRmemMax: + description: NetCoreRmemMax specifies the maximum + receive socket buffer size in bytes. Valid values + are 212992-134217728 (inclusive). Maps to net.core.rmem_max. + maximum: 134217728 + minimum: 212992 + type: integer + netCoreSomaxconn: + description: NetCoreSomaxconn specifies maximum number + of connection requests that can be queued for any + given listening socket. An upper limit for the value + of the backlog parameter passed to the listen(2)(https://man7.org/linux/man-pages/man2/listen.2.html) + function. If the backlog argument is greater than + the somaxconn, then it's silently truncated to this + limit. Valid values are 4096-3240000 (inclusive). + Maps to net.core.somaxconn. + maximum: 3240000 + minimum: 4096 + type: integer + netCoreWmemDefault: + description: NetCoreWmemDefault specifies the default + send socket buffer size in bytes. Valid values are + 212992-134217728 (inclusive). Maps to net.core.wmem_default. + maximum: 134217728 + minimum: 212992 + type: integer + netCoreWmemMax: + description: NetCoreWmemMax specifies the maximum + send socket buffer size in bytes. Valid values are + 212992-134217728 (inclusive). Maps to net.core.wmem_max. + maximum: 134217728 + minimum: 212992 + type: integer + netIpv4IPLocalPortRange: + description: NetIpv4IPLocalPortRange is used by TCP + and UDP traffic to choose the local port on the + agent node. PortRange should be specified in the + format "first last". First, being an integer, must + be between [1024 - 60999]. Last, being an integer, + must be between [32768 - 65000]. Maps to net.ipv4.ip_local_port_range. + type: string + netIpv4NeighDefaultGcThresh1: + description: NetIpv4NeighDefaultGcThresh1 specifies + the minimum number of entries that may be in the + ARP cache. Garbage collection won't be triggered + if the number of entries is below this setting. + Valid values are 128-80000 (inclusive). Maps to + net.ipv4.neigh.default.gc_thresh1. + maximum: 80000 + minimum: 128 + type: integer + netIpv4NeighDefaultGcThresh2: + description: NetIpv4NeighDefaultGcThresh2 specifies + soft maximum number of entries that may be in the + ARP cache. ARP garbage collection will be triggered + about 5 seconds after reaching this soft maximum. + Valid values are 512-90000 (inclusive). Maps to + net.ipv4.neigh.default.gc_thresh2. + maximum: 90000 + minimum: 512 + type: integer + netIpv4NeighDefaultGcThresh3: + description: NetIpv4NeighDefaultGcThresh3 specified + hard maximum number of entries in the ARP cache. + Valid values are 1024-100000 (inclusive). Maps to + net.ipv4.neigh.default.gc_thresh3. + maximum: 100000 + minimum: 1024 + type: integer + netIpv4TCPFinTimeout: + description: NetIpv4TCPFinTimeout specifies the length + of time an orphaned connection will remain in the + FIN_WAIT_2 state before it's aborted at the local + end. Valid values are 5-120 (inclusive). Maps to + net.ipv4.tcp_fin_timeout. + maximum: 120 + minimum: 5 + type: integer + netIpv4TCPKeepaliveProbes: + description: NetIpv4TCPKeepaliveProbes specifies the + number of keepalive probes TCP sends out, until + it decides the connection is broken. Valid values + are 1-15 (inclusive). Maps to net.ipv4.tcp_keepalive_probes. + maximum: 15 + minimum: 1 + type: integer + netIpv4TCPKeepaliveTime: + description: NetIpv4TCPKeepaliveTime specifies the + rate at which TCP sends out a keepalive message + when keepalive is enabled. Valid values are 30-432000 + (inclusive). Maps to net.ipv4.tcp_keepalive_time. + maximum: 432000 + minimum: 30 + type: integer + netIpv4TCPMaxSynBacklog: + description: NetIpv4TCPMaxSynBacklog specifies the + maximum number of queued connection requests that + have still not received an acknowledgment from the + connecting client. If this number is exceeded, the + kernel will begin dropping requests. Valid values + are 128-3240000 (inclusive). Maps to net.ipv4.tcp_max_syn_backlog. + maximum: 3240000 + minimum: 128 + type: integer + netIpv4TCPMaxTwBuckets: + description: NetIpv4TCPMaxTwBuckets specifies maximal + number of timewait sockets held by system simultaneously. + If this number is exceeded, time-wait socket is + immediately destroyed and warning is printed. Valid + values are 8000-1440000 (inclusive). Maps to net.ipv4.tcp_max_tw_buckets. + maximum: 1440000 + minimum: 8000 + type: integer + netIpv4TCPTwReuse: + description: NetIpv4TCPTwReuse is used to allow to + reuse TIME-WAIT sockets for new connections when + it's safe from protocol viewpoint. Maps to net.ipv4.tcp_tw_reuse. + type: boolean + netIpv4TCPkeepaliveIntvl: + description: NetIpv4TCPkeepaliveIntvl specifies the + frequency of the probes sent out. Multiplied by + tcpKeepaliveprobes, it makes up the time to kill + a connection that isn't responding, after probes + started. Valid values are 1-75 (inclusive). Maps + to net.ipv4.tcp_keepalive_intvl. + maximum: 75 + minimum: 1 + type: integer + netNetfilterNfConntrackBuckets: + description: NetNetfilterNfConntrackBuckets specifies + the size of hash table used by nf_conntrack module + to record the established connection record of the + TCP protocol. Valid values are 65536-147456 (inclusive). + Maps to net.netfilter.nf_conntrack_buckets. + maximum: 147456 + minimum: 65536 + type: integer + netNetfilterNfConntrackMax: + description: NetNetfilterNfConntrackMax specifies + the maximum number of connections supported by the + nf_conntrack module or the size of connection tracking + table. Valid values are 131072-1048576 (inclusive). + Maps to net.netfilter.nf_conntrack_max. + maximum: 1048576 + minimum: 131072 + type: integer + vmMaxMapCount: + description: VMMaxMapCount specifies the maximum number + of memory map areas a process may have. Maps to + vm.max_map_count. Valid values are 65530-262144 + (inclusive). + maximum: 262144 + minimum: 65530 + type: integer + vmSwappiness: + description: VMSwappiness specifies aggressiveness + of the kernel in swapping memory pages. Higher values + will increase aggressiveness, lower values decrease + the amount of swap. Valid values are 0-100 (inclusive). + Maps to vm.swappiness. + maximum: 100 + minimum: 0 + type: integer + vmVfsCachePressure: + description: VMVfsCachePressure specifies the percentage + value that controls tendency of the kernel to reclaim + the memory, which is used for caching of directory + and inode objects. Valid values are 1-500 (inclusive). + Maps to vm.vfs_cache_pressure. + maximum: 500 + minimum: 1 + type: integer + type: object + transparentHugePageDefrag: + description: "TransparentHugePageDefrag specifies whether + the kernel should make aggressive use of memory compaction + to make more hugepages available. See also [Linux doc]. + \n [Linux doc]: https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge + for more details." + enum: + - always + - defer + - defer+madvise + - madvise + - never + type: string + transparentHugePageEnabled: + description: "TransparentHugePageEnabled specifies various + modes of Transparent Hugepages. See also [Linux doc]. + \n [Linux doc]: https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge + for more details." + enum: + - always + - madvise + - never + type: string + type: object + maxPods: + description: "MaxPods specifies the kubelet `--max-pods` configuration + for the node pool. Immutable. See also [AKS doc], [K8s doc]. + \n [AKS doc]: https://learn.microsoft.com/azure/aks/configure-azure-cni#configure-maximum---new-clusters + [K8s doc]: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/" + type: integer + mode: + description: 'Mode represents the mode of an agent pool. Possible + values include: System, User.' + enum: + - System + - User + type: string + name: + description: Name is the name of the agent pool. If not specified, + CAPZ uses the name of the CR as the agent pool name. Immutable. + type: string + nodeLabels: + additionalProperties: + type: string + description: "Node labels represent the labels for all of + the nodes present in node pool. See also [AKS doc]. \n [AKS + doc]: https://learn.microsoft.com/azure/aks/use-labels" + type: object + nodePublicIPPrefixID: + description: NodePublicIPPrefixID specifies the public IP + prefix resource ID which VM nodes should use IPs from. Immutable. + type: string + osDiskSizeGB: + description: OSDiskSizeGB is the disk size for every machine + in this agent pool. If you specify 0, it will apply the + default osDisk size according to the vmSize specified. Immutable. + type: integer + osDiskType: + default: Managed + description: "OsDiskType specifies the OS disk type for each + node in the pool. Allowed values are 'Ephemeral' and 'Managed' + (default). Immutable. See also [AKS doc]. \n [AKS doc]: + https://learn.microsoft.com/azure/aks/cluster-configuration#ephemeral-os" + enum: + - Ephemeral + - Managed + type: string + osType: + description: "OSType specifies the virtual machine operating + system. Default to Linux. Possible values include: 'Linux', + 'Windows'. 'Windows' requires the AzureManagedControlPlane's + `spec.networkPlugin` to be `azure`. Immutable. See also + [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/rest/api/aks/agent-pools/create-or-update?tabs=HTTP#ostype" + enum: + - Linux + - Windows + type: string + scaleDownMode: + default: Delete + description: 'ScaleDownMode affects the cluster autoscaler + behavior. Default to Delete. Possible values include: ''Deallocate'', + ''Delete''' + enum: + - Deallocate + - Delete + type: string + scaleSetPriority: + description: 'ScaleSetPriority specifies the ScaleSetPriority + value. Default to Regular. Possible values include: ''Regular'', + ''Spot'' Immutable.' + enum: + - Regular + - Spot + type: string + scaling: + description: Scaling specifies the autoscaling parameters + for the node pool. + properties: + maxSize: + description: MaxSize is the maximum number of nodes for + auto-scaling. + type: integer + minSize: + description: MinSize is the minimum number of nodes for + auto-scaling. + type: integer + type: object + sku: + description: SKU is the size of the VMs in the node pool. + Immutable. + type: string + spotMaxPrice: + anyOf: + - type: integer + - type: string + description: SpotMaxPrice defines max price to pay for spot + instance. Possible values are any decimal value greater + than zero or -1. If you set the max price to be -1, the + VM won't be evicted based on price. The price for the VM + will be the current price for spot or the price for a standard + VM, which ever is less, as long as there's capacity and + quota available. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + subnetName: + description: SubnetName specifies the Subnet where the MachinePool + will be placed Immutable. + type: string + taints: + description: "Taints specifies the taints for nodes present + in this agent pool. See also [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/azure/aks/use-multiple-node-pools#setting-node-pool-taints" + items: + description: Taint represents a Kubernetes taint. + properties: + effect: + description: Effect specifies the effect for the taint + enum: + - NoSchedule + - NoExecute + - PreferNoSchedule + type: string + key: + description: Key is the key of the taint + type: string + value: + description: Value is the value of the taint + type: string + required: + - effect + - key + - value + type: object + type: array + required: + - mode + - sku + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index f41ea1f58f7..bfb526fd42c 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -15,6 +15,9 @@ resources: - bases/infrastructure.cluster.x-k8s.io_azuremanagedclusters.yaml - bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml - bases/infrastructure.cluster.x-k8s.io_azuremachinepoolmachines.yaml + - bases/infrastructure.cluster.x-k8s.io_azuremanagedclustertemplates.yaml + - bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml + - bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepooltemplates.yaml # +kubebuilder:scaffold:crdkustomizeresource diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 90a8fe4a8c8..a2c4a423bd9 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -114,6 +114,27 @@ webhooks: resources: - azuremanagedcontrolplanes sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplanetemplate + failurePolicy: Fail + name: default.azuremanagedcontrolplanetemplates.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedcontrolplanetemplates + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 @@ -136,6 +157,27 @@ webhooks: resources: - azuremanagedmachinepools sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedmachinepooltemplate + failurePolicy: Fail + name: default.azuremanagedmachinepooltemplates.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedmachinepooltemplates + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 @@ -295,6 +337,26 @@ webhooks: resources: - azuremanagedclusters sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedclustertemplate + failurePolicy: Fail + name: validation.azuremanagedclustertemplates.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - UPDATE + resources: + - azuremanagedclustertemplates + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 @@ -316,6 +378,27 @@ webhooks: resources: - azuremanagedcontrolplanes sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplanetemplate + failurePolicy: Fail + name: validation.azuremanagedcontrolplanetemplates.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedcontrolplanetemplates + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 @@ -339,6 +422,28 @@ webhooks: resources: - azuremanagedmachinepools sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedmachinepooltemplate + failurePolicy: Fail + name: validation.azuremanagedmachinepooltemplates.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - azuremanagedmachinepooltemplates + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 diff --git a/controllers/azuremanagedcontrolplane_controller.go b/controllers/azuremanagedcontrolplane_controller.go index 7b380ddedbe..b37431289be 100644 --- a/controllers/azuremanagedcontrolplane_controller.go +++ b/controllers/azuremanagedcontrolplane_controller.go @@ -266,6 +266,7 @@ func (amcpr *AzureManagedControlPlaneReconciler) reconcileNormal(ctx context.Con // No errors, so mark us ready so the Cluster API Cluster Controller can pull it scope.ControlPlane.Status.Ready = true scope.ControlPlane.Status.Initialized = true + scope.ControlPlane.Status.Version = scope.ControlPlane.Spec.Version log.Info("Successfully reconciled") diff --git a/controllers/azuremanagedcontrolplane_controller_test.go b/controllers/azuremanagedcontrolplane_controller_test.go index cb799f050c6..79d18fe8849 100644 --- a/controllers/azuremanagedcontrolplane_controller_test.go +++ b/controllers/azuremanagedcontrolplane_controller_test.go @@ -142,7 +142,9 @@ func TestAzureManagedControlPlaneReconcilePaused(t *testing.T) { }, }, Spec: infrav1.AzureManagedControlPlaneSpec{ - SubscriptionID: "something", + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "something", + }, ResourceGroupName: name, }, } diff --git a/controllers/helpers.go b/controllers/helpers.go index dbe23ac9aa0..49746b8b481 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -955,6 +955,10 @@ func MachinePoolToAzureManagedControlPlaneMapFunc(ctx context.Context, c client. gk := gvk.GroupKind() ref := cluster.Spec.ControlPlaneRef + if ref == nil || ref.Name == "" { + log.Info("control plane ref is nil or empty: control plane ref not found") + return nil + } // Return early if the GroupKind doesn't match what we expect. controlPlaneGK := ref.GroupVersionKind().GroupKind() if gk != controlPlaneGK { diff --git a/controllers/helpers_test.go b/controllers/helpers_test.go index 7bc0a509a91..bb63b5a508e 100644 --- a/controllers/helpers_test.go +++ b/controllers/helpers_test.go @@ -1189,16 +1189,18 @@ func newAzureManagedMachinePool(clusterName, poolName, mode string) *infrav1.Azu Namespace: "default", }, Spec: infrav1.AzureManagedMachinePoolSpec{ - Mode: mode, - SKU: "Standard_B2s", - OSDiskSizeGB: ptr.To(512), - KubeletConfig: &infrav1.KubeletConfig{ - CPUManagerPolicy: &cpuManagerPolicyStatic, - TopologyManagerPolicy: &topologyManagerPolicy, - }, - LinuxOSConfig: &infrav1.LinuxOSConfig{ - TransparentHugePageDefrag: &transparentHugePageDefragMAdvise, - TransparentHugePageEnabled: &transparentHugePageEnabledAlways, + AzureManagedMachinePoolClassSpec: infrav1.AzureManagedMachinePoolClassSpec{ + Mode: mode, + SKU: "Standard_B2s", + OSDiskSizeGB: ptr.To(512), + KubeletConfig: &infrav1.KubeletConfig{ + CPUManagerPolicy: &cpuManagerPolicyStatic, + TopologyManagerPolicy: &topologyManagerPolicy, + }, + LinuxOSConfig: &infrav1.LinuxOSConfig{ + TransparentHugePageDefrag: &transparentHugePageDefragMAdvise, + TransparentHugePageEnabled: &transparentHugePageEnabledAlways, + }, }, }, } diff --git a/docs/book/src/topics/clusterclass.md b/docs/book/src/topics/clusterclass.md new file mode 100644 index 00000000000..0d52147341d --- /dev/null +++ b/docs/book/src/topics/clusterclass.md @@ -0,0 +1,79 @@ +# ClusterClass + +- **Feature status:** Experimental +- **Feature gate:** `ClusterTopology=true` + +[ClusterClass](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/index.html) is a collection of templates that define a topology (control plane and machine deployments) to be used to continuously reconcile one or more Clusters. It is built on top of the existing Cluster API resources and provides a set of tools and operations to streamline cluster lifecycle management while maintaining the same underlying API. + +CAPZ currently supports ClusterClass for both managed (AKS) and self-managed clusters. CAPZ implements this with four custom resources: +1. AzureClusterTemplate +2. AzureManagedClusterTemplate +3. AzureManagedControlPlaneTemplate +4. AzureManagedMachinePoolTemplate + +Each resource is a template for the corresponding CAPZ resource. For example, the AzureClusterTemplate is a template for the CAPZ AzureCluster resource. The template contains a set of parameters that are able to be shared across multiple clusters. + +## Deploying a Self-Managed Cluster with ClusterClass + +Users must first create a ClusterClass resource to deploy a self-managed cluster with ClusterClass. The ClusterClass resource defines the cluster topology, including the control plane and machine deployment templates. The ClusterClass resource also defines the parameters that can be used to customize the cluster topology. + +Please refer to the Cluster API book for more information on how to write a ClusterClass topology: https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/write-clusterclass.html + +For a self-managed cluster, the AzureClusterTemplate defines the Azure infrastructure for the cluster. The following example shows a basic AzureClusterTemplate resource: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterTemplate +metadata: + name: capz-clusterclass-cluster + namespace: default +spec: + template: + spec: + location: westus2 + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + natGateway: + name: node-natgateway + role: node + subscriptionID: 00000000-0000-0000-0000-000000000000 +``` + +## Deploying a Managed Cluster (AKS) with ClusterClass + +**Feature gate:** `MachinePool=true` + +Deploying an AKS cluster with ClusterClass is similar to deploying a self-managed cluster. However, both an AzureManagedClusterTemplate and AzureManagedControlPlaneTemplate must be used instead of the AzureClusterTemplate. Due to the nature of managed Kubernetes and the control plane implementation, the infrastructure provider (and therefore the AzureManagedClusterTemplate) for AKS cluster is basically a no-op. The AzureManagedControlPlaneTemplate is used to define the AKS cluster configuration, such as the Kubernetes version and the number of nodes. Finally, the AzureManagedMachinePoolTemplate defines the worker nodes (agentpools) for the AKS cluster. + +The following example shows a basic AzureManagedClusterTemplate, AzureManagedControlPlaneTemplate, and AzureManagedMachinePoolTemplate resource: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedClusterTemplate +metadata: + name: capz-clusterclass-cluster +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: capz-clusterclass-control-plane +spec: + location: westus2 + subscriptionID: 00000000-0000-0000-0000-000000000000 + version: 1.25.2 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: capz-clusterclass-pool0 + namespace: default +spec: + template: + spec: + mode: System + name: pool0 + sku: Standard_D2s_v3 +``` diff --git a/main.go b/main.go index 9e1c3312bc2..3145e68609a 100644 --- a/main.go +++ b/main.go @@ -530,6 +530,11 @@ func registerWebhooks(mgr manager.Manager) { os.Exit(1) } + if err := (&infrav1.AzureManagedClusterTemplate{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AzureManagedClusterTemplate") + os.Exit(1) + } + if err := infrav1exp.SetupAzureMachinePoolWebhookWithManager(mgr); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "AzureMachinePool") os.Exit(1) @@ -545,11 +550,21 @@ func registerWebhooks(mgr manager.Manager) { os.Exit(1) } + if err := infrav1.SetupAzureManagedMachinePoolTemplateWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AzureManagedMachinePoolTemplate") + os.Exit(1) + } + if err := infrav1.SetupAzureManagedControlPlaneWebhookWithManager(mgr); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "AzureManagedControlPlane") os.Exit(1) } + if err := infrav1.SetupAzureManagedControlPlaneTemplateWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AzureManagedControlPlaneTemplate") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { setupLog.Error(err, "unable to create ready check") os.Exit(1) diff --git a/templates/cluster-template-aks-clusterclass.yaml b/templates/cluster-template-aks-clusterclass.yaml new file mode 100644 index 00000000000..13b7e1e0374 --- /dev/null +++ b/templates/cluster-template-aks-clusterclass.yaml @@ -0,0 +1,125 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: ${CLUSTER_CLASS_NAME} + namespace: default +spec: + controlPlane: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedControlPlaneTemplate + name: ${CLUSTER_NAME}-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedClusterTemplate + name: ${CLUSTER_NAME} + workers: + machinePools: + - class: default-system + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool0 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool0 + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool1 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool1 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedClusterTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + template: + spec: {} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: + mode: System + name: pool0 + sku: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: + mode: User + name: pool1 + sku: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: {} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: {} diff --git a/templates/cluster-template-aks-topology.yaml b/templates/cluster-template-aks-topology.yaml new file mode 100644 index 00000000000..c78efb4b389 --- /dev/null +++ b/templates/cluster-template-aks-topology.yaml @@ -0,0 +1,21 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: ${CLUSTER_CLASS_NAME} + version: ${KUBERNETES_VERSION} + workers: + machinePools: + - class: default-system + name: mp-0 + replicas: 1 + - class: default-worker + name: mp-1 + replicas: 1 diff --git a/templates/cluster-template-clusterclass.yaml b/templates/cluster-template-clusterclass.yaml index 7c9c39da4a2..c68df586f34 100644 --- a/templates/cluster-template-clusterclass.yaml +++ b/templates/cluster-template-clusterclass.yaml @@ -79,30 +79,6 @@ spec: kind: AzureMachineTemplate name: ${CLUSTER_NAME}-md-0 --- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - containerd-logger: enabled - csi-proxy: enabled - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - topology: - class: ${CLUSTER_CLASS_NAME} - controlPlane: - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} - workers: - machineDeployments: - - class: ${CLUSTER_NAME}-worker - name: md-0 - replicas: ${WORKER_MACHINE_COUNT} ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterTemplate metadata: diff --git a/templates/cluster-template-topology.yaml b/templates/cluster-template-topology.yaml new file mode 100644 index 00000000000..28ad70f5a82 --- /dev/null +++ b/templates/cluster-template-topology.yaml @@ -0,0 +1,23 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + containerd-logger: enabled + csi-proxy: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: ${CLUSTER_CLASS_NAME} + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} + workers: + machineDeployments: + - class: ${CLUSTER_NAME}-worker + name: md-0 + replicas: ${WORKER_MACHINE_COUNT} diff --git a/templates/flavors/aks-clusterclass/azure-managed-cluster-template.yaml b/templates/flavors/aks-clusterclass/azure-managed-cluster-template.yaml new file mode 100644 index 00000000000..664e7df5eb2 --- /dev/null +++ b/templates/flavors/aks-clusterclass/azure-managed-cluster-template.yaml @@ -0,0 +1,8 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedClusterTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + template: + spec: {} diff --git a/templates/flavors/aks-clusterclass/azure-managed-controlplane-template.yaml b/templates/flavors/aks-clusterclass/azure-managed-controlplane-template.yaml new file mode 100644 index 00000000000..12e31a5c904 --- /dev/null +++ b/templates/flavors/aks-clusterclass/azure-managed-controlplane-template.yaml @@ -0,0 +1,15 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} + version: ${KUBERNETES_VERSION} diff --git a/templates/flavors/aks-clusterclass/azure-managed-machinepool-template.yaml b/templates/flavors/aks-clusterclass/azure-managed-machinepool-template.yaml new file mode 100644 index 00000000000..647bc8524da --- /dev/null +++ b/templates/flavors/aks-clusterclass/azure-managed-machinepool-template.yaml @@ -0,0 +1,23 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: + mode: System + name: pool0 + sku: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: + mode: User + name: pool1 + sku: ${AZURE_NODE_MACHINE_TYPE} diff --git a/templates/flavors/aks-clusterclass/clusterclass.yaml b/templates/flavors/aks-clusterclass/clusterclass.yaml new file mode 100644 index 00000000000..f20813ac9c3 --- /dev/null +++ b/templates/flavors/aks-clusterclass/clusterclass.yaml @@ -0,0 +1,43 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: ${CLUSTER_CLASS_NAME} + namespace: default +spec: + controlPlane: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedControlPlaneTemplate + name: ${CLUSTER_NAME}-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedClusterTemplate + name: ${CLUSTER_NAME} + workers: + machinePools: + - class: default-system + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool0 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool0 + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool1 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool1 + \ No newline at end of file diff --git a/templates/flavors/aks-clusterclass/kubeadm-config-template.yaml b/templates/flavors/aks-clusterclass/kubeadm-config-template.yaml new file mode 100644 index 00000000000..0b179fa7354 --- /dev/null +++ b/templates/flavors/aks-clusterclass/kubeadm-config-template.yaml @@ -0,0 +1,17 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: {} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: {} diff --git a/templates/flavors/aks-clusterclass/kustomization.yaml b/templates/flavors/aks-clusterclass/kustomization.yaml new file mode 100644 index 00000000000..98083341a84 --- /dev/null +++ b/templates/flavors/aks-clusterclass/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- clusterclass.yaml +- azure-managed-controlplane-template.yaml +- azure-managed-cluster-template.yaml +- azure-managed-machinepool-template.yaml +- ../../azure-cluster-identity +- kubeadm-config-template.yaml diff --git a/templates/flavors/aks-clusterclass/patches/managedazurecluster-identity-ref.yaml b/templates/flavors/aks-clusterclass/patches/managedazurecluster-identity-ref.yaml new file mode 100644 index 00000000000..37d7ac29191 --- /dev/null +++ b/templates/flavors/aks-clusterclass/patches/managedazurecluster-identity-ref.yaml @@ -0,0 +1,9 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: "${CLUSTER_IDENTITY_NAME}" \ No newline at end of file diff --git a/templates/flavors/aks-topology/cluster.yaml b/templates/flavors/aks-topology/cluster.yaml new file mode 100644 index 00000000000..c78efb4b389 --- /dev/null +++ b/templates/flavors/aks-topology/cluster.yaml @@ -0,0 +1,21 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: ${CLUSTER_CLASS_NAME} + version: ${KUBERNETES_VERSION} + workers: + machinePools: + - class: default-system + name: mp-0 + replicas: 1 + - class: default-worker + name: mp-1 + replicas: 1 diff --git a/templates/flavors/aks-topology/kustomization.yaml b/templates/flavors/aks-topology/kustomization.yaml new file mode 100644 index 00000000000..7a5648beb9c --- /dev/null +++ b/templates/flavors/aks-topology/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- cluster.yaml diff --git a/templates/flavors/clusterclass/kustomization.yaml b/templates/flavors/clusterclass/kustomization.yaml index d32392fc71f..30b4276d555 100644 --- a/templates/flavors/clusterclass/kustomization.yaml +++ b/templates/flavors/clusterclass/kustomization.yaml @@ -1,7 +1,6 @@ namespace: default resources: - clusterclass.yaml - - cluster.yaml - azure-cluster-template.yaml - azure-machine-template-controlplane.yaml - azure-machine-template-worker.yaml diff --git a/templates/flavors/clusterclass/cluster.yaml b/templates/flavors/topology/cluster.yaml similarity index 100% rename from templates/flavors/clusterclass/cluster.yaml rename to templates/flavors/topology/cluster.yaml diff --git a/templates/flavors/topology/kustomization.yaml b/templates/flavors/topology/kustomization.yaml new file mode 100644 index 00000000000..fd7811e1145 --- /dev/null +++ b/templates/flavors/topology/kustomization.yaml @@ -0,0 +1,3 @@ +namespace: default +resources: +- cluster.yaml \ No newline at end of file diff --git a/templates/test/ci/cluster-template-prow-aks-clusterclass.yaml b/templates/test/ci/cluster-template-prow-aks-clusterclass.yaml new file mode 100644 index 00000000000..34da32fe61a --- /dev/null +++ b/templates/test/ci/cluster-template-prow-aks-clusterclass.yaml @@ -0,0 +1,273 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: ${CLUSTER_CLASS_NAME} + namespace: default +spec: + controlPlane: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedControlPlaneTemplate + name: ${CLUSTER_NAME}-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedClusterTemplate + name: ${CLUSTER_NAME} + patches: + - definitions: + - jsonPatches: + - op: replace + path: /spec/template/spec/files + valueFrom: + template: | + - contentFrom: + secret: + key: worker-node-azure.json + name: "{{ .builtin.machinePool.infrastructureRef.name }}-azure-json" + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machinePoolClass: + names: + - default-system + - default-worker + name: workerAzureJsonSecretName + workers: + machinePools: + - class: default-system + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool0 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool0 + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool1 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool1 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + addonProfiles: + - enabled: true + name: azurepolicy + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedClusterTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + template: + spec: {} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: + availabilityZones: + - "1" + - "2" + enableNodePublicIP: false + enableUltraSSD: true + maxPods: 30 + mode: System + name: pool0 + osDiskSizeGB: 30 + osDiskType: Managed + sku: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: + enableNodePublicIP: false + kubeletConfig: + allowedUnsafeSysctls: + - net.* + - kernel.msg* + containerLogMaxFiles: 50 + containerLogMaxSizeMB: 500 + cpuCfsQuota: true + cpuCfsQuotaPeriod: 110ms + cpuManagerPolicy: static + failSwapOn: false + imageGcHighThreshold: 70 + imageGcLowThreshold: 50 + podMaxPids: 2048 + topologyManagerPolicy: best-effort + linuxOSConfig: + swapFileSizeMB: 1500 + sysctls: + fsAioMaxNr: 65536 + fsFileMax: 709620 + fsInotifyMaxUserWatches: 1048576 + fsNrOpen: 1048576 + kernelThreadsMax: 55601 + netCoreNetdevMaxBacklog: 1000 + netCoreOptmemMax: 20480 + netCoreRmemDefault: 212992 + netCoreRmemMax: 212992 + netCoreSomaxconn: 16384 + netCoreWmemDefault: 212992 + netCoreWmemMax: 212992 + netIpv4IPLocalPortRange: 32768 60999 + netIpv4NeighDefaultGcThresh1: 4096 + netIpv4NeighDefaultGcThresh2: 8192 + netIpv4NeighDefaultGcThresh3: 16384 + netIpv4TCPFinTimeout: 60 + netIpv4TCPKeepaliveProbes: 9 + netIpv4TCPKeepaliveTime: 7200 + netIpv4TCPMaxSynBacklog: 16384 + netIpv4TCPMaxTwBuckets: 32768 + netIpv4TCPTwReuse: false + netIpv4TCPkeepaliveIntvl: 75 + netNetfilterNfConntrackBuckets: 65536 + netNetfilterNfConntrackMax: 131072 + vmMaxMapCount: 65530 + vmSwappiness: 60 + vmVfsCachePressure: 100 + transparentHugePageDefrag: madvise + transparentHugePageEnabled: always + maxPods: 64 + mode: User + name: pool1 + nodeLabels: + type: shared + osDiskSizeGB: 40 + osDiskType: Ephemeral + scaleSetPriority: Regular + sku: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3} + taints: + - effect: NoSchedule + key: type + value: shared +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: default + version: ${KUBERNETES_VERSION} + workers: + machinePools: + - class: default-system + name: mp-0 + replicas: 1 + - class: default-worker + name: mp-1 + replicas: 1 diff --git a/templates/test/ci/prow-aks-clusterclass/kustomization.yaml b/templates/test/ci/prow-aks-clusterclass/kustomization.yaml new file mode 100644 index 00000000000..6a5911e30a9 --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - ../../../flavors/aks-clusterclass + - ../../../flavors/aks-topology +patchesStrategicMerge: + - patches/tags-aks-clusterclass.yaml + - patches/aks-clusterclass-pool0.yaml + - patches/aks-clusterclass-pool1.yaml + - patches/cluster.yaml + - patches/addons.yaml + - patches/kubeadm-config-template.yaml + - patches.yaml diff --git a/templates/test/ci/prow-aks-clusterclass/patches.yaml b/templates/test/ci/prow-aks-clusterclass/patches.yaml new file mode 100644 index 00000000000..74d7e896aa7 --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches.yaml @@ -0,0 +1,54 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: ${CLUSTER_CLASS_NAME} +spec: + workers: + machinePools: + - class: default-system + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool0 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool0 + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool1 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool1 + patches: + - name: workerAzureJsonSecretName + definitions: + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machinePoolClass: + names: + - default-system + - default-worker + jsonPatches: + - op: replace + path: "/spec/template/spec/files" + valueFrom: + template: | + - contentFrom: + secret: + key: worker-node-azure.json + name: "{{ .builtin.machinePool.infrastructureRef.name }}-azure-json" + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" diff --git a/templates/test/ci/prow-aks-clusterclass/patches/addons.yaml b/templates/test/ci/prow-aks-clusterclass/patches/addons.yaml new file mode 100644 index 00000000000..51d2e93ca8a --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/addons.yaml @@ -0,0 +1,11 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + addonProfiles: + - enabled: true + name: azurepolicy diff --git a/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool0.yaml b/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool0.yaml new file mode 100644 index 00000000000..0c7b26b42bf --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool0.yaml @@ -0,0 +1,15 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: "${CLUSTER_NAME}-pool0" +spec: + template: + spec: + maxPods: 30 + osDiskType: "Managed" + osDiskSizeGB: 30 + enableNodePublicIP: false + enableUltraSSD: true + availabilityZones: ["1", "2"] + name: pool0 + sku: "${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3}" diff --git a/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool1.yaml b/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool1.yaml new file mode 100644 index 00000000000..a7e69ba69a2 --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool1.yaml @@ -0,0 +1,67 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: "${CLUSTER_NAME}-pool1" +spec: + template: + spec: + maxPods: 64 + osDiskType: "Ephemeral" + osDiskSizeGB: 40 + enableNodePublicIP: false + scaleSetPriority: Regular + taints: + - effect: NoSchedule + key: type + value: shared + nodeLabels: + "type": "shared" + name: pool1 + sku: "${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3}" + kubeletConfig: + cpuManagerPolicy: "static" + cpuCfsQuota: true + cpuCfsQuotaPeriod: "110ms" + imageGcHighThreshold: 70 + imageGcLowThreshold: 50 + topologyManagerPolicy: "best-effort" + allowedUnsafeSysctls: + - "net.*" + - "kernel.msg*" + failSwapOn: false + containerLogMaxSizeMB: 500 + containerLogMaxFiles: 50 + podMaxPids: 2048 + linuxOSConfig: + swapFileSizeMB: 1500 + sysctls: + fsAioMaxNr: 65536 + fsFileMax: 709620 + fsInotifyMaxUserWatches: 1048576 + fsNrOpen: 1048576 + kernelThreadsMax: 55601 + netCoreNetdevMaxBacklog: 1000 + netCoreOptmemMax: 20480 + netCoreRmemDefault: 212992 + netCoreRmemMax: 212992 + netCoreSomaxconn: 16384 + netCoreWmemDefault: 212992 + netCoreWmemMax: 212992 + netIpv4IPLocalPortRange: "32768 60999" + netIpv4NeighDefaultGcThresh1: 4096 + netIpv4NeighDefaultGcThresh2: 8192 + netIpv4NeighDefaultGcThresh3: 16384 + netIpv4TCPFinTimeout: 60 + netIpv4TCPKeepaliveProbes: 9 + netIpv4TCPKeepaliveTime: 7200 + netIpv4TCPMaxSynBacklog: 16384 + netIpv4TCPMaxTwBuckets: 32768 + netIpv4TCPTwReuse: false + netIpv4TCPkeepaliveIntvl: 75 + netNetfilterNfConntrackBuckets: 65536 + netNetfilterNfConntrackMax: 131072 + vmMaxMapCount: 65530 + vmSwappiness: 60 + vmVfsCachePressure: 100 + transparentHugePageDefrag: "madvise" + transparentHugePageEnabled: "always" diff --git a/templates/test/ci/prow-aks-clusterclass/patches/cluster.yaml b/templates/test/ci/prow-aks-clusterclass/patches/cluster.yaml new file mode 100644 index 00000000000..a1a4b8f8e39 --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/cluster.yaml @@ -0,0 +1,7 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + topology: + class: default diff --git a/templates/test/ci/prow-aks-clusterclass/patches/kubeadm-config-template.yaml b/templates/test/ci/prow-aks-clusterclass/patches/kubeadm-config-template.yaml new file mode 100644 index 00000000000..d5b21ef8c3b --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/kubeadm-config-template.yaml @@ -0,0 +1,45 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] diff --git a/templates/test/ci/prow-aks-clusterclass/patches/tags-aks-clusterclass.yaml b/templates/test/ci/prow-aks-clusterclass/patches/tags-aks-clusterclass.yaml new file mode 100644 index 00000000000..9a6f76edb6f --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/tags-aks-clusterclass.yaml @@ -0,0 +1,12 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + additionalTags: + jobName: ${JOB_NAME} + creationTimestamp: ${TIMESTAMP} + buildProvenance: ${BUILD_PROVENANCE} \ No newline at end of file diff --git a/templates/test/ci/prow-topology/kustomization.yaml b/templates/test/ci/prow-topology/kustomization.yaml index da93db782a5..a5a4ebcc189 100644 --- a/templates/test/ci/prow-topology/kustomization.yaml +++ b/templates/test/ci/prow-topology/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: - - ../../../flavors/clusterclass/cluster.yaml + - ../../../flavors/topology/cluster.yaml - cni-resource-set.yaml - ../../../addons/windows/csi-proxy/csi-proxy-resource-set.yaml - ../../../addons/cluster-api-helm/calico.yaml diff --git a/test/e2e/aks.go b/test/e2e/aks.go index 6ba7aa3c966..c08247cc243 100644 --- a/test/e2e/aks.go +++ b/test/e2e/aks.go @@ -55,7 +55,9 @@ func WaitForAKSControlPlaneInitialized(ctx context.Context, input clusterctl.App Getter: client, Cluster: result.Cluster, }, input.WaitForControlPlaneIntervals...) - InstallCNIManifest(ctx, input, cluster.Spec.ClusterNetwork.Services.CIDRBlocks, true) + if cluster.Spec.ClusterNetwork != nil && cluster.Spec.ClusterNetwork.Services != nil { + InstallCNIManifest(ctx, input, cluster.Spec.ClusterNetwork.Services.CIDRBlocks, true) + } } // WaitForAKSControlPlaneReady waits for the azure managed control plane to be ready. diff --git a/test/e2e/aks_clusterclass.go b/test/e2e/aks_clusterclass.go new file mode 100644 index 00000000000..9fc939cacf2 --- /dev/null +++ b/test/e2e/aks_clusterclass.go @@ -0,0 +1,127 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" +) + +type AKSClusterClassInput struct { + Cluster *clusterv1.Cluster + MachinePool *expv1.MachinePool + WaitIntervals []interface{} + WaitUpgradeIntervals []interface{} + KubernetesVersionUpgradeTo string +} + +func AKSClusterClassSpec(ctx context.Context, inputGetter func() AKSClusterClassInput) { + input := inputGetter() + + cred, err := azidentity.NewDefaultAzureCredential(nil) + Expect(err).NotTo(HaveOccurred()) + + managedClustersClient, err := armcontainerservice.NewManagedClustersClient(getSubscriptionID(Default), cred, nil) + Expect(err).NotTo(HaveOccurred()) + + agentPoolsClient, err := armcontainerservice.NewAgentPoolsClient(getSubscriptionID(Default), cred, nil) + Expect(err).NotTo(HaveOccurred()) + + mgmtClient := bootstrapClusterProxy.GetClient() + Expect(mgmtClient).NotTo(BeNil()) + + amcp := &infrav1.AzureManagedControlPlane{} + err = mgmtClient.Get(ctx, types.NamespacedName{ + Namespace: input.Cluster.Spec.ControlPlaneRef.Namespace, + Name: input.Cluster.Spec.ControlPlaneRef.Name, + }, amcp) + Expect(err).NotTo(HaveOccurred()) + + By("Editing the AzureManagedMachinePoolTemplate to change the scale down mode") + ammpt := &infrav1.AzureManagedMachinePoolTemplate{} + + // TODO: We are hard-coding the ammpt name suffix since to get the name from + // the ClusterClass, we have to upgrade CAPI to v1.6.0. Fix this once CAPI + // v1.6.0 is released. + // CAPI v1.6.0 Bump PR: https://github.com/kubernetes-sigs/cluster-api-provider-azure/pull/4182 + Eventually(func(g Gomega) { + err = mgmtClient.Get(ctx, types.NamespacedName{ + Namespace: input.Cluster.Namespace, + Name: input.Cluster.Name + "-pool0", + }, ammpt) + Expect(err).NotTo(HaveOccurred()) + ammpt.Spec.Template.Spec.ScaleDownMode = ptr.To("Deallocate") + g.Expect(mgmtClient.Update(ctx, ammpt)).To(Succeed()) + }, inputGetter().WaitIntervals...).Should(Succeed()) + + ammp := &infrav1.AzureManagedMachinePool{} + + Eventually(func(g Gomega) { + err = mgmtClient.Get(ctx, types.NamespacedName{ + Namespace: input.MachinePool.Spec.Template.Spec.InfrastructureRef.Namespace, + Name: input.MachinePool.Spec.Template.Spec.InfrastructureRef.Name, + }, ammp) + Expect(err).NotTo(HaveOccurred()) + g.Expect(ammp.Spec.ScaleDownMode).To(Equal(ptr.To("Deallocate"))) + }, inputGetter().WaitIntervals...).Should(Succeed()) + + Eventually(func(g Gomega) { + resp, err := agentPoolsClient.Get(ctx, amcp.Spec.ResourceGroupName, amcp.Name, *ammp.Spec.Name, nil) + g.Expect(err).NotTo(HaveOccurred()) + agentPool := resp.AgentPool + g.Expect(agentPool.Properties).NotTo(BeNil()) + g.Expect(agentPool.Properties.ScaleDownMode).NotTo(BeNil()) + g.Expect(*agentPool.Properties.ScaleDownMode).To(Equal(armcontainerservice.ScaleDownModeDeallocate)) + }, input.WaitIntervals...).Should(Succeed()) + + By("Upgrading the cluster topology version") + Eventually(func(g Gomega) { + input.Cluster.Spec.Topology.Version = input.KubernetesVersionUpgradeTo + g.Expect(mgmtClient.Update(ctx, input.Cluster)).To(Succeed()) + }, inputGetter().WaitIntervals...).Should(Succeed()) + + Eventually(func(g Gomega) { + resp, err := managedClustersClient.Get(ctx, amcp.Spec.ResourceGroupName, amcp.Name, nil) + g.Expect(err).NotTo(HaveOccurred()) + aksCluster := resp.ManagedCluster + g.Expect(aksCluster.Properties).NotTo(BeNil()) + g.Expect(aksCluster.Properties.KubernetesVersion).NotTo(BeNil()) + g.Expect("v" + *aksCluster.Properties.KubernetesVersion).To(Equal(input.KubernetesVersionUpgradeTo)) + g.Expect(aksCluster.Properties.ProvisioningState).To(Equal(ptr.To("Succeeded"))) + }, input.WaitUpgradeIntervals...).Should(Succeed()) + + By("Ensuring the upgrade is reflected in the amcp") + Eventually(func(g Gomega) { + g.Expect(mgmtClient.Get(ctx, types.NamespacedName{ + Namespace: input.Cluster.Spec.ControlPlaneRef.Namespace, + Name: input.Cluster.Spec.ControlPlaneRef.Name, + }, amcp)).To(Succeed()) + g.Expect(amcp.Spec.Version).To(Equal(input.KubernetesVersionUpgradeTo)) + }, input.WaitIntervals...).Should(Succeed()) +} diff --git a/test/e2e/aks_public_ip_prefix.go b/test/e2e/aks_public_ip_prefix.go index 735539ef37a..2878a275110 100644 --- a/test/e2e/aks_public_ip_prefix.go +++ b/test/e2e/aks_public_ip_prefix.go @@ -32,6 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/util/conditions" @@ -89,10 +90,12 @@ func AKSPublicIPPrefixSpec(ctx context.Context, inputGetter func() AKSPublicIPPr Namespace: input.Cluster.Namespace, }, Spec: infrav1.AzureManagedMachinePoolSpec{ - Mode: "User", - SKU: os.Getenv("AZURE_NODE_MACHINE_TYPE"), - EnableNodePublicIP: ptr.To(true), - NodePublicIPPrefixID: ptr.To("/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroupName + "/providers/Microsoft.Network/publicipprefixes/" + *publicIPPrefix.Name), + AzureManagedMachinePoolClassSpec: infrav1.AzureManagedMachinePoolClassSpec{ + Mode: "User", + SKU: os.Getenv("AZURE_NODE_MACHINE_TYPE"), + EnableNodePublicIP: ptr.To(true), + NodePublicIPPrefixID: ptr.To(azure.PublicIPPrefixID(subscriptionID, resourceGroupName, *publicIPPrefix.Name)), + }, }, } err = mgmtClient.Create(ctx, infraMachinePool) diff --git a/test/e2e/aks_spot.go b/test/e2e/aks_spot.go index 1853f29d11f..5f35e561104 100644 --- a/test/e2e/aks_spot.go +++ b/test/e2e/aks_spot.go @@ -66,12 +66,14 @@ func AKSSpotSpec(ctx context.Context, inputGetter func() AKSSpotSpecInput) { Namespace: input.Cluster.Namespace, }, Spec: infrav1.AzureManagedMachinePoolSpec{ - Mode: "User", - SKU: "Standard_D2s_v3", - ScaleSetPriority: ptr.To(string(armcontainerservice.ScaleSetPrioritySpot)), - Scaling: &scaling, - SpotMaxPrice: &spotMaxPrice, - ScaleDownMode: ptr.To("Deallocate"), + AzureManagedMachinePoolClassSpec: infrav1.AzureManagedMachinePoolClassSpec{ + Mode: "User", + SKU: "Standard_D2s_v3", + ScaleSetPriority: ptr.To(string(armcontainerservice.ScaleSetPrioritySpot)), + Scaling: &scaling, + SpotMaxPrice: &spotMaxPrice, + ScaleDownMode: ptr.To("Deallocate"), + }, }, } err = mgmtClient.Create(ctx, infraMachinePool) diff --git a/test/e2e/aks_upgrade.go b/test/e2e/aks_upgrade.go index c1c129783cc..fc6a12fad0d 100644 --- a/test/e2e/aks_upgrade.go +++ b/test/e2e/aks_upgrade.go @@ -71,6 +71,7 @@ func AKSUpgradeSpec(ctx context.Context, inputGetter func() AKSUpgradeSpecInput) g.Expect(aksCluster.Properties).NotTo(BeNil()) g.Expect(aksCluster.Properties.KubernetesVersion).NotTo(BeNil()) g.Expect("v" + *aksCluster.Properties.KubernetesVersion).To(Equal(input.KubernetesVersionUpgradeTo)) + g.Expect(aksCluster.Properties.ProvisioningState).To(Equal(ptr.To("Succeeded"))) }, input.WaitForControlPlane...).Should(Succeed()) By("Upgrading the machinepool instances") diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index c47996ed31b..95a9a382e45 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -847,6 +847,52 @@ var _ = Describe("Workload cluster creation", func() { }) }) + Context("Creating an AKS cluster using ClusterClass [Managed Kubernetes]", func() { + It("with a single control plane node and 1 node", func() { + // Use default as the clusterclass name so test infra can find the clusterclass template + os.Setenv("CLUSTER_CLASS_NAME", "default") + + // Use "cc" as spec name because NAT gateway pip name exceeds limit. + clusterName = getClusterName(clusterNamePrefix, "cc") + kubernetesVersionUpgradeFrom, err := GetAKSKubernetesVersion(ctx, e2eConfig, AKSKubernetesVersionUpgradeFrom) + Byf("Upgrading from k8s version %s", kubernetesVersionUpgradeFrom) + Expect(err).To(BeNil()) + kubernetesVersion, err := GetAKSKubernetesVersion(ctx, e2eConfig, AKSKubernetesVersion) + Byf("Upgrading to k8s version %s", kubernetesVersion) + Expect(err).To(BeNil()) + + // Create a cluster using the cluster class created above + clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( + specName, + withFlavor("aks-clusterclass"), + withAzureCNIv1Manifest(e2eConfig.GetVariable(AzureCNIv1Manifest)), + withNamespace(namespace.Name), + withClusterName(clusterName), + withKubernetesVersion(kubernetesVersionUpgradeFrom), + withControlPlaneMachineCount(1), + withWorkerMachineCount(1), + withMachineDeploymentInterval(specName, ""), + withMachinePoolInterval(specName, "wait-machine-pool-nodes"), + withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ + WaitForControlPlaneInitialized: WaitForAKSControlPlaneInitialized, + WaitForControlPlaneMachinesReady: WaitForAKSControlPlaneReady, + }), + ), result) + + By("Performing ClusterClass operations on the cluster", func() { + AKSClusterClassSpec(ctx, func() AKSClusterClassInput { + return AKSClusterClassInput{ + Cluster: result.Cluster, + MachinePool: result.MachinePools[0], + WaitIntervals: e2eConfig.GetIntervals(specName, "wait-machine-pool-nodes"), + WaitUpgradeIntervals: e2eConfig.GetIntervals(specName, "wait-machine-pool-upgrade"), + KubernetesVersionUpgradeTo: kubernetesVersion, + } + }) + }) + }) + }) + // ci-e2e.sh and Prow CI skip this test by default. To include this test, set `GINKGO_SKIP=""`. // This spec expects a user-assigned identity named "cloud-provider-user-identity" in a "capz-ci" // resource group. Override these defaults by setting the USER_IDENTITY and CI_RG environment variables. @@ -920,10 +966,10 @@ var _ = Describe("Workload cluster creation", func() { Context("Creating clusters using clusterclass [OPTIONAL]", func() { It("with a single control plane node, one linux worker node, and one windows worker node", func() { - // use ci-default as the clusterclass name so test infra can find the clusterclass template + // Use ci-default as the clusterclass name so test infra can find the clusterclass template os.Setenv("CLUSTER_CLASS_NAME", "ci-default") - // use "cc" as spec name because natgw pip name exceeds limit. + // Use "cc" as spec name because NAT gateway pip name exceeds limit. clusterName = getClusterName(clusterNamePrefix, "cc") // Opt into using windows with prow template diff --git a/test/e2e/common.go b/test/e2e/common.go index e4e9bfedc31..abd8c8bb01b 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -38,6 +38,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" e2e_namespace "sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/namespace" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -289,7 +290,8 @@ func ensureControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCu }, input.WaitForControlPlaneIntervals...).Should(Succeed(), "API Server was not reachable in time") _, hasWindows := cluster.Labels["cni-windows"] - if kubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.ControllerManager.ExtraArgs["cloud-provider"] != "azure" { + + if kubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.ControllerManager.ExtraArgs["cloud-provider"] != infrav1.AzureNetworkPluginName { // There is a co-dependency between cloud-provider and CNI so we install both together if cloud-provider is external. InstallCNIAndCloudProviderAzureHelmChart(ctx, input, installHelmCharts, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) } else { diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 16c6b67ac9e..5e5ef66c0d3 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -3,11 +3,11 @@ managementClusterName: capz-e2e images: - name: ${MANAGER_IMAGE} loadBehavior: mustLoad - - name: registry.k8s.io/cluster-api/cluster-api-controller:v1.5.3 + - name: registry.k8s.io/cluster-api/cluster-api-controller:v1.6.0-beta.1 loadBehavior: tryLoad - - name: registry.k8s.io/cluster-api/kubeadm-bootstrap-controller:v1.5.3 + - name: registry.k8s.io/cluster-api/kubeadm-bootstrap-controller:v1.6.0-beta.1 loadBehavior: tryLoad - - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.5.3 + - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.6.0-beta.1 loadBehavior: tryLoad - name: registry.k8s.io/cluster-api-helm/cluster-api-helm-controller:v0.1.0-alpha.10 loadBehavior: tryLoad @@ -25,8 +25,8 @@ providers: new: --metrics-addr=:8080 files: - sourcePath: "../data/shared/v1beta1/metadata.yaml" - - name: v1.5.3 - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.3/core-components.yaml + - name: v1.6.0-beta.1 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0-beta.1/core-components.yaml type: url contract: v1beta1 files: @@ -48,8 +48,8 @@ providers: new: --metrics-addr=:8080 files: - sourcePath: "../data/shared/v1beta1/metadata.yaml" - - name: v1.5.3 - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.3/bootstrap-components.yaml + - name: v1.6.0-beta.1 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0-beta.1/bootstrap-components.yaml type: url contract: v1beta1 files: @@ -70,8 +70,8 @@ providers: new: --metrics-addr=:8080 files: - sourcePath: "../data/shared/v1beta1/metadata.yaml" - - name: v1.5.3 - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.3/control-plane-components.yaml + - name: v1.6.0-beta.1 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0-beta.1/control-plane-components.yaml type: url contract: v1beta1 files: @@ -132,12 +132,16 @@ providers: targetName: "cluster-template-workload-identity.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks.yaml" targetName: "cluster-template-aks.yaml" + - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks-clusterclass.yaml" + targetName: "cluster-template-aks-clusterclass.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-custom-vnet.yaml" targetName: "cluster-template-custom-vnet.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-dual-stack.yaml" targetName: "cluster-template-dual-stack.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-clusterclass-ci-default.yaml" targetName: "clusterclass-ci-default.yaml" + - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks-clusterclass.yaml" + targetName: "clusterclass-default.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-topology.yaml" targetName: "cluster-template-topology.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-flatcar.yaml" diff --git a/test/e2e/data/shared/v1beta1/metadata.yaml b/test/e2e/data/shared/v1beta1/metadata.yaml index 100fbab0401..23faa295d63 100644 --- a/test/e2e/data/shared/v1beta1/metadata.yaml +++ b/test/e2e/data/shared/v1beta1/metadata.yaml @@ -1,6 +1,9 @@ apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 kind: Metadata releaseSeries: + - major: 1 + minor: 6 + contract: v1beta1 - major: 1 minor: 5 contract: v1beta1