diff --git a/api/v1beta1/azuremanagedclustertemplate_types.go b/api/v1beta1/azuremanagedclustertemplate_types.go new file mode 100644 index 00000000000..84a9dee91ce --- /dev/null +++ b/api/v1beta1/azuremanagedclustertemplate_types.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// AzureManagedClusterTemplateSpec defines the desired state of AzureManagedClusterTemplate. +type AzureManagedClusterTemplateSpec struct { + Template AzureManagedClusterTemplateResource `json:"template"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=azuremanagedclustertemplates,scope=Namespaced,categories=cluster-api +// +kubebuilder:storageversion + +// AzureManagedClusterTemplate is the Schema for the AzureManagedClusterTemplates API. +type AzureManagedClusterTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AzureManagedClusterTemplateSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// AzureManagedClusterTemplateList contains a list of AzureManagedClusterTemplates. +type AzureManagedClusterTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AzureManagedClusterTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&AzureManagedClusterTemplate{}, &AzureManagedClusterTemplateList{}) +} + +// AzureManagedClusterTemplateResource describes the data needed to create an AzureManagedCluster from a template. +type AzureManagedClusterTemplateResource struct { + Spec AzureManagedClusterTemplateResourceSpec `json:"spec"` +} diff --git a/api/v1beta1/azuremanagedclustertemplate_webhook.go b/api/v1beta1/azuremanagedclustertemplate_webhook.go new file mode 100644 index 00000000000..5f823fd1fc8 --- /dev/null +++ b/api/v1beta1/azuremanagedclustertemplate_webhook.go @@ -0,0 +1,77 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "reflect" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/cluster-api-provider-azure/feature" + capifeature "sigs.k8s.io/cluster-api/feature" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +// AzureManagedClusterTemplateImmutableMsg is the message used for errors on fields that are immutable. +const AzureManagedClusterTemplateImmutableMsg = "AzureManagedClusterTemplate spec.template.spec field is immutable. Please create new resource instead. ref doc: https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/change-clusterclass.html" + +// SetupWebhookWithManager sets up and registers the webhook with the manager. +func (r *AzureManagedClusterTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// +kubebuilder:webhook:verbs=update,path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedclustertemplate,mutating=false,failurePolicy=fail,groups=infrastructure.cluster.x-k8s.io,resources=azuremanagedclustertemplates,versions=v1beta1,name=validation.azuremanagedclustertemplates.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 + +var _ webhook.Validator = &AzureManagedClusterTemplate{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (r *AzureManagedClusterTemplate) ValidateCreate() error { + // NOTE: AzureManagedClusterTemplate relies upon MachinePools, which is behind a feature gate flag. + // The webhook must prevent creating new objects in case the feature flag is disabled. + if !feature.Gates.Enabled(capifeature.MachinePool) { + return field.Forbidden( + field.NewPath("spec"), + "can be set only if the Cluster API 'MachinePool' feature flag is enabled", + ) + } + return nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (r *AzureManagedClusterTemplate) ValidateUpdate(oldRaw runtime.Object) error { + var allErrs field.ErrorList + old := oldRaw.(*AzureManagedClusterTemplate) + if !reflect.DeepEqual(r.Spec.Template.Spec, old.Spec.Template.Spec) { + allErrs = append(allErrs, + field.Invalid(field.NewPath("AzureManagedClusterTemplate", "spec", "template", "spec"), rScanInterval, AzureManagedClusterTemplateImmutableMsg), + ) + } + + if len(allErrs) == 0 { + return nil + } + return apierrors.NewInvalid(GroupVersion.WithKind("AzureManagedClusterTemplate").GroupKind(), r.Name, allErrs) +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (r *AzureManagedClusterTemplate) ValidateDelete() error { + return nil +} diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook.go b/api/v1beta1/azuremanagedcontrolplane_webhook.go index 6fc6ed54c95..24bd49527f2 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -18,7 +18,6 @@ package v1beta1 import ( "context" - "errors" "fmt" "net" "reflect" @@ -257,45 +256,65 @@ func (m *AzureManagedControlPlane) ValidateDelete(_ client.Client) error { // Validate the Azure Managed Control Plane and return an aggregate error. func (m *AzureManagedControlPlane) Validate(cli client.Client) error { + var allErrs field.ErrorList validators := []func(client client.Client) error{ - m.validateName, - m.validateVersion, - m.validateDNSServiceIP, m.validateSSHKey, - m.validateLoadBalancerProfile, m.validateAPIServerAccessProfile, - m.validateManagedClusterNetwork, - m.validateAutoScalerProfile, } - var errs []error for _, validator := range validators { if err := validator(cli); err != nil { - errs = append(errs, err) + allErrs = append(allErrs, field.InternalError(field.NewPath("spec"), err)) } } - return kerrors.NewAggregate(errs) + allErrs = append(allErrs, validateDNSServiceIP( + m.Spec.DNSServiceIP, + field.NewPath("spec").Child("DNSServiceIP"))...) + + allErrs = append(allErrs, validateVersion( + m.Spec.Version, + field.NewPath("spec").Child("Version"))...) + + allErrs = append(allErrs, validateLoadBalancerProfile( + m.Spec.LoadBalancerProfile, + field.NewPath("spec").Child("LoadBalancerProfile"))...) + + allErrs = append(allErrs, validateManagedClusterNetwork( + cli, + m.Labels, + m.Namespace, + m.Spec.DNSServiceIP, + m.Spec.VirtualNetwork.Subnet, + field.NewPath("spec").Child("spec"))...) + + allErrs = append(allErrs, validateName(m.Name, field.NewPath("Name"))...) + + allErrs = append(allErrs, validateAutoScalerProfile(m.Spec.AutoScalerProfile, field.NewPath("spec").Child("AutoScalerProfile"))...) + + return allErrs.ToAggregate() } // validateDNSServiceIP validates the DNSServiceIP. -func (m *AzureManagedControlPlane) validateDNSServiceIP(_ client.Client) error { - if m.Spec.DNSServiceIP != nil { - if net.ParseIP(*m.Spec.DNSServiceIP) == nil { - return errors.New("DNSServiceIP must be a valid IP") +func validateDNSServiceIP(dnsServiceIP *string, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + if dnsServiceIP != nil { + if net.ParseIP(*dnsServiceIP) == nil { + allErrs = append(allErrs, field.Invalid(fldPath, dnsServiceIP, "DNSServiceIP must be a valid IP")) } } - return nil + return allErrs } // validateVersion validates the Kubernetes version. -func (m *AzureManagedControlPlane) validateVersion(_ client.Client) error { - if !kubeSemver.MatchString(m.Spec.Version) { - return errors.New("must be a valid semantic version") +func validateVersion(version string, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + if !kubeSemver.MatchString(version) { + allErrs = append(allErrs, field.Invalid(fldPath, version, "must be a valid semantic version")) } - return nil + return allErrs } // validateSSHKey validates an SSHKey. @@ -311,52 +330,44 @@ func (m *AzureManagedControlPlane) validateSSHKey(_ client.Client) error { } // validateLoadBalancerProfile validates a LoadBalancerProfile. -func (m *AzureManagedControlPlane) validateLoadBalancerProfile(_ client.Client) error { - if m.Spec.LoadBalancerProfile != nil { - var errs []error - var allErrs field.ErrorList +func validateLoadBalancerProfile(loadBalancerProfile *LoadBalancerProfile, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + if loadBalancerProfile != nil { numOutboundIPTypes := 0 - if m.Spec.LoadBalancerProfile.ManagedOutboundIPs != nil { - if *m.Spec.LoadBalancerProfile.ManagedOutboundIPs < 1 || *m.Spec.LoadBalancerProfile.ManagedOutboundIPs > 100 { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "LoadBalancerProfile", "ManagedOutboundIPs"), *m.Spec.LoadBalancerProfile.ManagedOutboundIPs, "value should be in between 1 and 100")) + if loadBalancerProfile.ManagedOutboundIPs != nil { + if *loadBalancerProfile.ManagedOutboundIPs < 1 || *loadBalancerProfile.ManagedOutboundIPs > 100 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("ManagedOutboundIPs"), *loadBalancerProfile.ManagedOutboundIPs, "value should be in between 1 and 100")) } } - if m.Spec.LoadBalancerProfile.AllocatedOutboundPorts != nil { - if *m.Spec.LoadBalancerProfile.AllocatedOutboundPorts < 0 || *m.Spec.LoadBalancerProfile.AllocatedOutboundPorts > 64000 { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "LoadBalancerProfile", "AllocatedOutboundPorts"), *m.Spec.LoadBalancerProfile.AllocatedOutboundPorts, "value should be in between 0 and 64000")) + if loadBalancerProfile.AllocatedOutboundPorts != nil { + if *loadBalancerProfile.AllocatedOutboundPorts < 0 || *loadBalancerProfile.AllocatedOutboundPorts > 64000 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("AllocatedOutboundPorts"), *loadBalancerProfile.AllocatedOutboundPorts, "value should be in between 0 and 64000")) } } - if m.Spec.LoadBalancerProfile.IdleTimeoutInMinutes != nil { - if *m.Spec.LoadBalancerProfile.IdleTimeoutInMinutes < 4 || *m.Spec.LoadBalancerProfile.IdleTimeoutInMinutes > 120 { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "LoadBalancerProfile", "IdleTimeoutInMinutes"), *m.Spec.LoadBalancerProfile.IdleTimeoutInMinutes, "value should be in between 4 and 120")) + if loadBalancerProfile.IdleTimeoutInMinutes != nil { + if *loadBalancerProfile.IdleTimeoutInMinutes < 4 || *loadBalancerProfile.IdleTimeoutInMinutes > 120 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("IdleTimeoutInMinutes"), *loadBalancerProfile.IdleTimeoutInMinutes, "value should be in between 4 and 120")) } } - if m.Spec.LoadBalancerProfile.ManagedOutboundIPs != nil { + if loadBalancerProfile.ManagedOutboundIPs != nil { numOutboundIPTypes++ } - if len(m.Spec.LoadBalancerProfile.OutboundIPPrefixes) > 0 { + if len(loadBalancerProfile.OutboundIPPrefixes) > 0 { numOutboundIPTypes++ } - if len(m.Spec.LoadBalancerProfile.OutboundIPs) > 0 { + if len(loadBalancerProfile.OutboundIPs) > 0 { numOutboundIPTypes++ } if numOutboundIPTypes > 1 { - errs = append(errs, errors.New("load balancer profile must specify at most one of ManagedOutboundIPs, OutboundIPPrefixes and OutboundIPs")) - } - - if len(allErrs) > 0 { - agg := kerrors.NewAggregate(allErrs.ToAggregate().Errors()) - errs = append(errs, agg) + allErrs = append(allErrs, field.Forbidden(fldPath, "load balancer profile must specify at most one of ManagedOutboundIPs, OutboundIPPrefixes and OutboundIPs")) } - - return kerrors.NewAggregate(errs) } - return nil + return allErrs } // validateAPIServerAccessProfile validates an APIServerAccessProfile. @@ -376,30 +387,31 @@ func (m *AzureManagedControlPlane) validateAPIServerAccessProfile(_ client.Clien } // validateManagedClusterNetwork validates the Cluster network values. -func (m *AzureManagedControlPlane) validateManagedClusterNetwork(cli client.Client) error { +func validateManagedClusterNetwork(cli client.Client, labels map[string]string, namespace string, dnsServiceIP *string, subnet ManagedControlPlaneSubnet, fldPath *field.Path) field.ErrorList { + var ( + allErrs field.ErrorList + serviceCIDR string + ) + ctx := context.Background() // Fetch the Cluster. - clusterName, ok := m.Labels[clusterv1.ClusterLabelName] + clusterName, ok := labels[clusterv1.ClusterLabelName] if !ok { return nil } ownerCluster := &clusterv1.Cluster{} key := client.ObjectKey{ - Namespace: m.Namespace, + Namespace: namespace, Name: clusterName, } if err := cli.Get(ctx, key, ownerCluster); err != nil { - return err + allErrs = append(allErrs, field.InternalError(fldPath, err)) + return allErrs } - var ( - allErrs field.ErrorList - serviceCIDR string - ) - if clusterNetwork := ownerCluster.Spec.ClusterNetwork; clusterNetwork != nil { if clusterNetwork.Services != nil { // A user may provide zero or one CIDR blocks. If they provide an empty array, @@ -420,7 +432,7 @@ func (m *AzureManagedControlPlane) validateManagedClusterNetwork(cli client.Clie } } - if m.Spec.DNSServiceIP != nil { + if dnsServiceIP != nil { if serviceCIDR == "" { allErrs = append(allErrs, field.Required(field.NewPath("Cluster", "Spec", "ClusterNetwork", "Services", "CIDRBlocks"), "service CIDR must be specified if specifying DNSServiceIP")) } @@ -428,20 +440,17 @@ func (m *AzureManagedControlPlane) validateManagedClusterNetwork(cli client.Clie if err != nil { allErrs = append(allErrs, field.Invalid(field.NewPath("Cluster", "Spec", "ClusterNetwork", "Services", "CIDRBlocks"), serviceCIDR, fmt.Sprintf("failed to parse cluster service cidr: %v", err))) } - ip := net.ParseIP(*m.Spec.DNSServiceIP) + ip := net.ParseIP(*dnsServiceIP) if !cidr.Contains(ip) { allErrs = append(allErrs, field.Invalid(field.NewPath("Cluster", "Spec", "ClusterNetwork", "Services", "CIDRBlocks"), serviceCIDR, "DNSServiceIP must reside within the associated cluster serviceCIDR")) } } - if errs := validatePrivateEndpoints(m.Spec.VirtualNetwork.Subnet.PrivateEndpoints, []string{m.Spec.VirtualNetwork.Subnet.CIDRBlock}, field.NewPath("Spec", "VirtualNetwork.Subnet.PrivateEndpoints")); len(errs) > 0 { + if errs := validatePrivateEndpoints(subnet.PrivateEndpoints, []string{subnet.CIDRBlock}, fldPath.Child("VirtualNetwork.Subnet.PrivateEndpoints")); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if len(allErrs) > 0 { - return kerrors.NewAggregate(allErrs.ToAggregate().Errors()) - } - return nil + return allErrs } // validateAPIServerAccessProfileUpdate validates update to APIServerAccessProfile. @@ -524,153 +533,151 @@ func (m *AzureManagedControlPlane) validateVirtualNetworkUpdate(old *AzureManage return allErrs } -func (m *AzureManagedControlPlane) validateName(_ client.Client) error { - if lName := strings.ToLower(m.Name); strings.Contains(lName, "microsoft") || +func validateName(name string, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + if lName := strings.ToLower(name); strings.Contains(lName, "microsoft") || strings.Contains(lName, "windows") { - return field.Invalid(field.NewPath("Name"), m.Name, - "cluster name is invalid because 'MICROSOFT' and 'WINDOWS' can't be used as either a whole word or a substring in the name") + allErrs = append(allErrs, field.Invalid(fldPath.Child("Name"), name, + "cluster name is invalid because 'MICROSOFT' and 'WINDOWS' can't be used as either a whole word or a substring in the name")) } - return nil + return allErrs } // validateAutoScalerProfile validates an AutoScalerProfile. -func (m *AzureManagedControlPlane) validateAutoScalerProfile(_ client.Client) error { +func validateAutoScalerProfile(autoScalerProfile *AutoScalerProfile, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - if m.Spec.AutoScalerProfile == nil { + if autoScalerProfile == nil { return nil } - if errs := m.validateIntegerStringGreaterThanZero(m.Spec.AutoScalerProfile.MaxEmptyBulkDelete, "MaxEmptyBulkDelete"); len(errs) > 0 { + if errs := validateIntegerStringGreaterThanZero(autoScalerProfile.MaxEmptyBulkDelete, fldPath, "MaxEmptyBulkDelete"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateIntegerStringGreaterThanZero(m.Spec.AutoScalerProfile.MaxGracefulTerminationSec, "MaxGracefulTerminationSec"); len(errs) > 0 { + if errs := validateIntegerStringGreaterThanZero(autoScalerProfile.MaxGracefulTerminationSec, fldPath, "MaxGracefulTerminationSec"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateMaxNodeProvisionTime(); len(errs) > 0 { + if errs := validateMaxNodeProvisionTime(autoScalerProfile.MaxNodeProvisionTime, fldPath); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if m.Spec.AutoScalerProfile.MaxTotalUnreadyPercentage != nil { - val, err := strconv.Atoi(*m.Spec.AutoScalerProfile.MaxTotalUnreadyPercentage) + if autoScalerProfile.MaxTotalUnreadyPercentage != nil { + val, err := strconv.Atoi(*autoScalerProfile.MaxTotalUnreadyPercentage) if err != nil || val < 0 || val > 100 { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "MaxTotalUnreadyPercentage"), m.Spec.AutoScalerProfile.MaxTotalUnreadyPercentage, "invalid value")) + allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "MaxTotalUnreadyPercentage"), autoScalerProfile.MaxTotalUnreadyPercentage, "invalid value")) } } - if errs := m.validateNewPodScaleUpDelay(); len(errs) > 0 { + if errs := validateNewPodScaleUpDelay(autoScalerProfile.NewPodScaleUpDelay, fldPath); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateIntegerStringGreaterThanZero(m.Spec.AutoScalerProfile.OkTotalUnreadyCount, "OkTotalUnreadyCount"); len(errs) > 0 { + if errs := validateIntegerStringGreaterThanZero(autoScalerProfile.OkTotalUnreadyCount, fldPath, "OkTotalUnreadyCount"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateScanInterval(); len(errs) > 0 { + if errs := validateScanInterval(autoScalerProfile.ScanInterval, fldPath); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateScaleDownTime(m.Spec.AutoScalerProfile.ScaleDownDelayAfterAdd, "ScaleDownDelayAfterAdd"); len(errs) > 0 { + if errs := validateScaleDownTime(autoScalerProfile.ScaleDownDelayAfterAdd, fldPath, "ScaleDownDelayAfterAdd"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateScaleDownDelayAfterDelete(); len(errs) > 0 { + if errs := validateScaleDownDelayAfterDelete(autoScalerProfile.ScaleDownDelayAfterDelete, fldPath); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateScaleDownTime(m.Spec.AutoScalerProfile.ScaleDownDelayAfterFailure, "ScaleDownDelayAfterFailure"); len(errs) > 0 { + if errs := validateScaleDownTime(autoScalerProfile.ScaleDownDelayAfterFailure, fldPath, "ScaleDownDelayAfterFailure"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateScaleDownTime(m.Spec.AutoScalerProfile.ScaleDownUnneededTime, "ScaleDownUnneededTime"); len(errs) > 0 { + if errs := validateScaleDownTime(autoScalerProfile.ScaleDownUnneededTime, fldPath, "ScaleDownUnneededTime"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if errs := m.validateScaleDownTime(m.Spec.AutoScalerProfile.ScaleDownUnreadyTime, "ScaleDownUnreadyTime"); len(errs) > 0 { + if errs := validateScaleDownTime(autoScalerProfile.ScaleDownUnreadyTime, fldPath, "ScaleDownUnreadyTime"); len(errs) > 0 { allErrs = append(allErrs, errs...) } - if m.Spec.AutoScalerProfile.ScaleDownUtilizationThreshold != nil { - val, err := strconv.ParseFloat(*m.Spec.AutoScalerProfile.ScaleDownUtilizationThreshold, 32) + if autoScalerProfile.ScaleDownUtilizationThreshold != nil { + val, err := strconv.ParseFloat(*autoScalerProfile.ScaleDownUtilizationThreshold, 32) if err != nil || val < 0 || val > 1 { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "ScaleDownUtilizationThreshold"), m.Spec.AutoScalerProfile.ScaleDownUtilizationThreshold, "invalid value")) + allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "ScaleDownUtilizationThreshold"), autoScalerProfile.ScaleDownUtilizationThreshold, "invalid value")) } } - if len(allErrs) > 0 { - return kerrors.NewAggregate(allErrs.ToAggregate().Errors()) - } - - return nil + return allErrs } // validateMaxNodeProvisionTime validates update to AutoscalerProfile.MaxNodeProvisionTime. -func (m *AzureManagedControlPlane) validateMaxNodeProvisionTime() field.ErrorList { +func validateMaxNodeProvisionTime(maxNodeProvisionTime *string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - if pointer.StringDeref(m.Spec.AutoScalerProfile.MaxNodeProvisionTime, "") != "" { - if !rMaxNodeProvisionTime.MatchString(pointer.StringDeref(m.Spec.AutoScalerProfile.MaxNodeProvisionTime, "")) { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "MaxNodeProvisionTime"), m.Spec.AutoScalerProfile.MaxNodeProvisionTime, "invalid value")) + if pointer.StringDeref(maxNodeProvisionTime, "") != "" { + if !rMaxNodeProvisionTime.MatchString(pointer.StringDeref(maxNodeProvisionTime, "")) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("MaxNodeProvisionTime"), maxNodeProvisionTime, "invalid value")) } } return allErrs } // validateScanInterval validates update to AutoscalerProfile.ScanInterval. -func (m *AzureManagedControlPlane) validateScanInterval() field.ErrorList { +func validateScanInterval(scanInterval *string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - if pointer.StringDeref(m.Spec.AutoScalerProfile.ScanInterval, "") != "" { - if !rScanInterval.MatchString(pointer.StringDeref(m.Spec.AutoScalerProfile.ScanInterval, "")) { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "ScanInterval"), m.Spec.AutoScalerProfile.ScanInterval, "invalid value")) + if pointer.StringDeref(scanInterval, "") != "" { + if !rScanInterval.MatchString(pointer.StringDeref(scanInterval, "")) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("ScanInterval"), scanInterval, "invalid value")) } } return allErrs } // validateNewPodScaleUpDelay validates update to AutoscalerProfile.NewPodScaleUpDelay. -func (m *AzureManagedControlPlane) validateNewPodScaleUpDelay() field.ErrorList { +func validateNewPodScaleUpDelay(newPodScaleUpDelay *string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - if pointer.StringDeref(m.Spec.AutoScalerProfile.NewPodScaleUpDelay, "") != "" { - _, err := time.ParseDuration(pointer.StringDeref(m.Spec.AutoScalerProfile.NewPodScaleUpDelay, "")) + if pointer.StringDeref(newPodScaleUpDelay, "") != "" { + _, err := time.ParseDuration(pointer.StringDeref(newPodScaleUpDelay, "")) if err != nil { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "NewPodScaleUpDelay"), m.Spec.AutoScalerProfile.NewPodScaleUpDelay, "invalid value")) + allErrs = append(allErrs, field.Invalid(fldPath.Child("NewPodScaleUpDelay"), newPodScaleUpDelay, "invalid value")) } } return allErrs } // validateScaleDownDelayAfterDelete validates update to AutoscalerProfile.ScaleDownDelayAfterDelete value. -func (m *AzureManagedControlPlane) validateScaleDownDelayAfterDelete() field.ErrorList { +func validateScaleDownDelayAfterDelete(scaleDownDelayAfterDelete *string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - if pointer.StringDeref(m.Spec.AutoScalerProfile.ScaleDownDelayAfterDelete, "") != "" { - if !rScaleDownDelayAfterDelete.MatchString(pointer.StringDeref(m.Spec.AutoScalerProfile.ScaleDownDelayAfterDelete, "")) { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", "ScaleDownDelayAfterDelete"), pointer.StringDeref(m.Spec.AutoScalerProfile.ScaleDownDelayAfterDelete, ""), "invalid value")) + if pointer.StringDeref(scaleDownDelayAfterDelete, "") != "" { + if !rScaleDownDelayAfterDelete.MatchString(pointer.StringDeref(scaleDownDelayAfterDelete, "")) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("ScaleDownDelayAfterDelete"), pointer.StringDeref(scaleDownDelayAfterDelete, ""), "invalid value")) } } return allErrs } // validateScaleDownTime validates update to AutoscalerProfile.ScaleDown* values. -func (m *AzureManagedControlPlane) validateScaleDownTime(scaleDownValue *string, fieldName string) field.ErrorList { +func validateScaleDownTime(scaleDownValue *string, fldPath *field.Path, fieldName string) field.ErrorList { var allErrs field.ErrorList if pointer.StringDeref(scaleDownValue, "") != "" { if !rScaleDownTime.MatchString(pointer.StringDeref(scaleDownValue, "")) { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", fieldName), pointer.StringDeref(scaleDownValue, ""), "invalid value")) + allErrs = append(allErrs, field.Invalid(fldPath.Child(fieldName), pointer.StringDeref(scaleDownValue, ""), "invalid value")) } } return allErrs } // validateIntegerStringGreaterThanZero validates that a string value is an integer greater than zero. -func (m *AzureManagedControlPlane) validateIntegerStringGreaterThanZero(input *string, fieldName string) field.ErrorList { +func validateIntegerStringGreaterThanZero(input *string, fldPath *field.Path, fieldName string) field.ErrorList { var allErrs field.ErrorList if input != nil { val, err := strconv.Atoi(*input) if err != nil || val < 0 { - allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "AutoscalerProfile", fieldName), input, "invalid value")) + allErrs = append(allErrs, field.Invalid(fldPath.Child(fieldName), input, "invalid value")) } } diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index c62161c6906..74e93ef1684 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation/field" utilfeature "k8s.io/component-base/featuregate/testing" "k8s.io/utils/pointer" "sigs.k8s.io/cluster-api-provider-azure/feature" @@ -78,471 +79,362 @@ func TestDefaultingWebhook(t *testing.T) { g.Expect(amcp.Spec.SKU.Tier).To(Equal(PaidManagedControlPlaneTier)) } -func TestValidatingWebhook(t *testing.T) { - // NOTE: AzureManageControlPlane is behind AKS feature gate flag; the webhook - // must prevent creating new objects in case the feature flag is disabled. - defer utilfeature.SetFeatureGateDuringTest(t, feature.Gates, capifeature.MachinePool, true)() +func TestValidateDNSServiceIP(t *testing.T) { g := NewWithT(t) tests := []struct { name string - amcp AzureManagedControlPlane + dnsIP *string expectErr bool }{ { - name: "Testing valid DNSServiceIP", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.String("192.168.0.0"), - Version: "v1.17.8", - }, - }, + name: "Testing valid DNSServiceIP", + dnsIP: pointer.String("192.168.0.0"), expectErr: false, }, { - name: "Testing invalid DNSServiceIP", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.String("192.168.0.0.3"), - Version: "v1.17.8", - }, - }, - expectErr: true, + name: "Testing invalid DNSServiceIP", + dnsIP: pointer.String("192.168.0.0.3"), + expectErr: false, }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + allErrs := validateDNSServiceIP(tt.dnsIP, field.NewPath("spec").Child("DNSServiceIP")) + if tt.expectErr { + g.Expect(allErrs).NotTo(BeNil()) + } else { + g.Expect(allErrs).To(BeNil()) + } + }) + } +} + +func TestValidateVersion(t *testing.T) { + g := NewWithT(t) + tests := []struct { + name string + version string + expectErr bool + }{ { - name: "Invalid Version", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.String("192.168.0.0"), - Version: "honk", - }, - }, + name: "Invalid Version", + version: "honk", expectErr: true, }, { - name: "not following the Kubernetes Version pattern", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.String("192.168.0.0"), - Version: "1.19.0", - }, - }, + name: "not following the Kubernetes Version pattern", + version: "1.19.0", expectErr: true, }, { - name: "Version not set", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.String("192.168.0.0"), - Version: "", - }, - }, + name: "Version not set", + version: "", expectErr: true, }, { - name: "Valid Version", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.String("192.168.0.0"), - Version: "v1.17.8", - }, - }, - expectErr: false, - }, - { - name: "Valid Managed AADProfile", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - AADProfile: &AADProfile{ - Managed: true, - AdminGroupObjectIDs: []string{ - "616077a8-5db7-4c98-b856-b34619afg75h", - }, - }, - }, - }, + name: "Valid Version", + version: "v1.17.8", expectErr: false, }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + allErrs := validateVersion(tt.version, field.NewPath("spec").Child("Version")) + if tt.expectErr { + g.Expect(allErrs).NotTo(BeNil()) + } else { + g.Expect(allErrs).To(BeNil()) + } + }) + } +} + +func TestValidateLoadBalancerProfile(t *testing.T) { + g := NewWithT(t) + tests := []struct { + name string + profile *LoadBalancerProfile + expectErr bool + }{ { name: "Valid LoadBalancerProfile", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - LoadBalancerProfile: &LoadBalancerProfile{ - ManagedOutboundIPs: pointer.Int32(10), - AllocatedOutboundPorts: pointer.Int32(1000), - IdleTimeoutInMinutes: pointer.Int32(60), - }, - }, + profile: &LoadBalancerProfile{ + ManagedOutboundIPs: pointer.Int32(10), + AllocatedOutboundPorts: pointer.Int32(1000), + IdleTimeoutInMinutes: pointer.Int32(60), }, expectErr: false, }, { name: "Invalid LoadBalancerProfile.ManagedOutboundIPs", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - LoadBalancerProfile: &LoadBalancerProfile{ - ManagedOutboundIPs: pointer.Int32(200), - }, - }, - }, - expectErr: true, - }, - { - name: "Invalid LoadBalancerProfile.AllocatedOutboundPorts", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - LoadBalancerProfile: &LoadBalancerProfile{ - AllocatedOutboundPorts: pointer.Int32(80000), - }, - }, + profile: &LoadBalancerProfile{ + ManagedOutboundIPs: pointer.Int32(200), }, expectErr: true, }, { name: "Invalid LoadBalancerProfile.IdleTimeoutInMinutes", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - LoadBalancerProfile: &LoadBalancerProfile{ - IdleTimeoutInMinutes: pointer.Int32(600), - }, - }, + profile: &LoadBalancerProfile{ + IdleTimeoutInMinutes: pointer.Int32(600), }, expectErr: true, }, { name: "LoadBalancerProfile must specify at most one of ManagedOutboundIPs, OutboundIPPrefixes and OutboundIPs", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - LoadBalancerProfile: &LoadBalancerProfile{ - ManagedOutboundIPs: pointer.Int32(1), - OutboundIPs: []string{ - "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/foo-bar/providers/Microsoft.Network/publicIPAddresses/my-public-ip", - }, - }, - }, - }, - expectErr: true, - }, - { - name: "Invalid CIDR for AuthorizedIPRanges", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.21.2", - APIServerAccessProfile: &APIServerAccessProfile{ - AuthorizedIPRanges: []string{"1.2.3.400/32"}, - }, + profile: &LoadBalancerProfile{ + ManagedOutboundIPs: pointer.Int32(1), + OutboundIPs: []string{ + "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/foo-bar/providers/Microsoft.Network/publicIPAddresses/my-public-ip", }, }, expectErr: true, }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + allErrs := validateLoadBalancerProfile(tt.profile, field.NewPath("spec").Child("LoadBalancerProfile")) + if tt.expectErr { + g.Expect(allErrs).NotTo(BeNil()) + } else { + g.Expect(allErrs).To(BeNil()) + } + }) + } +} + +func TestValidateAutoScalerProfile(t *testing.T) { + g := NewWithT(t) + tests := []struct { + name string + profile *AutoScalerProfile + expectErr bool + }{ { - name: "Testing valid AutoScalerProfile", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(pointer.String(string(BalanceSimilarNodeGroupsFalse))), - Expander: (*Expander)(pointer.String(string(ExpanderRandom))), - MaxEmptyBulkDelete: pointer.String("10"), - MaxGracefulTerminationSec: pointer.String("600"), - MaxNodeProvisionTime: pointer.String("10m"), - MaxTotalUnreadyPercentage: pointer.String("45"), - NewPodScaleUpDelay: pointer.String("10m"), - OkTotalUnreadyCount: pointer.String("3"), - ScanInterval: pointer.String("60s"), - ScaleDownDelayAfterAdd: pointer.String("10m"), - ScaleDownDelayAfterDelete: pointer.String("10s"), - ScaleDownDelayAfterFailure: pointer.String("10m"), - ScaleDownUnneededTime: pointer.String("10m"), - ScaleDownUnreadyTime: pointer.String("10m"), - ScaleDownUtilizationThreshold: pointer.String("0.5"), - SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(pointer.String(string(SkipNodesWithLocalStorageTrue))), - SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(pointer.String(string(SkipNodesWithSystemPodsTrue))), - }, - }, + name: "Valid AutoScalerProfile", + profile: &AutoScalerProfile{ + BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(pointer.String(string(BalanceSimilarNodeGroupsFalse))), + Expander: (*Expander)(pointer.String(string(ExpanderRandom))), + MaxEmptyBulkDelete: pointer.String("10"), + MaxGracefulTerminationSec: pointer.String("600"), + MaxNodeProvisionTime: pointer.String("10m"), + MaxTotalUnreadyPercentage: pointer.String("45"), + NewPodScaleUpDelay: pointer.String("10m"), + OkTotalUnreadyCount: pointer.String("3"), + ScanInterval: pointer.String("60s"), + ScaleDownDelayAfterAdd: pointer.String("10m"), + ScaleDownDelayAfterDelete: pointer.String("10s"), + ScaleDownDelayAfterFailure: pointer.String("10m"), + ScaleDownUnneededTime: pointer.String("10m"), + ScaleDownUnreadyTime: pointer.String("10m"), + ScaleDownUtilizationThreshold: pointer.String("0.5"), + SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(pointer.String(string(SkipNodesWithLocalStorageTrue))), + SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(pointer.String(string(SkipNodesWithSystemPodsTrue))), }, expectErr: false, }, { name: "Testing valid AutoScalerProfile.ExpanderRandom", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - Expander: (*Expander)(pointer.String(string(ExpanderRandom))), - }, - }, + profile: &AutoScalerProfile{ + Expander: (*Expander)(pointer.String(string(ExpanderRandom))), }, expectErr: false, }, { name: "Testing valid AutoScalerProfile.ExpanderLeastWaste", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - Expander: (*Expander)(pointer.String(string(ExpanderLeastWaste))), - }, - }, + profile: &AutoScalerProfile{ + Expander: (*Expander)(pointer.String(string(ExpanderLeastWaste))), }, expectErr: false, }, { name: "Testing valid AutoScalerProfile.ExpanderMostPods", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - Expander: (*Expander)(pointer.String(string(ExpanderMostPods))), - }, - }, + profile: &AutoScalerProfile{ + Expander: (*Expander)(pointer.String(string(ExpanderMostPods))), }, expectErr: false, }, { name: "Testing valid AutoScalerProfile.ExpanderPriority", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - Expander: (*Expander)(pointer.String(string(ExpanderPriority))), - }, - }, + profile: &AutoScalerProfile{ + Expander: (*Expander)(pointer.String(string(ExpanderPriority))), }, expectErr: false, }, { name: "Testing valid AutoScalerProfile.BalanceSimilarNodeGroupsTrue", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(pointer.String(string(BalanceSimilarNodeGroupsTrue))), - }, - }, + profile: &AutoScalerProfile{ + BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(pointer.String(string(BalanceSimilarNodeGroupsTrue))), }, expectErr: false, }, { name: "Testing valid AutoScalerProfile.BalanceSimilarNodeGroupsFalse", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(pointer.String(string(BalanceSimilarNodeGroupsFalse))), - }, - }, + profile: &AutoScalerProfile{ + BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(pointer.String(string(BalanceSimilarNodeGroupsFalse))), }, expectErr: false, }, { name: "Testing invalid AutoScalerProfile.MaxEmptyBulkDelete", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - MaxEmptyBulkDelete: pointer.String("invalid"), - }, - }, + profile: &AutoScalerProfile{ + MaxEmptyBulkDelete: pointer.String("invalid"), }, expectErr: true, }, { name: "Testing invalid AutoScalerProfile.MaxGracefulTerminationSec", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - MaxGracefulTerminationSec: pointer.String("invalid"), - }, - }, + profile: &AutoScalerProfile{ + MaxGracefulTerminationSec: pointer.String("invalid"), }, expectErr: true, }, { name: "Testing invalid AutoScalerProfile.MaxNodeProvisionTime", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - MaxNodeProvisionTime: pointer.String("invalid"), - }, - }, + profile: &AutoScalerProfile{ + MaxNodeProvisionTime: pointer.String("invalid"), }, expectErr: true, }, { name: "Testing invalid AutoScalerProfile.MaxTotalUnreadyPercentage", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - MaxTotalUnreadyPercentage: pointer.String("invalid"), - }, - }, + profile: &AutoScalerProfile{ + MaxTotalUnreadyPercentage: pointer.String("invalid"), }, expectErr: true, }, { name: "Testing invalid AutoScalerProfile.NewPodScaleUpDelay", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - NewPodScaleUpDelay: pointer.String("invalid"), - }, - }, + profile: &AutoScalerProfile{ + NewPodScaleUpDelay: pointer.String("invalid"), }, expectErr: true, }, { name: "Testing invalid AutoScalerProfile.OkTotalUnreadyCount", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - OkTotalUnreadyCount: pointer.String("invalid"), - }, - }, + profile: &AutoScalerProfile{ + OkTotalUnreadyCount: pointer.String("invalid"), }, expectErr: true, }, { name: "Testing invalid AutoScalerProfile.ScanInterval", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScanInterval: pointer.String("invalid"), - }, - }, + profile: &AutoScalerProfile{ + ScanInterval: pointer.String("invalid"), }, expectErr: true, }, { name: "Testing invalid AutoScalerProfile.ScaleDownDelayAfterAdd", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScaleDownDelayAfterAdd: pointer.String("invalid"), - }, - }, + profile: &AutoScalerProfile{ + ScaleDownDelayAfterAdd: pointer.String("invalid"), }, expectErr: true, }, { name: "Testing invalid AutoScalerProfile.ScaleDownDelayAfterDelete", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScaleDownDelayAfterDelete: pointer.String("invalid"), - }, - }, + profile: &AutoScalerProfile{ + ScaleDownDelayAfterDelete: pointer.String("invalid"), }, expectErr: true, }, { name: "Testing invalid AutoScalerProfile.ScaleDownDelayAfterFailure", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScaleDownDelayAfterFailure: pointer.String("invalid"), - }, - }, + profile: &AutoScalerProfile{ + ScaleDownDelayAfterFailure: pointer.String("invalid"), }, expectErr: true, }, { name: "Testing invalid AutoScalerProfile.ScaleDownUnneededTime", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScaleDownUnneededTime: pointer.String("invalid"), - }, - }, + profile: &AutoScalerProfile{ + ScaleDownUnneededTime: pointer.String("invalid"), }, expectErr: true, }, { name: "Testing invalid AutoScalerProfile.ScaleDownUnreadyTime", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScaleDownUnreadyTime: pointer.String("invalid"), - }, - }, + profile: &AutoScalerProfile{ + ScaleDownUnreadyTime: pointer.String("invalid"), }, expectErr: true, }, { name: "Testing invalid AutoScalerProfile.ScaleDownUtilizationThreshold", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - ScaleDownUtilizationThreshold: pointer.String("invalid"), - }, - }, + profile: &AutoScalerProfile{ + ScaleDownUtilizationThreshold: pointer.String("invalid"), }, expectErr: true, }, { name: "Testing valid AutoScalerProfile.SkipNodesWithLocalStorageTrue", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(pointer.String(string(SkipNodesWithLocalStorageTrue))), - }, - }, + profile: &AutoScalerProfile{ + SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(pointer.String(string(SkipNodesWithLocalStorageTrue))), }, expectErr: false, }, { name: "Testing valid AutoScalerProfile.SkipNodesWithLocalStorageFalse", - amcp: AzureManagedControlPlane{ - Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(pointer.String(string(SkipNodesWithLocalStorageFalse))), - }, - }, + profile: &AutoScalerProfile{ + SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(pointer.String(string(SkipNodesWithSystemPodsFalse))), }, expectErr: false, }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + allErrs := validateAutoScalerProfile(tt.profile, field.NewPath("spec").Child("AutoScalerProfile")) + if tt.expectErr { + g.Expect(allErrs).NotTo(BeNil()) + } else { + g.Expect(allErrs).To(BeNil()) + } + }) + } +} + +func TestValidatingWebhook(t *testing.T) { + // NOTE: AzureManageControlPlane is behind AKS feature gate flag; the webhook + // must prevent creating new objects in case the feature flag is disabled. + defer utilfeature.SetFeatureGateDuringTest(t, feature.Gates, capifeature.MachinePool, true)() + g := NewWithT(t) + tests := []struct { + name string + amcp AzureManagedControlPlane + expectErr bool + }{ { - name: "Testing valid AutoScalerProfile.SkipNodesWithSystemPodsTrue", + name: "Valid Managed AADProfile", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(pointer.String(string(SkipNodesWithSystemPodsTrue))), + Version: "v1.21.2", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{ + "616077a8-5db7-4c98-b856-b34619afg75h", + }, }, }, }, expectErr: false, }, { - name: "Testing valid AutoScalerProfile.SkipNodesWithSystemPodsFalse", + name: "Invalid CIDR for AuthorizedIPRanges", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - Version: "v1.24.1", - AutoScalerProfile: &AutoScalerProfile{ - SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(pointer.String(string(SkipNodesWithSystemPodsFalse))), + Version: "v1.21.2", + APIServerAccessProfile: &APIServerAccessProfile{ + AuthorizedIPRanges: []string{"1.2.3.400/32"}, }, }, }, - expectErr: false, + expectErr: true, }, } diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_default.go b/api/v1beta1/azuremanagedcontrolplanetemplate_default.go new file mode 100644 index 00000000000..2feca15b274 --- /dev/null +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_default.go @@ -0,0 +1,133 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "strings" + + "k8s.io/utils/pointer" +) + +func (mcp *AzureManagedControlPlaneTemplate) setDefaults() { + if mcp.Spec.Template.Spec.NetworkPlugin == nil { + networkPlugin := "azure" + mcp.Spec.Template.Spec.NetworkPlugin = &networkPlugin + } + if mcp.Spec.Template.Spec.LoadBalancerSKU == nil { + loadBalancerSKU := "Standard" + mcp.Spec.Template.Spec.LoadBalancerSKU = &loadBalancerSKU + } + + if mcp.Spec.Template.Spec.Version != "" && !strings.HasPrefix(mcp.Spec.Template.Spec.Version, "v") { + normalizedVersion := "v" + mcp.Spec.Template.Spec.Version + mcp.Spec.Template.Spec.Version = normalizedVersion + } + + mcp.setDefaultVirtualNetwork() + mcp.setDefaultSubnet() + mcp.setDefaultSku() + mcp.setDefaultAutoScalerProfile() +} + +// setDefaultVirtualNetwork sets the default VirtualNetwork for an AzureManagedControlPlaneTemplate. +func (mcp *AzureManagedControlPlaneTemplate) setDefaultVirtualNetwork() { + if mcp.Spec.Template.Spec.VirtualNetwork.Name == "" { + mcp.Spec.Template.Spec.VirtualNetwork.Name = mcp.Name + } + if mcp.Spec.Template.Spec.VirtualNetwork.CIDRBlock == "" { + mcp.Spec.Template.Spec.VirtualNetwork.CIDRBlock = defaultAKSVnetCIDR + } +} + +// setDefaultSubnet sets the default Subnet for an AzureManagedControlPlaneTemplate. +func (mcp *AzureManagedControlPlaneTemplate) setDefaultSubnet() { + if mcp.Spec.Template.Spec.VirtualNetwork.Subnet.Name == "" { + mcp.Spec.Template.Spec.VirtualNetwork.Subnet.Name = mcp.Name + } + if mcp.Spec.Template.Spec.VirtualNetwork.Subnet.CIDRBlock == "" { + mcp.Spec.Template.Spec.VirtualNetwork.Subnet.CIDRBlock = defaultAKSNodeSubnetCIDR + } +} + +func (mcp *AzureManagedControlPlaneTemplate) setDefaultSku() { + if mcp.Spec.Template.Spec.SKU == nil { + mcp.Spec.Template.Spec.SKU = &AKSSku{ + Tier: FreeManagedControlPlaneTier, + } + } +} + +func (mcp *AzureManagedControlPlaneTemplate) setDefaultAutoScalerProfile() { + if mcp.Spec.Template.Spec.AutoScalerProfile == nil { + return + } + + // Default values are from https://learn.microsoft.com/en-us/azure/aks/cluster-autoscaler#using-the-autoscaler-profile + // If any values are set, they all need to be set. + if mcp.Spec.Template.Spec.AutoScalerProfile.BalanceSimilarNodeGroups == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.BalanceSimilarNodeGroups = (*BalanceSimilarNodeGroups)(pointer.String(string(BalanceSimilarNodeGroupsFalse))) + } + if mcp.Spec.Template.Spec.AutoScalerProfile.Expander == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.Expander = (*Expander)(pointer.String(string(ExpanderRandom))) + } + if mcp.Spec.Template.Spec.AutoScalerProfile.MaxEmptyBulkDelete == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.MaxEmptyBulkDelete = pointer.String("10") + } + if mcp.Spec.Template.Spec.AutoScalerProfile.MaxGracefulTerminationSec == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.MaxGracefulTerminationSec = pointer.String("600") + } + if mcp.Spec.Template.Spec.AutoScalerProfile.MaxNodeProvisionTime == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.MaxNodeProvisionTime = pointer.String("15m") + } + if mcp.Spec.Template.Spec.AutoScalerProfile.MaxTotalUnreadyPercentage == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.MaxTotalUnreadyPercentage = pointer.String("45") + } + if mcp.Spec.Template.Spec.AutoScalerProfile.NewPodScaleUpDelay == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.NewPodScaleUpDelay = pointer.String("0s") + } + if mcp.Spec.Template.Spec.AutoScalerProfile.OkTotalUnreadyCount == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.OkTotalUnreadyCount = pointer.String("3") + } + if mcp.Spec.Template.Spec.AutoScalerProfile.ScanInterval == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.ScanInterval = pointer.String("10s") + } + if mcp.Spec.Template.Spec.AutoScalerProfile.ScaleDownDelayAfterAdd == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.ScaleDownDelayAfterAdd = pointer.String("10m") + } + if mcp.Spec.Template.Spec.AutoScalerProfile.ScaleDownDelayAfterDelete == nil { + // Default is the same as the ScanInterval so default to that same value if it isn't set + mcp.Spec.Template.Spec.AutoScalerProfile.ScaleDownDelayAfterDelete = mcp.Spec.Template.Spec.AutoScalerProfile.ScanInterval + } + if mcp.Spec.Template.Spec.AutoScalerProfile.ScaleDownDelayAfterFailure == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.ScaleDownDelayAfterFailure = pointer.String("3m") + } + if mcp.Spec.Template.Spec.AutoScalerProfile.ScaleDownUnneededTime == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.ScaleDownUnneededTime = pointer.String("10m") + } + if mcp.Spec.Template.Spec.AutoScalerProfile.ScaleDownUnreadyTime == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.ScaleDownUnreadyTime = pointer.String("20m") + } + if mcp.Spec.Template.Spec.AutoScalerProfile.ScaleDownUtilizationThreshold == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.ScaleDownUtilizationThreshold = pointer.String("0.5") + } + if mcp.Spec.Template.Spec.AutoScalerProfile.SkipNodesWithLocalStorage == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.SkipNodesWithLocalStorage = (*SkipNodesWithLocalStorage)(pointer.String(string(SkipNodesWithLocalStorageFalse))) + } + if mcp.Spec.Template.Spec.AutoScalerProfile.SkipNodesWithSystemPods == nil { + mcp.Spec.Template.Spec.AutoScalerProfile.SkipNodesWithSystemPods = (*SkipNodesWithSystemPods)(pointer.String(string(SkipNodesWithSystemPodsTrue))) + } +} diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_default_test.go b/api/v1beta1/azuremanagedcontrolplanetemplate_default_test.go new file mode 100644 index 00000000000..ec5c228d903 --- /dev/null +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_default_test.go @@ -0,0 +1,424 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "encoding/json" + "reflect" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" +) + +func TestDefaultVirtualNetworkTemplate(t *testing.T) { + cases := []struct { + name string + controlPlaneTemplate *AzureManagedControlPlaneTemplate + outputTemplate *AzureManagedControlPlaneTemplate + }{ + { + name: "virtual network not specified", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{}, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetworkTemplate{ + Name: "test-cluster-template", + CIDRBlock: defaultAKSVnetCIDR, + }, + }, + }, + }, + }, + }, + { + name: "custom name", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetworkTemplate{ + Name: "custom-vnet-name", + }, + }, + }, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetworkTemplate{ + Name: "custom-vnet-name", + CIDRBlock: defaultAKSVnetCIDR, + }, + }, + }, + }, + }, + }, + { + name: "custom cidr block", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetworkTemplate{ + CIDRBlock: "10.0.0.16/24", + }, + }, + }, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetworkTemplate{ + Name: "test-cluster-template", + CIDRBlock: "10.0.0.16/24", + }, + }, + }, + }, + }, + }, + } + + for _, c := range cases { + tc := c + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.controlPlaneTemplate.setDefaultVirtualNetwork() + if !reflect.DeepEqual(tc.controlPlaneTemplate, tc.outputTemplate) { + expected, _ := json.MarshalIndent(tc.outputTemplate, "", "\t") + actual, _ := json.MarshalIndent(tc.controlPlaneTemplate, "", "\t") + t.Errorf("Expected %s, got %s", string(expected), string(actual)) + } + }) + } +} + +func TestDefaultSubnetTemplate(t *testing.T) { + cases := []struct { + name string + controlPlaneTemplate *AzureManagedControlPlaneTemplate + outputTemplate *AzureManagedControlPlaneTemplate + }{ + { + name: "subnet not specified", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{}, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetworkTemplate{ + Subnet: ManagedControlPlaneSubnet{ + Name: "test-cluster-template", + CIDRBlock: defaultAKSNodeSubnetCIDR, + }, + }, + }, + }, + }, + }, + }, + { + name: "custom name", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetworkTemplate{ + Subnet: ManagedControlPlaneSubnet{ + Name: "custom-subnet-name", + }, + }, + }, + }, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetworkTemplate{ + Subnet: ManagedControlPlaneSubnet{ + Name: "custom-subnet-name", + CIDRBlock: defaultAKSNodeSubnetCIDR, + }, + }, + }, + }, + }, + }, + }, + { + name: "custom cidr block", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetworkTemplate{ + Subnet: ManagedControlPlaneSubnet{ + CIDRBlock: "10.0.0.16/24", + }, + }, + }, + }, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + VirtualNetwork: ManagedControlPlaneVirtualNetworkTemplate{ + Subnet: ManagedControlPlaneSubnet{ + Name: "test-cluster-template", + CIDRBlock: "10.0.0.16/24", + }, + }, + }, + }, + }, + }, + }, + } + + for _, c := range cases { + tc := c + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.controlPlaneTemplate.setDefaultSubnet() + if !reflect.DeepEqual(tc.controlPlaneTemplate, tc.outputTemplate) { + expected, _ := json.MarshalIndent(tc.outputTemplate, "", "\t") + actual, _ := json.MarshalIndent(tc.controlPlaneTemplate, "", "\t") + t.Errorf("Expected %s, got %s", string(expected), string(actual)) + } + }) + } +} + +func TestDefaultSkuTemplate(t *testing.T) { + cases := []struct { + name string + controlPlaneTemplate *AzureManagedControlPlaneTemplate + outputTemplate *AzureManagedControlPlaneTemplate + }{ + { + name: "sku not specified", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{}, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + SKU: &AKSSku{ + Tier: FreeManagedControlPlaneTier, + }, + }, + }, + }, + }, + }, + { + name: "paid sku", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + SKU: &AKSSku{ + Tier: PaidManagedControlPlaneTier, + }, + }, + }, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + SKU: &AKSSku{ + Tier: PaidManagedControlPlaneTier, + }, + }, + }, + }, + }, + }, + } + + for _, c := range cases { + tc := c + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.controlPlaneTemplate.setDefaultSku() + if !reflect.DeepEqual(tc.controlPlaneTemplate, tc.outputTemplate) { + expected, _ := json.MarshalIndent(tc.outputTemplate, "", "\t") + actual, _ := json.MarshalIndent(tc.controlPlaneTemplate, "", "\t") + t.Errorf("Expected %s, got %s", string(expected), string(actual)) + } + }) + } +} + +func TestDefaultAutoScalerProfile(t *testing.T) { + cases := []struct { + name string + controlPlaneTemplate *AzureManagedControlPlaneTemplate + outputTemplate *AzureManagedControlPlaneTemplate + }{ + { + name: "autoscaler profile not specified", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{}, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{}, + }, + }, + }, + { + name: "autoscaler profile empty but specified", + controlPlaneTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + AutoScalerProfile: &AutoScalerProfile{}, + }, + }, + }, + }, + outputTemplate: &AzureManagedControlPlaneTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-template", + }, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + AutoScalerProfile: &AutoScalerProfile{ + BalanceSimilarNodeGroups: (*BalanceSimilarNodeGroups)(pointer.String(string(BalanceSimilarNodeGroupsFalse))), + Expander: (*Expander)(pointer.String(string(ExpanderRandom))), + MaxEmptyBulkDelete: pointer.String("10"), + MaxGracefulTerminationSec: pointer.String("600"), + MaxNodeProvisionTime: pointer.String("15m"), + MaxTotalUnreadyPercentage: pointer.String("45"), + NewPodScaleUpDelay: pointer.String("0s"), + OkTotalUnreadyCount: pointer.String("3"), + ScanInterval: pointer.String("10s"), + ScaleDownDelayAfterAdd: pointer.String("10m"), + ScaleDownDelayAfterDelete: pointer.String("10s"), + ScaleDownDelayAfterFailure: pointer.String("3m"), + ScaleDownUnneededTime: pointer.String("10m"), + ScaleDownUnreadyTime: pointer.String("20m"), + ScaleDownUtilizationThreshold: pointer.String("0.5"), + SkipNodesWithLocalStorage: (*SkipNodesWithLocalStorage)(pointer.String(string(SkipNodesWithLocalStorageFalse))), + SkipNodesWithSystemPods: (*SkipNodesWithSystemPods)(pointer.String(string(SkipNodesWithSystemPodsTrue))), + }, + }, + }, + }, + }, + }, + } + + for _, c := range cases { + tc := c + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.controlPlaneTemplate.setDefaultAutoScalerProfile() + if !reflect.DeepEqual(tc.controlPlaneTemplate, tc.outputTemplate) { + expected, _ := json.MarshalIndent(tc.outputTemplate, "", "\t") + actual, _ := json.MarshalIndent(tc.controlPlaneTemplate, "", "\t") + t.Errorf("Expected %s, got %s", string(expected), string(actual)) + } + }) + } +} diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_types.go b/api/v1beta1/azuremanagedcontrolplanetemplate_types.go new file mode 100644 index 00000000000..f5be513bfb4 --- /dev/null +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_types.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// AzureManagedControlPlaneTemplateSpec defines the desired state of AzureManagedControlPlaneTemplate. +type AzureManagedControlPlaneTemplateSpec struct { + Template AzureManagedControlPlaneTemplateResource `json:"template"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=azuremanagedcontrolplanetemplates,scope=Namespaced,categories=cluster-api +// +kubebuilder:storageversion + +// AzureManagedControlPlaneTemplate is the Schema for the AzureManagedControlPlaneTemplates API. +type AzureManagedControlPlaneTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AzureManagedControlPlaneTemplateSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// AzureManagedControlPlaneTemplateList contains a list of AzureManagedControlPlaneTemplates. +type AzureManagedControlPlaneTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AzureManagedControlPlaneTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&AzureManagedControlPlaneTemplate{}, &AzureManagedControlPlaneTemplateList{}) +} + +// AzureManagedControlPlaneTemplateResource describes the data needed to create an AzureManagedCluster from a template. +type AzureManagedControlPlaneTemplateResource struct { + Spec AzureManagedControlPlaneTemplateResourceSpec `json:"spec"` +} diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_validation.go b/api/v1beta1/azuremanagedcontrolplanetemplate_validation.go new file mode 100644 index 00000000000..23e681e6fc7 --- /dev/null +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_validation.go @@ -0,0 +1,53 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Validate the Azure Managed Control Plane Template and return an aggregate error. +func (mcp *AzureManagedControlPlaneTemplate) validateManagedControlPlaneTemplate(cli client.Client) error { + var allErrs field.ErrorList + + allErrs = append(allErrs, validateDNSServiceIP( + mcp.Spec.Template.Spec.DNSServiceIP, + field.NewPath("spec").Child("template").Child("spec").Child("DNSServiceIP"))...) + + allErrs = append(allErrs, validateVersion( + mcp.Spec.Template.Spec.Version, + field.NewPath("spec").Child("template").Child("spec").Child("Version"))...) + + allErrs = append(allErrs, validateLoadBalancerProfile( + mcp.Spec.Template.Spec.LoadBalancerProfile, + field.NewPath("spec").Child("template").Child("spec").Child("LoadBalancerProfile"))...) + + allErrs = append(allErrs, validateManagedClusterNetwork( + cli, + mcp.Labels, + mcp.Namespace, + mcp.Spec.Template.Spec.DNSServiceIP, + mcp.Spec.Template.Spec.VirtualNetwork.Subnet, + field.NewPath("spec").Child("template").Child("spec"))...) + + allErrs = append(allErrs, validateName(mcp.Name, field.NewPath("Name"))...) + + allErrs = append(allErrs, validateAutoScalerProfile(mcp.Spec.Template.Spec.AutoScalerProfile, field.NewPath("spec").Child("template").Child("spec").Child("AutoScalerProfile"))...) + + return allErrs.ToAggregate() +} diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go new file mode 100644 index 00000000000..05936f0a8bd --- /dev/null +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook.go @@ -0,0 +1,82 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "reflect" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/cluster-api-provider-azure/feature" + capifeature "sigs.k8s.io/cluster-api/feature" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// AzureManagedControlPlaneTemplateImmutableMsg is the message used for errors on fields that are immutable. +const AzureManagedControlPlaneTemplateImmutableMsg = "AzureManagedControlPlaneTemplate spec.template.spec field is immutable. Please create new resource instead. ref doc: https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/change-clusterclass.html" + +// SetupWebhookWithManager will set up the webhook to be managed by the specified manager. +func (mcp *AzureManagedControlPlaneTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(mcp). + Complete() +} + +// +kubebuilder:webhook:verbs=create;update,path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplanetemplate,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=infrastructure.cluster.x-k8s.io,resources=azuremanagedcontrolplanetemplates,versions=v1beta1,name=validation.azuremanagedcontrolplanetemplate.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// +kubebuilder:webhook:verbs=create;update,path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplanetemplate,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=infrastructure.cluster.x-k8s.io,resources=azuremanagedcontrolplanetemplates,versions=v1beta1,name=default.azuremanagedcontrolplanetemplate.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 + +// Default implements webhook.Defaulter so a webhook will be registered for the type. +func (mcp *AzureManagedControlPlaneTemplate) Default(_ client.Client) { + mcp.setDefaults() +} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (mcp *AzureManagedControlPlaneTemplate) ValidateCreate(client client.Client) error { + // NOTE: AzureManagedControlPlane relies upon MachinePools, which is behind a feature gate flag. + // The webhook must prevent creating new objects in case the feature flag is disabled. + if !feature.Gates.Enabled(capifeature.MachinePool) { + return field.Forbidden( + field.NewPath("spec"), + "can be set only if the Cluster API 'MachinePool' feature flag is enabled", + ) + } + + return mcp.validateManagedControlPlaneTemplate(client) +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (mcp *AzureManagedControlPlaneTemplate) ValidateUpdate(oldRaw runtime.Object, _ client.Client) error { + var allErrs field.ErrorList + old := oldRaw.(*AzureManagedControlPlaneTemplate) + if !reflect.DeepEqual(mcp.Spec.Template.Spec, old.Spec.Template.Spec) { + allErrs = append(allErrs, + field.Invalid(field.NewPath("AzureManagedControlPlaneTemplate", "spec", "template", "spec"), mcp, AzureManagedControlPlaneTemplateImmutableMsg), + ) + } + + if len(allErrs) == 0 { + return nil + } + return apierrors.NewInvalid(GroupVersion.WithKind("AzureManagedControlPlaneTemplate").GroupKind(), mcp.Name, allErrs) +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (mcp *AzureManagedControlPlaneTemplate) ValidateDelete(_ client.Client) error { + return nil +} diff --git a/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go new file mode 100644 index 00000000000..b258bb19c85 --- /dev/null +++ b/api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go @@ -0,0 +1,138 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/component-base/featuregate/testing" + "k8s.io/utils/pointer" + "sigs.k8s.io/cluster-api-provider-azure/feature" + capifeature "sigs.k8s.io/cluster-api/feature" +) + +func TestAzureManagedControlPlaneTemplate_ValidateCreate(t *testing.T) { + // NOTE: AzureManageControlPlaneTemplate is behind AKS feature gate flag; the webhook + // must prevent creating new objects in case the feature flag is disabled. + defer utilfeature.SetFeatureGateDuringTest(t, feature.Gates, capifeature.MachinePool, true)() + g := NewWithT(t) + + tests := []struct { + name string + amcpt *AzureManagedControlPlaneTemplate + wantErr bool + }{ + { + name: "all valid", + amcpt: getKnownValidAzureManagedControlPlaneTemplate(), + wantErr: false, + }, + { + name: "invalid DNSServiceIP", + amcpt: createAzureManagedControlPlaneTemplate(metav1.ObjectMeta{}, "192.168.0.0.3", "v1.18.0"), + wantErr: true, + }, + { + name: "invalid version", + amcpt: createAzureManagedControlPlaneTemplate(metav1.ObjectMeta{}, "192.168.0.0", "honk.version"), + wantErr: true, + }, + { + name: "invalid name with microsoft", + amcpt: createAzureManagedControlPlaneTemplate(metav1.ObjectMeta{Name: "microsoft-cluster"}, "192.168.0.0", "v1.18.0"), + wantErr: true, + }, + { + name: "invalid name with windows", + amcpt: createAzureManagedControlPlaneTemplate(metav1.ObjectMeta{Name: "a-windows-cluster"}, "192.168.0.0", "v1.18.0"), + wantErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := tc.amcpt.ValidateCreate(nil) + if tc.wantErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} + +func TestAzureManagedControlPlaneTemplate_ValidateUpdate(t *testing.T) { + + oldManagedControlPlaneTemplate := &AzureManagedControlPlaneTemplate{ + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + NetworkPolicy: pointer.String("azure"), + }, + }, + }, + } + + newManagedControlPlaneTemplate := &AzureManagedControlPlaneTemplate{ + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + NetworkPolicy: pointer.String("calico"), + }, + }, + }, + } + + t.Run("template is immutable", func(t *testing.T) { + g := NewWithT(t) + g.Expect(newManagedControlPlaneTemplate.ValidateUpdate(oldManagedControlPlaneTemplate, nil)).NotTo(Succeed()) + }) +} + +func createAzureManagedControlPlaneTemplate(objectMeta metav1.ObjectMeta, serviceIP, version string) *AzureManagedControlPlaneTemplate { + return &AzureManagedControlPlaneTemplate{ + ObjectMeta: objectMeta, + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + DNSServiceIP: pointer.String(serviceIP), + Version: version, + }, + }, + }, + } +} + +func getKnownValidAzureManagedControlPlaneTemplate() *AzureManagedControlPlaneTemplate { + return &AzureManagedControlPlaneTemplate{ + Spec: AzureManagedControlPlaneTemplateSpec{ + Template: AzureManagedControlPlaneTemplateResource{ + Spec: AzureManagedControlPlaneTemplateResourceSpec{ + DNSServiceIP: pointer.String("192.168.0.0"), + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{ + "616077a8-5db7-4c98-b856-b34619afg75h", + }, + }, + }, + }, + }, + } +} diff --git a/api/v1beta1/types_template.go b/api/v1beta1/types_template.go index 010be4700e5..bd6ce649a36 100644 --- a/api/v1beta1/types_template.go +++ b/api/v1beta1/types_template.go @@ -16,7 +16,109 @@ limitations under the License. package v1beta1 -import "github.com/pkg/errors" +import ( + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" +) + +// AzureManagedControlPlaneTemplateResourceSpec specifies an Azure managed control plane template resource. +type AzureManagedControlPlaneTemplateResourceSpec struct { + // Version defines the desired Kubernetes version. + // +kubebuilder:validation:MinLength:=2 + Version string `json:"version"` + + // VirtualNetwork describes the vnet for the AKS cluster. Will be created if it does not exist. + // +optional + VirtualNetwork ManagedControlPlaneVirtualNetworkTemplate `json:"virtualNetwork,omitempty"` + + // SubscriptionID is the GUID of the Azure subscription to hold this cluster. + // +optional + SubscriptionID string `json:"subscriptionID,omitempty"` + + // Location is a string matching one of the canonical Azure region names. Examples: "westus2", "eastus". + Location string `json:"location"` + + // AdditionalTags is an optional set of tags to add to Azure resources managed by the Azure provider, in addition to the + // ones added by default. + // +optional + AdditionalTags Tags `json:"additionalTags,omitempty"` + + // NetworkPlugin used for building Kubernetes network. + // +kubebuilder:validation:Enum=azure;kubenet + // +optional + NetworkPlugin *string `json:"networkPlugin,omitempty"` + + // NetworkPolicy used for building Kubernetes network. + // +kubebuilder:validation:Enum=azure;calico + // +optional + NetworkPolicy *string `json:"networkPolicy,omitempty"` + + // Outbound configuration used by Nodes. + // +kubebuilder:validation:Enum=loadBalancer;managedNATGateway;userAssignedNATGateway;userDefinedRouting + // +optional + OutboundType *ManagedControlPlaneOutboundType `json:"outboundType,omitempty"` + + // DNSServiceIP is an IP address assigned to the Kubernetes DNS service. + // It must be within the Kubernetes service address range specified in serviceCidr. + // +optional + DNSServiceIP *string `json:"dnsServiceIP,omitempty"` + + // LoadBalancerSKU is the SKU of the loadBalancer to be provisioned. + // +kubebuilder:validation:Enum=Basic;Standard + // +optional + LoadBalancerSKU *string `json:"loadBalancerSKU,omitempty"` + + // IdentityRef is a reference to a AzureClusterIdentity to be used when reconciling this cluster + // +optional + IdentityRef *corev1.ObjectReference `json:"identityRef,omitempty"` + + // AadProfile is Azure Active Directory configuration to integrate with AKS for aad authentication. + // +optional + AADProfile *AADProfile `json:"aadProfile,omitempty"` + + // AddonProfiles are the profiles of managed cluster add-on. + // +optional + AddonProfiles []AddonProfile `json:"addonProfiles,omitempty"` + + // SKU is the SKU of the AKS to be provisioned. + // +optional + SKU *AKSSku `json:"sku,omitempty"` + + // LoadBalancerProfile is the profile of the cluster load balancer. + // +optional + LoadBalancerProfile *LoadBalancerProfile `json:"loadBalancerProfile,omitempty"` + + // APIServerAccessProfile is the access profile for AKS API server. + // +optional + APIServerAccessProfile *APIServerAccessProfileTemplate `json:"apiServerAccessProfile,omitempty"` + + // AutoscalerProfile is the parameters to be applied to the cluster-autoscaler when enabled + // +optional + AutoScalerProfile *AutoScalerProfile `json:"autoscalerProfile,omitempty"` +} + +type APIServerAccessProfileTemplate struct { + // EnablePrivateCluster - Whether to create the cluster as a private cluster or not. + // +optional + EnablePrivateCluster *bool `json:"enablePrivateCluster,omitempty"` + // PrivateDNSZone - Private dns zone mode for private cluster. + // +kubebuilder:validation:Enum=System;None + // +optional + PrivateDNSZone *string `json:"privateDNSZone,omitempty"` + // EnablePrivateClusterPublicFQDN - Whether to create additional public FQDN for private cluster or not. + // +optional + EnablePrivateClusterPublicFQDN *bool `json:"enablePrivateClusterPublicFQDN,omitempty"` +} + +type ManagedControlPlaneVirtualNetworkTemplate struct { + Name string `json:"name"` + CIDRBlock string `json:"cidrBlock"` + // +optional + Subnet ManagedControlPlaneSubnet `json:"subnet,omitempty"` +} + +// AzureManagedClusterTemplateResourceSpec specifies an Azure managed cluster template resource. +type AzureManagedClusterTemplateResourceSpec struct{} // AzureClusterTemplateResourceSpec specifies an Azure cluster template resource. type AzureClusterTemplateResourceSpec struct { diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 8d24ee954fc..e86be9b6271 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -99,6 +99,36 @@ func (in *APIServerAccessProfile) DeepCopy() *APIServerAccessProfile { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIServerAccessProfileTemplate) DeepCopyInto(out *APIServerAccessProfileTemplate) { + *out = *in + if in.EnablePrivateCluster != nil { + in, out := &in.EnablePrivateCluster, &out.EnablePrivateCluster + *out = new(bool) + **out = **in + } + if in.PrivateDNSZone != nil { + in, out := &in.PrivateDNSZone, &out.PrivateDNSZone + *out = new(string) + **out = **in + } + if in.EnablePrivateClusterPublicFQDN != nil { + in, out := &in.EnablePrivateClusterPublicFQDN, &out.EnablePrivateClusterPublicFQDN + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIServerAccessProfileTemplate. +func (in *APIServerAccessProfileTemplate) DeepCopy() *APIServerAccessProfileTemplate { + if in == nil { + return nil + } + out := new(APIServerAccessProfileTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AdditionalCapabilities) DeepCopyInto(out *AdditionalCapabilities) { *out = *in @@ -1089,6 +1119,111 @@ func (in *AzureManagedClusterStatus) DeepCopy() *AzureManagedClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedClusterTemplate) DeepCopyInto(out *AzureManagedClusterTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedClusterTemplate. +func (in *AzureManagedClusterTemplate) DeepCopy() *AzureManagedClusterTemplate { + if in == nil { + return nil + } + out := new(AzureManagedClusterTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AzureManagedClusterTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedClusterTemplateList) DeepCopyInto(out *AzureManagedClusterTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AzureManagedClusterTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedClusterTemplateList. +func (in *AzureManagedClusterTemplateList) DeepCopy() *AzureManagedClusterTemplateList { + if in == nil { + return nil + } + out := new(AzureManagedClusterTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AzureManagedClusterTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedClusterTemplateResource) DeepCopyInto(out *AzureManagedClusterTemplateResource) { + *out = *in + out.Spec = in.Spec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedClusterTemplateResource. +func (in *AzureManagedClusterTemplateResource) DeepCopy() *AzureManagedClusterTemplateResource { + if in == nil { + return nil + } + out := new(AzureManagedClusterTemplateResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedClusterTemplateResourceSpec) DeepCopyInto(out *AzureManagedClusterTemplateResourceSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedClusterTemplateResourceSpec. +func (in *AzureManagedClusterTemplateResourceSpec) DeepCopy() *AzureManagedClusterTemplateResourceSpec { + if in == nil { + return nil + } + out := new(AzureManagedClusterTemplateResourceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedClusterTemplateSpec) DeepCopyInto(out *AzureManagedClusterTemplateSpec) { + *out = *in + out.Template = in.Template +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedClusterTemplateSpec. +func (in *AzureManagedClusterTemplateSpec) DeepCopy() *AzureManagedClusterTemplateSpec { + if in == nil { + return nil + } + out := new(AzureManagedClusterTemplateSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureManagedControlPlane) DeepCopyInto(out *AzureManagedControlPlane) { *out = *in @@ -1261,6 +1396,181 @@ func (in *AzureManagedControlPlaneStatus) DeepCopy() *AzureManagedControlPlaneSt return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedControlPlaneTemplate) DeepCopyInto(out *AzureManagedControlPlaneTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneTemplate. +func (in *AzureManagedControlPlaneTemplate) DeepCopy() *AzureManagedControlPlaneTemplate { + if in == nil { + return nil + } + out := new(AzureManagedControlPlaneTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AzureManagedControlPlaneTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedControlPlaneTemplateList) DeepCopyInto(out *AzureManagedControlPlaneTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AzureManagedControlPlaneTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneTemplateList. +func (in *AzureManagedControlPlaneTemplateList) DeepCopy() *AzureManagedControlPlaneTemplateList { + if in == nil { + return nil + } + out := new(AzureManagedControlPlaneTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AzureManagedControlPlaneTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedControlPlaneTemplateResource) DeepCopyInto(out *AzureManagedControlPlaneTemplateResource) { + *out = *in + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneTemplateResource. +func (in *AzureManagedControlPlaneTemplateResource) DeepCopy() *AzureManagedControlPlaneTemplateResource { + if in == nil { + return nil + } + out := new(AzureManagedControlPlaneTemplateResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedControlPlaneTemplateResourceSpec) DeepCopyInto(out *AzureManagedControlPlaneTemplateResourceSpec) { + *out = *in + in.VirtualNetwork.DeepCopyInto(&out.VirtualNetwork) + if in.AdditionalTags != nil { + in, out := &in.AdditionalTags, &out.AdditionalTags + *out = make(Tags, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NetworkPlugin != nil { + in, out := &in.NetworkPlugin, &out.NetworkPlugin + *out = new(string) + **out = **in + } + if in.NetworkPolicy != nil { + in, out := &in.NetworkPolicy, &out.NetworkPolicy + *out = new(string) + **out = **in + } + if in.OutboundType != nil { + in, out := &in.OutboundType, &out.OutboundType + *out = new(ManagedControlPlaneOutboundType) + **out = **in + } + if in.DNSServiceIP != nil { + in, out := &in.DNSServiceIP, &out.DNSServiceIP + *out = new(string) + **out = **in + } + if in.LoadBalancerSKU != nil { + in, out := &in.LoadBalancerSKU, &out.LoadBalancerSKU + *out = new(string) + **out = **in + } + if in.IdentityRef != nil { + in, out := &in.IdentityRef, &out.IdentityRef + *out = new(corev1.ObjectReference) + **out = **in + } + if in.AADProfile != nil { + in, out := &in.AADProfile, &out.AADProfile + *out = new(AADProfile) + (*in).DeepCopyInto(*out) + } + if in.AddonProfiles != nil { + in, out := &in.AddonProfiles, &out.AddonProfiles + *out = make([]AddonProfile, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SKU != nil { + in, out := &in.SKU, &out.SKU + *out = new(AKSSku) + **out = **in + } + if in.LoadBalancerProfile != nil { + in, out := &in.LoadBalancerProfile, &out.LoadBalancerProfile + *out = new(LoadBalancerProfile) + (*in).DeepCopyInto(*out) + } + if in.APIServerAccessProfile != nil { + in, out := &in.APIServerAccessProfile, &out.APIServerAccessProfile + *out = new(APIServerAccessProfileTemplate) + (*in).DeepCopyInto(*out) + } + if in.AutoScalerProfile != nil { + in, out := &in.AutoScalerProfile, &out.AutoScalerProfile + *out = new(AutoScalerProfile) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneTemplateResourceSpec. +func (in *AzureManagedControlPlaneTemplateResourceSpec) DeepCopy() *AzureManagedControlPlaneTemplateResourceSpec { + if in == nil { + return nil + } + out := new(AzureManagedControlPlaneTemplateResourceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureManagedControlPlaneTemplateSpec) DeepCopyInto(out *AzureManagedControlPlaneTemplateSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneTemplateSpec. +func (in *AzureManagedControlPlaneTemplateSpec) DeepCopy() *AzureManagedControlPlaneTemplateSpec { + if in == nil { + return nil + } + out := new(AzureManagedControlPlaneTemplateSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureManagedMachinePool) DeepCopyInto(out *AzureManagedMachinePool) { *out = *in @@ -2131,6 +2441,22 @@ func (in *ManagedControlPlaneVirtualNetwork) DeepCopy() *ManagedControlPlaneVirt return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ManagedControlPlaneVirtualNetworkTemplate) DeepCopyInto(out *ManagedControlPlaneVirtualNetworkTemplate) { + *out = *in + in.Subnet.DeepCopyInto(&out.Subnet) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagedControlPlaneVirtualNetworkTemplate. +func (in *ManagedControlPlaneVirtualNetworkTemplate) DeepCopy() *ManagedControlPlaneVirtualNetworkTemplate { + if in == nil { + return nil + } + out := new(ManagedControlPlaneVirtualNetworkTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ManagedDiskParameters) DeepCopyInto(out *ManagedDiskParameters) { *out = *in diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedclustertemplates.yaml new file mode 100644 index 00000000000..cbde332f327 --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedclustertemplates.yaml @@ -0,0 +1,58 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: azuremanagedclustertemplates.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureManagedClusterTemplate + listKind: AzureManagedClusterTemplateList + plural: azuremanagedclustertemplates + singular: azuremanagedclustertemplate + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureManagedClusterTemplate is the Schema for the AzureManagedClusterTemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedClusterTemplateSpec defines the desired state + of AzureManagedClusterTemplate. + properties: + template: + description: AzureManagedClusterTemplateResource describes the data + needed to create an AzureManagedCluster from a template. + properties: + spec: + description: AzureManagedClusterTemplateResourceSpec specifies + an Azure managed cluster template resource. + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml new file mode 100644 index 00000000000..e9e7a620189 --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml @@ -0,0 +1,506 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: azuremanagedcontrolplanetemplates.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureManagedControlPlaneTemplate + listKind: AzureManagedControlPlaneTemplateList + plural: azuremanagedcontrolplanetemplates + singular: azuremanagedcontrolplanetemplate + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureManagedControlPlaneTemplate is the Schema for the AzureManagedControlPlaneTemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedControlPlaneTemplateSpec defines the desired + state of AzureManagedControlPlaneTemplate. + properties: + template: + description: AzureManagedControlPlaneTemplateResource describes the + data needed to create an AzureManagedCluster from a template. + properties: + spec: + description: AzureManagedControlPlaneTemplateResourceSpec specifies + an Azure managed control plane template resource. + properties: + aadProfile: + description: AadProfile is Azure Active Directory configuration + to integrate with AKS for aad authentication. + properties: + adminGroupObjectIDs: + description: AdminGroupObjectIDs - AAD group object IDs + that will have admin role of the cluster. + items: + type: string + type: array + managed: + description: Managed - Whether to enable managed AAD. + type: boolean + required: + - adminGroupObjectIDs + - managed + type: object + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to Azure resources managed by the Azure provider, in + addition to the ones added by default. + type: object + addonProfiles: + description: AddonProfiles are the profiles of managed cluster + add-on. + items: + description: AddonProfile represents a managed cluster add-on. + properties: + config: + additionalProperties: + type: string + description: Config - Key-value pairs for configuring + the add-on. + type: object + enabled: + description: Enabled - Whether the add-on is enabled + or not. + type: boolean + name: + description: Name - The name of the managed cluster + add-on. + type: string + required: + - enabled + - name + type: object + type: array + apiServerAccessProfile: + description: APIServerAccessProfile is the access profile + for AKS API server. + properties: + enablePrivateCluster: + description: EnablePrivateCluster - Whether to create + the cluster as a private cluster or not. + type: boolean + enablePrivateClusterPublicFQDN: + description: EnablePrivateClusterPublicFQDN - Whether + to create additional public FQDN for private cluster + or not. + type: boolean + privateDNSZone: + description: PrivateDNSZone - Private dns zone mode for + private cluster. + enum: + - System + - None + type: string + type: object + autoscalerProfile: + description: AutoscalerProfile is the parameters to be applied + to the cluster-autoscaler when enabled + properties: + balanceSimilarNodeGroups: + description: BalanceSimilarNodeGroups - Valid values are + 'true' and 'false'. The default is false. + enum: + - "true" + - "false" + type: string + expander: + description: Expander - If not specified, the default + is 'random'. See [expanders](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-expanders) + for more information. + enum: + - least-waste + - most-pods + - priority + - random + type: string + maxEmptyBulkDelete: + description: MaxEmptyBulkDelete - The default is 10. + type: string + maxGracefulTerminationSec: + description: MaxGracefulTerminationSec - The default is + 600. + pattern: ^(\d+)$ + type: string + maxNodeProvisionTime: + description: MaxNodeProvisionTime - The default is '15m'. + Values must be an integer followed by an 'm'. No unit + of time other than minutes (m) is supported. + pattern: ^(\d+)m$ + type: string + maxTotalUnreadyPercentage: + description: MaxTotalUnreadyPercentage - The default is + 45. The maximum is 100 and the minimum is 0. + maxLength: 3 + minLength: 1 + pattern: ^(\d+)$ + type: string + newPodScaleUpDelay: + description: NewPodScaleUpDelay - For scenarios like burst/batch + scale where you don't want CA to act before the kubernetes + scheduler could schedule all the pods, you can tell + CA to ignore unscheduled pods before they're a certain + age. The default is '0s'. Values must be an integer + followed by a unit ('s' for seconds, 'm' for minutes, + 'h' for hours, etc). + type: string + okTotalUnreadyCount: + description: OkTotalUnreadyCount - This must be an integer. + The default is 3. + pattern: ^(\d+)$ + type: string + scaleDownDelayAfterAdd: + description: ScaleDownDelayAfterAdd - The default is '10m'. + Values must be an integer followed by an 'm'. No unit + of time other than minutes (m) is supported. + pattern: ^(\d+)m$ + type: string + scaleDownDelayAfterDelete: + description: ScaleDownDelayAfterDelete - The default is + the scan-interval. Values must be an integer followed + by an 's'. No unit of time other than seconds (s) is + supported. + pattern: ^(\d+)s$ + type: string + scaleDownDelayAfterFailure: + description: ScaleDownDelayAfterFailure - The default + is '3m'. Values must be an integer followed by an 'm'. + No unit of time other than minutes (m) is supported. + pattern: ^(\d+)m$ + type: string + scaleDownUnneededTime: + description: ScaleDownUnneededTime - The default is '10m'. + Values must be an integer followed by an 'm'. No unit + of time other than minutes (m) is supported. + pattern: ^(\d+)m$ + type: string + scaleDownUnreadyTime: + description: ScaleDownUnreadyTime - The default is '20m'. + Values must be an integer followed by an 'm'. No unit + of time other than minutes (m) is supported. + pattern: ^(\d+)m$ + type: string + scaleDownUtilizationThreshold: + description: ScaleDownUtilizationThreshold - The default + is '0.5'. + type: string + scanInterval: + description: ScanInterval - How often cluster is reevaluated + for scale up or down. The default is '10s'. + pattern: ^(\d+)s$ + type: string + skipNodesWithLocalStorage: + description: SkipNodesWithLocalStorage - The default is + false. + enum: + - "true" + - "false" + type: string + skipNodesWithSystemPods: + description: SkipNodesWithSystemPods - The default is + true. + enum: + - "true" + - "false" + type: string + type: object + dnsServiceIP: + description: DNSServiceIP is an IP address assigned to the + Kubernetes DNS service. It must be within the Kubernetes + service address range specified in serviceCidr. + type: string + identityRef: + description: IdentityRef is a reference to a AzureClusterIdentity + to be used when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this + pod). This syntax is chosen only to have some well-defined + way of referencing a part of an object. TODO: this design + is not final and this field is subject to change in + the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + loadBalancerProfile: + description: LoadBalancerProfile is the profile of the cluster + load balancer. + properties: + allocatedOutboundPorts: + description: AllocatedOutboundPorts - Desired number of + allocated SNAT ports per VM. Allowed values must be + in the range of 0 to 64000 (inclusive). The default + value is 0 which results in Azure dynamically allocating + ports. + format: int32 + type: integer + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes - Desired outbound flow + idle timeout in minutes. Allowed values must be in the + range of 4 to 120 (inclusive). The default value is + 30 minutes. + format: int32 + type: integer + managedOutboundIPs: + description: ManagedOutboundIPs - Desired managed outbound + IPs for the cluster load balancer. + format: int32 + type: integer + outboundIPPrefixes: + description: OutboundIPPrefixes - Desired outbound IP + Prefix resources for the cluster load balancer. + items: + type: string + type: array + outboundIPs: + description: OutboundIPs - Desired outbound IP resources + for the cluster load balancer. + items: + type: string + type: array + type: object + loadBalancerSKU: + description: LoadBalancerSKU is the SKU of the loadBalancer + to be provisioned. + enum: + - Basic + - Standard + type: string + location: + description: 'Location is a string matching one of the canonical + Azure region names. Examples: "westus2", "eastus".' + type: string + networkPlugin: + description: NetworkPlugin used for building Kubernetes network. + enum: + - azure + - kubenet + type: string + networkPolicy: + description: NetworkPolicy used for building Kubernetes network. + enum: + - azure + - calico + type: string + outboundType: + description: Outbound configuration used by Nodes. + enum: + - loadBalancer + - managedNATGateway + - userAssignedNATGateway + - userDefinedRouting + type: string + sku: + description: SKU is the SKU of the AKS to be provisioned. + properties: + tier: + description: Tier - Tier of an AKS cluster. + enum: + - Free + - Paid + type: string + required: + - tier + type: object + subscriptionID: + description: SubscriptionID is the GUID of the Azure subscription + to hold this cluster. + type: string + version: + description: Version defines the desired Kubernetes version. + minLength: 2 + type: string + virtualNetwork: + description: VirtualNetwork describes the vnet for the AKS + cluster. Will be created if it does not exist. + properties: + cidrBlock: + type: string + name: + type: string + subnet: + description: ManagedControlPlaneSubnet describes a subnet + for an AKS cluster. + properties: + cidrBlock: + type: string + name: + type: string + privateEndpoints: + description: PrivateEndpoints is a slice of Virtual + Network private endpoints to create for the subnets. + items: + description: PrivateEndpointSpec configures an Azure + Private Endpoint. + properties: + applicationSecurityGroups: + description: ApplicationSecurityGroups specifies + the Application security group in which the + private endpoint IP configuration is included. + items: + type: string + type: array + customNetworkInterfaceName: + description: CustomNetworkInterfaceName specifies + the network interface name associated with + the private endpoint. + type: string + location: + description: Location specifies the region to + create the private endpoint. + type: string + manualApproval: + description: ManualApproval specifies if the + connection approval needs to be done manually + or not. Set it true when the network admin + does not have access to approve connections + to the remote resource. Defaults to false. + type: boolean + name: + description: Name specifies the name of the + private endpoint. + type: string + privateIPAddresses: + description: PrivateIPAddresses specifies the + IP addresses for the network interface associated + with the private endpoint. They have to be + part of the subnet where the private endpoint + is linked. + items: + type: string + type: array + privateLinkServiceConnections: + description: PrivateLinkServiceConnections specifies + Private Link Service Connections of the private + endpoint. + items: + description: PrivateLinkServiceConnection + defines the specification for a private + link service connection associated with + a private endpoint. + properties: + groupIDs: + description: GroupIDs specifies the ID(s) + of the group(s) obtained from the remote + resource that this private endpoint + should connect to. + items: + type: string + type: array + name: + description: Name specifies the name of + the private link service. + type: string + privateLinkServiceID: + description: PrivateLinkServiceID specifies + the resource ID of the private link + service. + type: string + requestMessage: + description: RequestMessage specifies + a message passed to the owner of the + remote resource with the private endpoint + connection request. + maxLength: 140 + type: string + type: object + type: array + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + serviceEndpoints: + description: ServiceEndpoints is a slice of Virtual + Network service endpoints to enable for the subnets. + items: + description: ServiceEndpointSpec configures an Azure + Service Endpoint. + properties: + locations: + items: + type: string + type: array + service: + type: string + required: + - locations + - service + type: object + type: array + x-kubernetes-list-map-keys: + - service + x-kubernetes-list-type: map + required: + - cidrBlock + - name + type: object + required: + - cidrBlock + - name + type: object + required: + - location + - version + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 5b81638d0c3..021c7b58c50 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -114,6 +114,28 @@ webhooks: resources: - azuremanagedcontrolplanes sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplanetemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azuremanagedcontrolplanetemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedcontrolplanetemplates + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 @@ -294,6 +316,26 @@ webhooks: resources: - azuremanagedclusters sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedclustertemplate + failurePolicy: Fail + name: validation.azuremanagedclustertemplates.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - UPDATE + resources: + - azuremanagedclustertemplates + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 @@ -315,6 +357,28 @@ webhooks: resources: - azuremanagedcontrolplanes sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplanetemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.azuremanagedcontrolplanetemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedcontrolplanetemplates + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 diff --git a/docs/book/src/topics/clusterclass.md b/docs/book/src/topics/clusterclass.md new file mode 100644 index 00000000000..326f82015a1 --- /dev/null +++ b/docs/book/src/topics/clusterclass.md @@ -0,0 +1,88 @@ +# ClusterClass + +- **Feature status:** GA +- **Feature gate:** MachinePool=true ClusterTopology=true + +[ClusterClass](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/index.html) is a collection of templates that define a topology (control plane and machine deployments) to be used to continuously reconcile one or more Clusters. It is a new Cluster API feature that is built on top of the existing Cluster API resources and provides a set of tools and operations to streamline cluster lifecycle management while maintaining the same underlying API. + +CAPZ currently supports ClusterClass for both managed (AKS) and self-managed clusters. CAPZ implements this with three custom resources: +1. AzureClusterTemplate +2. AzureManagedClusterTemplate +3. AzureManagedControlPlaneTemplate + +Each resource is a template for the corresponding CAPZ resource. For example, the AzureClusterTemplate is a template for the CAPZ AzureCluster resource. The template contains a set of parameters that are able to be shared across multiple clusters. + +## Deploying a Self-Managed Cluster with ClusterClass + +To deploy a self-managed cluster with ClusterClass, you must first create a ClusterClass resource. The ClusterClass resource defines the cluster topology, including the control plane and machine deployment templates. The ClusterClass resource also defines the parameters that can be used to customize the cluster topology. + +Please refer to the Cluster API book for more information on how to write a ClusterClass topology: https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/write-clusterclass.html + +For a self-managed cluster, the AzureClusterTemplate is used to define the Azure infrastructure for the cluster. The following example shows a basic AzureClusterTemplate resource: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterTemplate +metadata: + name: capz-clusterclass-cluster + namespace: default +spec: + template: + spec: + location: westus2 + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + natGateway: + name: node-natgateway + role: node + subscriptionID: 00000000-0000-0000-0000-000000000000 +``` + +## Deploying a Managed Cluster (AKS) with ClusterClass + +Deploying an AKS cluster with ClusterClass is similar to deploying a self-managed cluster. Instead of using the AzureClusterTemplate, you must use both an AzureManagedClusterTemplate and AzureManagedControlPlaneTemplate. Due to the nature of managed Kubernetes and the control plane implementation, the infrastructure provider (and therefore the AzureManagedClusterTemplate) for AKS cluster is basically a no-op. The AzureManagedControlPlaneTemplate is used to define the AKS cluster configuration, such as the Kubernetes version and the number of nodes. + +The following example shows a basic AzureManagedClusterTemplate and AzureManagedControlPlaneTemplate resource: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedClusterTemplate +metadata: + name: capz-clusterclass-cluster +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: capz-clusterclass-control-plane +spec: + location: westus2 + subscriptionID: 00000000-0000-0000-0000-000000000000 + version: 1.25.2 +``` + +## Excluded Fields + +Since a ClusterClass is a template for a Cluster, there are some fields that are not allowed to be shared across multiple clusters. For each of the ClusterClass resources, the following fields are excluded: + +### AzureClusterTemplate +- `spec.resourceGroup` +- `spec.controlPlaneEndpoint` +- `spec.bastionSpec.azureBastion.name` +- `spec.bastionSpec.azureBastion.subnetSpec.routeTable` +- `spec.bastionSpec.azureBastion.publicIP` +- `spec.bastionSpec.azureBastion.sku` +- `spec.bastionSpec.azureBastion.enableTunneling` + +### AzureManagedControlPlaneTemplate + +- `spec.resourceGroupName` +- `spec.nodeResourceGroupName` +- `spec.virtualNetwork.name` +- `spec.virtualNetwork.subnet` +- `spec.virtualNetwork.resourceGroup` +- `spec.controlPlaneEndpoint` +- `spec.sshPublicKey` +- `spec.apiServerAccessProfile.authorizedIPRanges` diff --git a/main.go b/main.go index 6a16701a98b..871daa7b46c 100644 --- a/main.go +++ b/main.go @@ -512,6 +512,11 @@ func registerWebhooks(mgr manager.Manager) { os.Exit(1) } + if err := (&infrav1.AzureManagedClusterTemplate{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AzureManagedClusterTemplate") + os.Exit(1) + } + hookServer := mgr.GetWebhookServer() hookServer.Register("/mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinepool", webhookutils.NewMutatingWebhook( &infrav1exp.AzureMachinePool{}, mgr.GetClient(), @@ -537,6 +542,12 @@ func registerWebhooks(mgr manager.Manager) { hookServer.Register("/validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplane", webhookutils.NewValidatingWebhook( &infrav1.AzureManagedControlPlane{}, mgr.GetClient(), )) + hookServer.Register("/mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplanetemplate", webhookutils.NewMutatingWebhook( + &infrav1.AzureManagedControlPlaneTemplate{}, mgr.GetClient(), + )) + hookServer.Register("/validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplanetemplate", webhookutils.NewValidatingWebhook( + &infrav1.AzureManagedControlPlaneTemplate{}, mgr.GetClient(), + )) if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { setupLog.Error(err, "unable to create ready check") diff --git a/templates/cluster-template-aks-clusterclass.yaml b/templates/cluster-template-aks-clusterclass.yaml new file mode 100644 index 00000000000..76bc4273752 --- /dev/null +++ b/templates/cluster-template-aks-clusterclass.yaml @@ -0,0 +1,153 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: default + namespace: default +spec: + controlPlane: + machineInfrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePool + name: ${CLUSTER_NAME}-pool0 + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedControlPlaneTemplate + name: ${CLUSTER_NAME}-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedClusterTemplate + name: ${CLUSTER_NAME}-cluster + workers: + machinePools: + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool1 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePool + name: ${CLUSTER_NAME}-pool1 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: default + controlPlane: + replicas: 1 + version: ${KUBERNETES_VERSION} + workers: + machineDeployments: + - class: default-worker + name: md-0 + replicas: 1 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedClusterTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: default +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + template: + metadata: {} + spec: + bootstrap: + dataSecretName: "" + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePool + name: ${CLUSTER_NAME}-pool0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + template: + metadata: {} + spec: + bootstrap: + dataSecretName: "" + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePool + name: ${CLUSTER_NAME}-pool1 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePool +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + mode: System + name: pool0 + sku: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePool +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + mode: User + name: pool1 + sku: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal diff --git a/templates/cluster-template-aks.yaml b/templates/cluster-template-aks.yaml index 3f411230c1f..c4a882b94a5 100644 --- a/templates/cluster-template-aks.yaml +++ b/templates/cluster-template-aks.yaml @@ -59,16 +59,6 @@ spec: name: ${CLUSTER_NAME}-pool0 version: ${KUBERNETES_VERSION} --- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedMachinePool -metadata: - name: ${CLUSTER_NAME}-pool0 - namespace: default -spec: - mode: System - name: pool0 - sku: ${AZURE_NODE_MACHINE_TYPE} ---- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachinePool metadata: @@ -91,6 +81,16 @@ spec: --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureManagedMachinePool +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + mode: System + name: pool0 + sku: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePool metadata: name: ${CLUSTER_NAME}-pool1 namespace: default diff --git a/templates/flavors/aks-clusterclass/azure-managed-cluster-template.yaml b/templates/flavors/aks-clusterclass/azure-managed-cluster-template.yaml new file mode 100644 index 00000000000..b95b5701247 --- /dev/null +++ b/templates/flavors/aks-clusterclass/azure-managed-cluster-template.yaml @@ -0,0 +1,5 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedClusterTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: default \ No newline at end of file diff --git a/templates/flavors/aks-clusterclass/azure-managed-controlplane-template.yaml b/templates/flavors/aks-clusterclass/azure-managed-controlplane-template.yaml new file mode 100644 index 00000000000..61edd69ef6c --- /dev/null +++ b/templates/flavors/aks-clusterclass/azure-managed-controlplane-template.yaml @@ -0,0 +1,14 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} + version: ${KUBERNETES_VERSION} \ No newline at end of file diff --git a/templates/flavors/aks-clusterclass/cluster.yaml b/templates/flavors/aks-clusterclass/cluster.yaml new file mode 100644 index 00000000000..caa28e2c5e3 --- /dev/null +++ b/templates/flavors/aks-clusterclass/cluster.yaml @@ -0,0 +1,20 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: default + version: ${KUBERNETES_VERSION} + controlPlane: + replicas: 1 + workers: + machineDeployments: + - class: default-worker + name: md-0 + replicas: 1 \ No newline at end of file diff --git a/templates/flavors/aks-clusterclass/clusterclass.yaml b/templates/flavors/aks-clusterclass/clusterclass.yaml new file mode 100644 index 00000000000..7555668dce8 --- /dev/null +++ b/templates/flavors/aks-clusterclass/clusterclass.yaml @@ -0,0 +1,35 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: default + namespace: default +spec: + controlPlane: + machineInfrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePool + name: ${CLUSTER_NAME}-pool0 + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedControlPlaneTemplate + name: ${CLUSTER_NAME}-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedClusterTemplate + name: ${CLUSTER_NAME}-cluster + workers: + machinePools: + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool1 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePool + name: ${CLUSTER_NAME}-pool1 \ No newline at end of file diff --git a/templates/flavors/aks-clusterclass/kustomization.yaml b/templates/flavors/aks-clusterclass/kustomization.yaml new file mode 100644 index 00000000000..52e001c43bf --- /dev/null +++ b/templates/flavors/aks-clusterclass/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- clusterclass.yaml +- cluster.yaml +- azure-managed-controlplane-template.yaml +- azure-managed-cluster-template.yaml +- ../aks/machinepool.yaml +- ../aks/azure-managed-machinepool.yaml +- ../../azure-cluster-identity +patchesStrategicMerge: +- patches/managedazurecluster-identity-ref.yaml \ No newline at end of file diff --git a/templates/flavors/aks-clusterclass/patches/managedazurecluster-identity-ref.yaml b/templates/flavors/aks-clusterclass/patches/managedazurecluster-identity-ref.yaml new file mode 100644 index 00000000000..37d7ac29191 --- /dev/null +++ b/templates/flavors/aks-clusterclass/patches/managedazurecluster-identity-ref.yaml @@ -0,0 +1,9 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: "${CLUSTER_IDENTITY_NAME}" \ No newline at end of file diff --git a/templates/flavors/aks/azure-managed-cluster.yaml b/templates/flavors/aks/azure-managed-cluster.yaml new file mode 100644 index 00000000000..6dc2478fb60 --- /dev/null +++ b/templates/flavors/aks/azure-managed-cluster.yaml @@ -0,0 +1,7 @@ +# Due to the nature of managed Kubernetes and the control plane implementation, +# the infrastructure provider for AKS cluster is basically a no-op. +# It sets itself to ready as soon as it sees the control plane ready. +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedCluster +metadata: + name: ${CLUSTER_NAME} \ No newline at end of file diff --git a/templates/flavors/aks/azure-managed-controlplane.yaml b/templates/flavors/aks/azure-managed-controlplane.yaml new file mode 100644 index 00000000000..14036ffeb95 --- /dev/null +++ b/templates/flavors/aks/azure-managed-controlplane.yaml @@ -0,0 +1,13 @@ +# The control plane abstracts readiness and provisioning of an AKS cluster. +# Because AKS requires a default pool, this also requires a reference to the +# default machine pool. +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlane +metadata: + name: ${CLUSTER_NAME} +spec: + subscriptionID: ${AZURE_SUBSCRIPTION_ID} + resourceGroupName: "${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}" + location: "${AZURE_LOCATION}" + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + version: "${KUBERNETES_VERSION}" \ No newline at end of file diff --git a/templates/flavors/aks/azure-managed-machinepool.yaml b/templates/flavors/aks/azure-managed-machinepool.yaml new file mode 100644 index 00000000000..b4814dd6455 --- /dev/null +++ b/templates/flavors/aks/azure-managed-machinepool.yaml @@ -0,0 +1,21 @@ +# This first Azure-specific machine pool implementation drives the configuration of the +# AKS "System" node pool to schedule and run kube-system and other system pods +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePool +metadata: + name: "${CLUSTER_NAME}-pool0" +spec: + mode: System + sku: "${AZURE_NODE_MACHINE_TYPE}" + name: pool0 +--- +# This first Azure-specific machine pool implementation drives the configuration of the +# AKS "User" node pool to schedule and run user workloads +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePool +metadata: + name: "${CLUSTER_NAME}-pool1" +spec: + mode: User + sku: "${AZURE_NODE_MACHINE_TYPE}" + name: pool1 \ No newline at end of file diff --git a/templates/flavors/aks/cluster.yaml b/templates/flavors/aks/cluster.yaml new file mode 100644 index 00000000000..528f9e824e1 --- /dev/null +++ b/templates/flavors/aks/cluster.yaml @@ -0,0 +1,20 @@ +# The Cluster object is the top level owner of all resources. +# It coordinates between the control plane and the infrastructure/machines. +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + services: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedControlPlane + name: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedCluster + name: ${CLUSTER_NAME} \ No newline at end of file diff --git a/templates/flavors/aks/kustomization.yaml b/templates/flavors/aks/kustomization.yaml index c11bca8e2cf..efc4ea37aec 100644 --- a/templates/flavors/aks/kustomization.yaml +++ b/templates/flavors/aks/kustomization.yaml @@ -2,7 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: -- cluster-template.yaml +- cluster.yaml +- azure-managed-controlplane.yaml +- azure-managed-cluster.yaml +- machinepool.yaml +- azure-managed-machinepool.yaml - ../../azure-cluster-identity patchesStrategicMerge: - ../../azure-cluster-identity/managedazurecluster-identity-ref.yaml diff --git a/templates/flavors/aks/machinepool.yaml b/templates/flavors/aks/machinepool.yaml new file mode 100644 index 00000000000..a601b066d58 --- /dev/null +++ b/templates/flavors/aks/machinepool.yaml @@ -0,0 +1,41 @@ +# We provision a default machine pool with no bootstrap data (AKS will provide it). +# We specify an AzureManagedMachinePool as the infrastructure machine it, which +# will be reflected in Azure as VMSS node pools attached to an AKS cluster. +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: "${CLUSTER_NAME}-pool0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + template: + metadata: {} + spec: + bootstrap: + dataSecretName: "" + clusterName: "${CLUSTER_NAME}" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePool + name: "${CLUSTER_NAME}-pool0" + version: "${KUBERNETES_VERSION}" +--- +# Deploy a second agent pool with the same number of machines, but using potentially different infrastructure. +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: "${CLUSTER_NAME}-pool1" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + template: + metadata: {} + spec: + bootstrap: + dataSecretName: "" + clusterName: "${CLUSTER_NAME}" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePool + name: "${CLUSTER_NAME}-pool1" + version: "${KUBERNETES_VERSION}" \ No newline at end of file diff --git a/templates/test/ci/cluster-template-prow-aks.yaml b/templates/test/ci/cluster-template-prow-aks.yaml index 2d31ecbc4ae..0649b5cfe1e 100644 --- a/templates/test/ci/cluster-template-prow-aks.yaml +++ b/templates/test/ci/cluster-template-prow-aks.yaml @@ -66,24 +66,6 @@ spec: name: ${CLUSTER_NAME}-pool0 version: ${KUBERNETES_VERSION} --- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedMachinePool -metadata: - name: ${CLUSTER_NAME}-pool0 - namespace: default -spec: - availabilityZones: - - "1" - - "2" - enableNodePublicIP: false - enableUltraSSD: true - maxPods: 30 - mode: System - name: pool0 - osDiskSizeGB: 30 - osDiskType: Managed - sku: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3} ---- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachinePool metadata: @@ -106,6 +88,24 @@ spec: --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureManagedMachinePool +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + availabilityZones: + - "1" + - "2" + enableNodePublicIP: false + enableUltraSSD: true + maxPods: 30 + mode: System + name: pool0 + osDiskSizeGB: 30 + osDiskType: Managed + sku: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePool metadata: name: ${CLUSTER_NAME}-pool1 namespace: default diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index dc267aa8f77..9bd15b609b6 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -910,5 +910,57 @@ var _ = Describe("Workload cluster creation", func() { By("PASSED!") }) + + Context("Creating an aks cluster using clusterclass [Managed Kubernetes]", func() { + It("with a single control plane node and 1 node", func() { + // use "cc" as spec name because natgw pip name exceeds limit. + clusterName = getClusterName(clusterNamePrefix, aksClusterNameSuffix) + kubernetesVersionUpgradeFrom, err := GetAKSKubernetesVersion(ctx, e2eConfig, AKSKubernetesVersionUpgradeFrom) + Expect(err).To(BeNil()) + kubernetesVersion, err := GetAKSKubernetesVersion(ctx, e2eConfig, AKSKubernetesVersion) + Expect(err).To(BeNil()) + + // Create a cluster using the cluster class created above + clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( + specName, + withFlavor("aks-clusterclass"), + withNamespace(namespace.Name), + withClusterName(clusterName), + withKubernetesVersion(kubernetesVersionUpgradeFrom), + withControlPlaneMachineCount(1), + withWorkerMachineCount(1), + withMachineDeploymentInterval(specName, ""), + withMachinePoolInterval(specName, "wait-worker-nodes"), + withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ + WaitForControlPlaneInitialized: WaitForAKSControlPlaneInitialized, + WaitForControlPlaneMachinesReady: WaitForAKSControlPlaneReady, + }), + ), result) + + By("Upgrading the Kubernetes version of the cluster", func() { + AKSUpgradeSpec(ctx, func() AKSUpgradeSpecInput { + return AKSUpgradeSpecInput{ + Cluster: result.Cluster, + MachinePools: result.MachinePools, + KubernetesVersionUpgradeTo: kubernetesVersion, + WaitForControlPlane: e2eConfig.GetIntervals(specName, "wait-machine-upgrade"), + WaitForMachinePools: e2eConfig.GetIntervals(specName, "wait-machine-pool-upgrade"), + } + }) + }) + + By("Exercising machine pools", func() { + AKSMachinePoolSpec(ctx, func() AKSMachinePoolSpecInput { + return AKSMachinePoolSpecInput{ + Cluster: result.Cluster, + MachinePools: result.MachinePools, + WaitIntervals: e2eConfig.GetIntervals(specName, "wait-machine-pool-nodes"), + } + }) + }) + + By("PASSED!") + }) + }) }) })