Merge pull request kubernetes-sigs#4069 from nojnhuh/aso-managedclusters

convert managedclusters and agentpools to ASO
willie-yao · Nov 15, 2023 · d586453 · d586453
2 parents fc261f0 + 468dda3
commit d586453
Show file tree

Hide file tree

Showing 48 changed files with 25,017 additions and 3,581 deletions.
diff --git a/Makefile b/Makefile
@@ -158,7 +158,7 @@ WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook
 RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
 ASO_CRDS_PATH := $(MANIFEST_ROOT)/aso/crds.yaml
 ASO_VERSION := v2.3.0
-ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com
+ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com
 
 # Allow overriding the imagePullPolicy
 PULL_POLICY ?= Always
@@ -308,7 +308,7 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create
 
 	# Deploy CAPZ
 	$(KIND) load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=$(KIND_CLUSTER_NAME)
-	$(KUSTOMIZE) build config/default | $(ENVSUBST) | $(KUBECTL) apply -f -
+	$(KUSTOMIZE) build config/default | $(ENVSUBST) | $(KUBECTL) apply -f - --server-side=true
 
 	# Wait for CAPI deployments
 	$(KUBECTL) wait --for=condition=Available --timeout=5m -n capi-system deployment -l cluster.x-k8s.io/provider=cluster-api

diff --git a/api/v1beta1/azuremanagedcluster_webhook.go b/api/v1beta1/azuremanagedcluster_webhook.go
@@ -17,14 +17,9 @@ limitations under the License.
 package v1beta1
 
 import (
-	"fmt"
-	"reflect"
-
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"sigs.k8s.io/cluster-api-provider-azure/feature"
-	"sigs.k8s.io/cluster-api-provider-azure/util/maps"
 	capifeature "sigs.k8s.io/cluster-api/feature"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -57,24 +52,6 @@ func (r *AzureManagedCluster) ValidateCreate() (admission.Warnings, error) {
 
 // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type.
 func (r *AzureManagedCluster) ValidateUpdate(oldRaw runtime.Object) (admission.Warnings, error) {
-	old := oldRaw.(*AzureManagedCluster)
-	var allErrs field.ErrorList
-
-	// custom headers are immutable
-	oldCustomHeaders := maps.FilterByKeyPrefix(old.ObjectMeta.Annotations, CustomHeaderPrefix)
-	newCustomHeaders := maps.FilterByKeyPrefix(r.ObjectMeta.Annotations, CustomHeaderPrefix)
-	if !reflect.DeepEqual(oldCustomHeaders, newCustomHeaders) {
-		allErrs = append(allErrs,
-			field.Invalid(
-				field.NewPath("metadata", "annotations"),
-				r.ObjectMeta.Annotations,
-				fmt.Sprintf("annotations with '%s' prefix are immutable", CustomHeaderPrefix)))
-	}
-
-	if len(allErrs) != 0 {
-		return nil, apierrors.NewInvalid(GroupVersion.WithKind("AzureManagedCluster").GroupKind(), r.Name, allErrs)
-	}
-
 	return nil, nil
 }
 

diff --git a/api/v1beta1/azuremanagedcluster_webhook_test.go b/api/v1beta1/azuremanagedcluster_webhook_test.go
@@ -38,87 +38,6 @@ func TestAzureManagedCluster_ValidateUpdate(t *testing.T) {
 		amc     *AzureManagedCluster
 		wantErr bool
 	}{
-		{
-			name: "custom header annotation values are immutable",
-			oldAMC: &AzureManagedCluster{
-				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{
-						"infrastructure.cluster.x-k8s.io/custom-header-SomeFeature": "true",
-					},
-				},
-				Spec: AzureManagedClusterSpec{},
-			},
-			amc: &AzureManagedCluster{
-				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{
-						"infrastructure.cluster.x-k8s.io/custom-header-SomeFeature": "false",
-					},
-				},
-				Spec: AzureManagedClusterSpec{},
-			},
-			wantErr: true,
-		},
-		{
-			name: "custom header annotations cannot be removed after resource creation",
-			oldAMC: &AzureManagedCluster{
-				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{
-						"infrastructure.cluster.x-k8s.io/custom-header-SomeFeature": "true",
-					},
-				},
-				Spec: AzureManagedClusterSpec{},
-			},
-			amc: &AzureManagedCluster{
-				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{},
-				},
-				Spec: AzureManagedClusterSpec{},
-			},
-			wantErr: true,
-		},
-		{
-			name: "custom header annotations cannot be added after resource creation",
-			oldAMC: &AzureManagedCluster{
-				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{
-						"infrastructure.cluster.x-k8s.io/custom-header-SomeFeature": "true",
-					},
-				},
-				Spec: AzureManagedClusterSpec{},
-			},
-			amc: &AzureManagedCluster{
-				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{
-						"infrastructure.cluster.x-k8s.io/custom-header-SomeFeature":    "true",
-						"infrastructure.cluster.x-k8s.io/custom-header-AnotherFeature": "true",
-					},
-				},
-				Spec: AzureManagedClusterSpec{},
-			},
-			wantErr: true,
-		},
-		{
-			name: "non-custom header annotations are mutable",
-			oldAMC: &AzureManagedCluster{
-				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{
-						"annotation-a": "true",
-						"infrastructure.cluster.x-k8s.io/custom-header-SomeFeature": "true",
-					},
-				},
-				Spec: AzureManagedClusterSpec{},
-			},
-			amc: &AzureManagedCluster{
-				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{
-						"infrastructure.cluster.x-k8s.io/custom-header-SomeFeature": "true",
-						"annotation-b": "true",
-					},
-				},
-				Spec: AzureManagedClusterSpec{},
-			},
-			wantErr: false,
-		},
 		{
 			name: "ControlPlaneEndpoint.Port update (AKS API-derived update scenario)",
 			oldAMC: &AzureManagedCluster{

diff --git a/api/v1beta1/azuremanagedcontrolplane_types.go b/api/v1beta1/azuremanagedcontrolplane_types.go
@@ -74,6 +74,13 @@ const (
 	NetworkPluginModeOverlay NetworkPluginMode = "overlay"
 )
 
+const (
+	// LoadBalancerSKUStandard is the Standard load balancer SKU.
+	LoadBalancerSKUStandard = "Standard"
+	// LoadBalancerSKUBasic is the Basic load balancer SKU.
+	LoadBalancerSKUBasic = "Basic"
+)
+
 // AzureManagedControlPlaneSpec defines the desired state of AzureManagedControlPlane.
 type AzureManagedControlPlaneSpec struct {
 	// Version defines the desired Kubernetes version.
@@ -156,6 +163,7 @@ type AzureManagedControlPlaneSpec struct {
 	// LoadBalancerSKU is the SKU of the loadBalancer to be provisioned.
 	// Immutable.
 	// +kubebuilder:validation:Enum=Basic;Standard
+	// +kubebuilder:default:=Standard
 	// +optional
 	LoadBalancerSKU *string `json:"loadBalancerSKU,omitempty"`
 
@@ -296,7 +304,7 @@ type AKSSku struct {
 type LoadBalancerProfile struct {
 	// ManagedOutboundIPs - Desired managed outbound IPs for the cluster load balancer.
 	// +optional
-	ManagedOutboundIPs *int32 `json:"managedOutboundIPs,omitempty"`
+	ManagedOutboundIPs *int `json:"managedOutboundIPs,omitempty"`
 
 	// OutboundIPPrefixes - Desired outbound IP Prefix resources for the cluster load balancer.
 	// +optional
@@ -308,11 +316,11 @@ type LoadBalancerProfile struct {
 
 	// AllocatedOutboundPorts - Desired number of allocated SNAT ports per VM. Allowed values must be in the range of 0 to 64000 (inclusive). The default value is 0 which results in Azure dynamically allocating ports.
 	// +optional
-	AllocatedOutboundPorts *int32 `json:"allocatedOutboundPorts,omitempty"`
+	AllocatedOutboundPorts *int `json:"allocatedOutboundPorts,omitempty"`
 
 	// IdleTimeoutInMinutes - Desired outbound flow idle timeout in minutes. Allowed values must be in the range of 4 to 120 (inclusive). The default value is 30 minutes.
 	// +optional
-	IdleTimeoutInMinutes *int32 `json:"idleTimeoutInMinutes,omitempty"`
+	IdleTimeoutInMinutes *int `json:"idleTimeoutInMinutes,omitempty"`
 }
 
 // APIServerAccessProfile tunes the accessibility of the cluster's control plane.

diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook.go b/api/v1beta1/azuremanagedcontrolplane_webhook.go
@@ -76,10 +76,6 @@ func (mw *azureManagedControlPlaneWebhook) Default(ctx context.Context, obj runt
 		networkPlugin := "azure"
 		m.Spec.NetworkPlugin = &networkPlugin
 	}
-	if m.Spec.LoadBalancerSKU == nil {
-		loadBalancerSKU := "Standard"
-		m.Spec.LoadBalancerSKU = &loadBalancerSKU
-	}
 
 	if m.Spec.Version != "" && !strings.HasPrefix(m.Spec.Version, "v") {
 		normalizedVersion := "v" + m.Spec.Version

diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go
@@ -48,7 +48,6 @@ func TestDefaultingWebhook(t *testing.T) {
 	err := mcpw.Default(context.Background(), amcp)
 	g.Expect(err).NotTo(HaveOccurred())
 	g.Expect(*amcp.Spec.NetworkPlugin).To(Equal("azure"))
-	g.Expect(*amcp.Spec.LoadBalancerSKU).To(Equal("Standard"))
 	g.Expect(amcp.Spec.Version).To(Equal("v1.17.5"))
 	g.Expect(*amcp.Spec.SSHPublicKey).NotTo(BeEmpty())
 	g.Expect(amcp.Spec.NodeResourceGroupName).To(Equal("MC_fooRg_fooName_fooLocation"))
@@ -64,10 +63,8 @@ func TestDefaultingWebhook(t *testing.T) {
 
 	t.Logf("Testing amcp defaulting webhook with baseline")
 	netPlug := "kubenet"
-	lbSKU := "Basic"
 	netPol := "azure"
 	amcp.Spec.NetworkPlugin = &netPlug
-	amcp.Spec.LoadBalancerSKU = &lbSKU
 	amcp.Spec.NetworkPolicy = &netPol
 	amcp.Spec.Version = "9.99.99"
 	amcp.Spec.SSHPublicKey = nil
@@ -83,7 +80,6 @@ func TestDefaultingWebhook(t *testing.T) {
 	err = mcpw.Default(context.Background(), amcp)
 	g.Expect(err).NotTo(HaveOccurred())
 	g.Expect(*amcp.Spec.NetworkPlugin).To(Equal(netPlug))
-	g.Expect(*amcp.Spec.LoadBalancerSKU).To(Equal(lbSKU))
 	g.Expect(*amcp.Spec.NetworkPolicy).To(Equal(netPol))
 	g.Expect(amcp.Spec.Version).To(Equal("v9.99.99"))
 	g.Expect(amcp.Spec.SSHPublicKey).To(BeNil())
@@ -233,9 +229,9 @@ func TestValidatingWebhook(t *testing.T) {
 				Spec: AzureManagedControlPlaneSpec{
 					Version: "v1.21.2",
 					LoadBalancerProfile: &LoadBalancerProfile{
-						ManagedOutboundIPs:     ptr.To[int32](10),
-						AllocatedOutboundPorts: ptr.To[int32](1000),
-						IdleTimeoutInMinutes:   ptr.To[int32](60),
+						ManagedOutboundIPs:     ptr.To(10),
+						AllocatedOutboundPorts: ptr.To(1000),
+						IdleTimeoutInMinutes:   ptr.To(60),
 					},
 				},
 			},
@@ -248,7 +244,7 @@ func TestValidatingWebhook(t *testing.T) {
 				Spec: AzureManagedControlPlaneSpec{
 					Version: "v1.21.2",
 					LoadBalancerProfile: &LoadBalancerProfile{
-						ManagedOutboundIPs: ptr.To[int32](200),
+						ManagedOutboundIPs: ptr.To(200),
 					},
 				},
 			},
@@ -261,7 +257,7 @@ func TestValidatingWebhook(t *testing.T) {
 				Spec: AzureManagedControlPlaneSpec{
 					Version: "v1.21.2",
 					LoadBalancerProfile: &LoadBalancerProfile{
-						AllocatedOutboundPorts: ptr.To[int32](80000),
+						AllocatedOutboundPorts: ptr.To(80000),
 					},
 				},
 			},
@@ -274,7 +270,7 @@ func TestValidatingWebhook(t *testing.T) {
 				Spec: AzureManagedControlPlaneSpec{
 					Version: "v1.21.2",
 					LoadBalancerProfile: &LoadBalancerProfile{
-						IdleTimeoutInMinutes: ptr.To[int32](600),
+						IdleTimeoutInMinutes: ptr.To(600),
 					},
 				},
 			},
@@ -287,7 +283,7 @@ func TestValidatingWebhook(t *testing.T) {
 				Spec: AzureManagedControlPlaneSpec{
 					Version: "v1.21.2",
 					LoadBalancerProfile: &LoadBalancerProfile{
-						ManagedOutboundIPs: ptr.To[int32](1),
+						ManagedOutboundIPs: ptr.To(1),
 						OutboundIPs: []string{
 							"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/foo-bar/providers/Microsoft.Network/publicIPAddresses/my-public-ip",
 						},
@@ -1328,14 +1324,14 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) {
 			oldAMCP: &AzureManagedControlPlane{
 				Spec: AzureManagedControlPlaneSpec{
 					DNSServiceIP:    ptr.To("192.168.0.10"),
-					LoadBalancerSKU: ptr.To("Standard"),
+					LoadBalancerSKU: ptr.To(LoadBalancerSKUStandard),
 					Version:         "v1.18.0",
 				},
 			},
 			amcp: &AzureManagedControlPlane{
 				Spec: AzureManagedControlPlaneSpec{
 					DNSServiceIP:    ptr.To("192.168.0.10"),
-					LoadBalancerSKU: ptr.To("Basic"),
+					LoadBalancerSKU: ptr.To(LoadBalancerSKUBasic),
 					Version:         "v1.18.0",
 				},
 			},
@@ -1346,7 +1342,7 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) {
 			oldAMCP: &AzureManagedControlPlane{
 				Spec: AzureManagedControlPlaneSpec{
 					DNSServiceIP:    ptr.To("192.168.0.10"),
-					LoadBalancerSKU: ptr.To("Standard"),
+					LoadBalancerSKU: ptr.To(LoadBalancerSKUStandard),
 					Version:         "v1.18.0",
 				},
 			},