Merge pull request #3988 from nojnhuh/aks-cni-overlay

add spec.networkPluginMode to AzureManagedControlPlane

api/v1beta1/azuremanagedcontrolplane_default.go

@@ -30,6 +30,10 @@ const (
 	defaultAKSVnetCIDR = "10.0.0.0/8"
 	// defaultAKSNodeSubnetCIDR is the default Node Subnet CIDR.
 	defaultAKSNodeSubnetCIDR = "10.240.0.0/16"
+	// defaultAKSVnetCIDRForOverlay is the default Vnet CIDR when Azure CNI overlay is enabled.
+	defaultAKSVnetCIDRForOverlay = "10.224.0.0/12"
+	// defaultAKSNodeSubnetCIDRForOverlay is the default Node Subnet CIDR when Azure CNI overlay is enabled.
+	defaultAKSNodeSubnetCIDRForOverlay = "10.224.0.0/16"
 )
 
 // setDefaultSSHPublicKey sets the default SSHPublicKey for an AzureManagedControlPlane.
@@ -60,6 +64,9 @@ func (m *AzureManagedControlPlane) setDefaultVirtualNetwork() {
 	}
 	if m.Spec.VirtualNetwork.CIDRBlock == "" {
 		m.Spec.VirtualNetwork.CIDRBlock = defaultAKSVnetCIDR
+		if ptr.Deref(m.Spec.NetworkPluginMode, "") == NetworkPluginModeOverlay {
+			m.Spec.VirtualNetwork.CIDRBlock = defaultAKSVnetCIDRForOverlay
+		}
 	}
 	if m.Spec.VirtualNetwork.ResourceGroup == "" {
 		m.Spec.VirtualNetwork.ResourceGroup = m.Spec.ResourceGroupName
@@ -73,6 +80,9 @@ func (m *AzureManagedControlPlane) setDefaultSubnet() {
 	}
 	if m.Spec.VirtualNetwork.Subnet.CIDRBlock == "" {
 		m.Spec.VirtualNetwork.Subnet.CIDRBlock = defaultAKSNodeSubnetCIDR
+		if ptr.Deref(m.Spec.NetworkPluginMode, "") == NetworkPluginModeOverlay {
+			m.Spec.VirtualNetwork.Subnet.CIDRBlock = defaultAKSNodeSubnetCIDRForOverlay
+		}
 	}
 }
 

api/v1beta1/azuremanagedcontrolplane_types.go

@@ -62,6 +62,18 @@ const (
 	ManagedControlPlaneIdentityTypeUserAssigned ManagedControlPlaneIdentityType = ManagedControlPlaneIdentityType(VMIdentityUserAssigned)
 )
 
+// NetworkPluginMode is the mode the network plugin should use.
+type NetworkPluginMode string
+
+const (
+	// NetworkPluginModeOverlay is used with networkPlugin=azure, pods are given IPs from the PodCIDR address space but use Azure
+	// Routing Domains rather than Kubenet's method of route tables.
+	// See also [AKS doc].
+	//
+	// [AKS doc]: https://aka.ms/aks/azure-cni-overlay
+	NetworkPluginModeOverlay NetworkPluginMode = "overlay"
+)
+
 // AzureManagedControlPlaneSpec defines the desired state of AzureManagedControlPlane.
 type AzureManagedControlPlaneSpec struct {
 	// Version defines the desired Kubernetes version.
@@ -110,6 +122,12 @@ type AzureManagedControlPlaneSpec struct {
 	// +optional
 	NetworkPlugin *string `json:"networkPlugin,omitempty"`
 
+	// NetworkPluginMode is the mode the network plugin should use.
+	// Allowed value is "overlay".
+	// +kubebuilder:validation:Enum=overlay
+	// +optional
+	NetworkPluginMode *NetworkPluginMode `json:"networkPluginMode,omitempty"`
+
 	// NetworkPolicy used for building Kubernetes network.
 	// Allowed values are "azure", "calico".
 	// Immutable.

api/v1beta1/azuremanagedcontrolplane_webhook.go

@@ -258,6 +258,10 @@ func (mw *azureManagedControlPlaneWebhook) ValidateUpdate(ctx context.Context, o
 		allErrs = append(allErrs, errs...)
 	}
 
+	if errs := m.validateNetworkPluginModeUpdate(old); len(errs) > 0 {
+		allErrs = append(allErrs, errs...)
+	}
+
 	if errs := m.validateOIDCIssuerProfileUpdate(old); len(errs) > 0 {
 		allErrs = append(allErrs, errs...)
 	}
@@ -285,6 +289,7 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error {
 		m.validateManagedClusterNetwork,
 		m.validateAutoScalerProfile,
 		m.validateIdentity,
+		m.validateNetworkPluginMode,
 	}
 
 	var errs []error
@@ -543,6 +548,17 @@ func (m *AzureManagedControlPlane) validateVirtualNetworkUpdate(old *AzureManage
 	return allErrs
 }
 
+// validateNetworkPluginModeUpdate validates update to NetworkPluginMode.
+func (m *AzureManagedControlPlane) validateNetworkPluginModeUpdate(old *AzureManagedControlPlane) field.ErrorList {
+	var allErrs field.ErrorList
+
+	if ptr.Deref(m.Spec.NetworkPluginMode, "") == NetworkPluginModeOverlay && old.Spec.NetworkPolicy != nil {
+		allErrs = append(allErrs, field.Forbidden(field.NewPath("Spec", "NetworkPluginMode"), fmt.Sprintf("%q NetworkPolicyMode cannot be enabled when NetworkPolicy is set", NetworkPluginModeOverlay)))
+	}
+
+	return allErrs
+}
+
 // validateOIDCIssuerProfile validates an OIDCIssuerProfile.
 func (m *AzureManagedControlPlane) validateOIDCIssuerProfileUpdate(old *AzureManagedControlPlane) field.ErrorList {
 	var allErrs field.ErrorList
@@ -737,3 +753,20 @@ func (m *AzureManagedControlPlane) validateIdentity(_ client.Client) error {
 
 	return nil
 }
+
+// validateNetworkPluginMode validates a NetworkPluginMode.
+func (m *AzureManagedControlPlane) validateNetworkPluginMode(_ client.Client) error {
+	var allErrs field.ErrorList
+
+	const kubenet = "kubenet"
+	if ptr.Deref(m.Spec.NetworkPluginMode, "") == NetworkPluginModeOverlay &&
+		ptr.Deref(m.Spec.NetworkPlugin, "") == kubenet {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "NetworkPluginMode"), m.Spec.NetworkPluginMode, fmt.Sprintf("cannot be set to %q when NetworkPlugin is %q", NetworkPluginModeOverlay, kubenet)))
+	}
+
+	if len(allErrs) > 0 {
+		return kerrors.NewAggregate(allErrs.ToAggregate().Errors())
+	}
+
+	return nil
+}

api/v1beta1/azuremanagedcontrolplane_webhook_test.go

@@ -53,7 +53,9 @@ func TestDefaultingWebhook(t *testing.T) {
 	g.Expect(*amcp.Spec.SSHPublicKey).NotTo(BeEmpty())
 	g.Expect(amcp.Spec.NodeResourceGroupName).To(Equal("MC_fooRg_fooName_fooLocation"))
 	g.Expect(amcp.Spec.VirtualNetwork.Name).To(Equal("fooName"))
+	g.Expect(amcp.Spec.VirtualNetwork.CIDRBlock).To(Equal(defaultAKSVnetCIDR))
 	g.Expect(amcp.Spec.VirtualNetwork.Subnet.Name).To(Equal("fooName"))
+	g.Expect(amcp.Spec.VirtualNetwork.Subnet.CIDRBlock).To(Equal(defaultAKSNodeSubnetCIDR))
 	g.Expect(amcp.Spec.SKU.Tier).To(Equal(FreeManagedControlPlaneTier))
 	g.Expect(amcp.Spec.Identity.Type).To(Equal(ManagedControlPlaneIdentityTypeSystemAssigned))
 	g.Expect(*amcp.Spec.OIDCIssuerProfile.Enabled).To(BeFalse())
@@ -87,6 +89,24 @@ func TestDefaultingWebhook(t *testing.T) {
 	g.Expect(amcp.Spec.VirtualNetwork.Subnet.Name).To(Equal("fooSubnetName"))
 	g.Expect(amcp.Spec.SKU.Tier).To(Equal(PaidManagedControlPlaneTier))
 	g.Expect(*amcp.Spec.OIDCIssuerProfile.Enabled).To(BeTrue())
+
+	t.Logf("Testing amcp defaulting webhook with overlay")
+	amcp = &AzureManagedControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "fooName",
+		},
+		Spec: AzureManagedControlPlaneSpec{
+			ResourceGroupName: "fooRg",
+			Location:          "fooLocation",
+			Version:           "1.17.5",
+			SSHPublicKey:      ptr.To(""),
+			NetworkPluginMode: ptr.To(NetworkPluginModeOverlay),
+		},
+	}
+	err = mcpw.Default(context.Background(), amcp)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(amcp.Spec.VirtualNetwork.CIDRBlock).To(Equal(defaultAKSVnetCIDRForOverlay))
+	g.Expect(amcp.Spec.VirtualNetwork.Subnet.CIDRBlock).To(Equal(defaultAKSNodeSubnetCIDRForOverlay))
 }
 
 func TestValidatingWebhook(t *testing.T) {
@@ -667,6 +687,30 @@ func TestValidatingWebhook(t *testing.T) {
 			},
 			expectErr: true,
 		},
+		{
+			name: "overlay cannot be used with kubenet",
+			amcp: AzureManagedControlPlane{
+				ObjectMeta: getAMCPMetaData(),
+				Spec: AzureManagedControlPlaneSpec{
+					Version:           "v1.24.1",
+					NetworkPlugin:     ptr.To("kubenet"),
+					NetworkPluginMode: ptr.To(NetworkPluginModeOverlay),
+				},
+			},
+			expectErr: true,
+		},
+		{
+			name: "overlay can be used with azure",
+			amcp: AzureManagedControlPlane{
+				ObjectMeta: getAMCPMetaData(),
+				Spec: AzureManagedControlPlaneSpec{
+					Version:           "v1.24.1",
+					NetworkPlugin:     ptr.To("azure"),
+					NetworkPluginMode: ptr.To(NetworkPluginModeOverlay),
+				},
+			},
+			expectErr: false,
+		},
 	}
 
 	for _, tt := range tests {
@@ -1422,6 +1466,49 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "NetworkPluginMode cannot change to \"overlay\" when NetworkPolicy is set",
+			oldAMCP: &AzureManagedControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-cluster",
+				},
+				Spec: AzureManagedControlPlaneSpec{
+					NetworkPolicy:     ptr.To("anything"),
+					NetworkPluginMode: nil,
+				},
+			},
+			amcp: &AzureManagedControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-cluster",
+				},
+				Spec: AzureManagedControlPlaneSpec{
+					NetworkPluginMode: ptr.To(NetworkPluginModeOverlay),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "NetworkPluginMode can change to \"overlay\" when NetworkPolicy is not set",
+			oldAMCP: &AzureManagedControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-cluster",
+				},
+				Spec: AzureManagedControlPlaneSpec{
+					NetworkPolicy:     nil,
+					NetworkPluginMode: nil,
+				},
+			},
+			amcp: &AzureManagedControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-cluster",
+				},
+				Spec: AzureManagedControlPlaneSpec{
+					NetworkPluginMode: ptr.To(NetworkPluginModeOverlay),
+					Version:           "v0.0.0",
+				},
+			},
+			wantErr: false,
+		},
 		{
 			name: "AzureManagedControlPlane OIDCIssuerProfile.Enabled false -> false OK",
 			oldAMCP: &AzureManagedControlPlane{

azure/scope/managedcontrolplane.go

@@ -484,6 +484,7 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec() azure.ResourceSpecGetter
 		OutboundType:                s.ControlPlane.Spec.OutboundType,
 		Identity:                    s.ControlPlane.Spec.Identity,
 		KubeletUserAssignedIdentity: s.ControlPlane.Spec.KubeletUserAssignedIdentity,
+		NetworkPluginMode:           s.ControlPlane.Spec.NetworkPluginMode,
 	}
 
 	if s.ControlPlane.Spec.SSHPublicKey != nil {

azure/services/managedclusters/spec.go

@@ -66,6 +66,9 @@ type ManagedClusterSpec struct {
 	// NetworkPlugin used for building Kubernetes network. Possible values include: 'azure', 'kubenet'. Defaults to azure.
 	NetworkPlugin string
 
+	// NetworkPluginMode is the mode the network plugin should use.
+	NetworkPluginMode *infrav1.NetworkPluginMode
+
 	// NetworkPolicy used for building Kubernetes network. Possible values include: 'calico', 'azure'.
 	NetworkPolicy string
 
@@ -351,6 +354,10 @@ func (s *ManagedClusterSpec) Parameters(ctx context.Context, existing interface{
 		}
 	}
 
+	if s.NetworkPluginMode != nil {
+		managedCluster.Properties.NetworkProfile.NetworkPluginMode = ptr.To(armcontainerservice.NetworkPluginMode(*s.NetworkPluginMode))
+	}
+
 	if s.PodCIDR != "" {
 		managedCluster.Properties.NetworkProfile.PodCidr = &s.PodCIDR
 	}
@@ -590,12 +597,18 @@ func computeDiffOfNormalizedClusters(managedCluster armcontainerservice.ManagedC
 		}
 	}
 
+	if existingMC.Properties.NetworkProfile != nil {
+		existingMCPropertiesNormalized.NetworkProfile.LoadBalancerProfile = existingMC.Properties.NetworkProfile.LoadBalancerProfile
+
+		existingMCPropertiesNormalized.NetworkProfile.NetworkPluginMode = existingMC.Properties.NetworkProfile.NetworkPluginMode
+	}
 	if managedCluster.Properties.NetworkProfile != nil {
 		propertiesNormalized.NetworkProfile.LoadBalancerProfile = managedCluster.Properties.NetworkProfile.LoadBalancerProfile
-	}
 
-	if existingMC.Properties.NetworkProfile != nil {
-		existingMCPropertiesNormalized.NetworkProfile.LoadBalancerProfile = existingMC.Properties.NetworkProfile.LoadBalancerProfile
+		propertiesNormalized.NetworkProfile.NetworkPluginMode = managedCluster.Properties.NetworkProfile.NetworkPluginMode
+		if propertiesNormalized.NetworkProfile.NetworkPluginMode == nil {
+			propertiesNormalized.NetworkProfile.NetworkPluginMode = existingMCPropertiesNormalized.NetworkProfile.NetworkPluginMode
+		}
 	}
 
 	if managedCluster.Properties.APIServerAccessProfile != nil {

azure/services/managedclusters/spec_test.go

@@ -68,9 +68,10 @@ func TestParameters(t *testing.T) {
 				Tags: map[string]string{
 					"test-tag": "test-value",
 				},
-				Version:         "v1.22.0",
-				LoadBalancerSKU: "standard",
-				SSHPublicKey:    base64.StdEncoding.EncodeToString([]byte("test-ssh-key")),
+				Version:           "v1.22.0",
+				LoadBalancerSKU:   "standard",
+				SSHPublicKey:      base64.StdEncoding.EncodeToString([]byte("test-ssh-key")),
+				NetworkPluginMode: ptr.To(infrav1.NetworkPluginModeOverlay),
 				OIDCIssuerProfile: &OIDCIssuerProfile{
 					Enabled: ptr.To(true),
 				},
@@ -415,6 +416,54 @@ func TestParameters(t *testing.T) {
 				g.Expect(result).To(BeNil())
 			},
 		},
+		{
+			name: "setting networkPluginMode from nil to \"overlay\" will update",
+			existing: func() armcontainerservice.ManagedCluster {
+				c := getExistingCluster()
+				c.Properties.NetworkProfile.NetworkPluginMode = nil
+				return c
+			}(),
+			spec: &ManagedClusterSpec{
+				Name:          "test-managedcluster",
+				ResourceGroup: "test-rg",
+				Location:      "test-location",
+				Tags: map[string]string{
+					"test-tag": "test-value",
+				},
+				Version:         "v1.22.0",
+				LoadBalancerSKU: "standard",
+				OIDCIssuerProfile: &OIDCIssuerProfile{
+					Enabled: ptr.To(true),
+				},
+				NetworkPluginMode: ptr.To(infrav1.NetworkPluginModeOverlay),
+			},
+			expect: func(g *WithT, result interface{}) {
+				g.Expect(result).To(BeAssignableToTypeOf(armcontainerservice.ManagedCluster{}))
+				g.Expect(result.(armcontainerservice.ManagedCluster).Properties.NetworkProfile.NetworkPluginMode).NotTo(BeNil())
+				g.Expect(*result.(armcontainerservice.ManagedCluster).Properties.NetworkProfile.NetworkPluginMode).To(Equal(armcontainerservice.NetworkPluginModeOverlay))
+			},
+		},
+		{
+			name:     "setting networkPluginMode from \"overlay\" to nil doesn't require update",
+			existing: getExistingCluster(),
+			spec: &ManagedClusterSpec{
+				Name:          "test-managedcluster",
+				ResourceGroup: "test-rg",
+				Location:      "test-location",
+				Tags: map[string]string{
+					"test-tag": "test-value",
+				},
+				Version:         "v1.22.0",
+				LoadBalancerSKU: "standard",
+				OIDCIssuerProfile: &OIDCIssuerProfile{
+					Enabled: ptr.To(true),
+				},
+				NetworkPluginMode: nil,
+			},
+			expect: func(g *WithT, result interface{}) {
+				g.Expect(result).To(BeNil())
+			},
+		},
 		{
 			name:     "update needed when oidc issuer profile enabled changes",
 			existing: getExistingCluster(),
@@ -584,7 +633,8 @@ func getSampleManagedCluster() armcontainerservice.ManagedCluster {
 			NodeResourceGroup:       ptr.To("test-node-rg"),
 			EnableRBAC:              ptr.To(true),
 			NetworkProfile: &armcontainerservice.NetworkProfile{
-				LoadBalancerSKU: ptr.To(armcontainerservice.LoadBalancerSKUStandard),
+				LoadBalancerSKU:   ptr.To(armcontainerservice.LoadBalancerSKUStandard),
+				NetworkPluginMode: ptr.To(armcontainerservice.NetworkPluginModeOverlay),
 			},
 			OidcIssuerProfile: &armcontainerservice.ManagedClusterOIDCIssuerProfile{
 				Enabled: ptr.To(true),

config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml

@@ -375,6 +375,12 @@ spec:
                 - azure
                 - kubenet
                 type: string
+              networkPluginMode:
+                description: NetworkPluginMode is the mode the network plugin should
+                  use. Allowed value is "overlay".
+                enum:
+                - overlay
+                type: string
               networkPolicy:
                 description: NetworkPolicy used for building Kubernetes network. Allowed
                   values are "azure", "calico". Immutable.