diff --git a/api/v1beta1/azuremanagedcontrolplane_default.go b/api/v1beta1/azuremanagedcontrolplane_default.go index 27e5d1bfd36..9cae98ad80f 100644 --- a/api/v1beta1/azuremanagedcontrolplane_default.go +++ b/api/v1beta1/azuremanagedcontrolplane_default.go @@ -30,6 +30,10 @@ const ( defaultAKSVnetCIDR = "10.0.0.0/8" // defaultAKSNodeSubnetCIDR is the default Node Subnet CIDR. defaultAKSNodeSubnetCIDR = "10.240.0.0/16" + // defaultAKSVnetCIDRForOverlay is the default Vnet CIDR when Azure CNI overlay is enabled. + defaultAKSVnetCIDRForOverlay = "10.224.0.0/12" + // defaultAKSNodeSubnetCIDRForOverlay is the default Node Subnet CIDR when Azure CNI overlay is enabled. + defaultAKSNodeSubnetCIDRForOverlay = "10.224.0.0/16" ) // setDefaultSSHPublicKey sets the default SSHPublicKey for an AzureManagedControlPlane. @@ -60,6 +64,9 @@ func (m *AzureManagedControlPlane) setDefaultVirtualNetwork() { } if m.Spec.VirtualNetwork.CIDRBlock == "" { m.Spec.VirtualNetwork.CIDRBlock = defaultAKSVnetCIDR + if ptr.Deref(m.Spec.NetworkPluginMode, "") == NetworkPluginModeOverlay { + m.Spec.VirtualNetwork.CIDRBlock = defaultAKSVnetCIDRForOverlay + } } if m.Spec.VirtualNetwork.ResourceGroup == "" { m.Spec.VirtualNetwork.ResourceGroup = m.Spec.ResourceGroupName @@ -73,6 +80,9 @@ func (m *AzureManagedControlPlane) setDefaultSubnet() { } if m.Spec.VirtualNetwork.Subnet.CIDRBlock == "" { m.Spec.VirtualNetwork.Subnet.CIDRBlock = defaultAKSNodeSubnetCIDR + if ptr.Deref(m.Spec.NetworkPluginMode, "") == NetworkPluginModeOverlay { + m.Spec.VirtualNetwork.Subnet.CIDRBlock = defaultAKSNodeSubnetCIDRForOverlay + } } } diff --git a/api/v1beta1/azuremanagedcontrolplane_types.go b/api/v1beta1/azuremanagedcontrolplane_types.go index b4bd042e675..e417a56b9fc 100644 --- a/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/api/v1beta1/azuremanagedcontrolplane_types.go @@ -62,6 +62,18 @@ const ( ManagedControlPlaneIdentityTypeUserAssigned ManagedControlPlaneIdentityType = ManagedControlPlaneIdentityType(VMIdentityUserAssigned) ) +// NetworkPluginMode is the mode the network plugin should use. +type NetworkPluginMode string + +const ( + // NetworkPluginModeOverlay is used with networkPlugin=azure, pods are given IPs from the PodCIDR address space but use Azure + // Routing Domains rather than Kubenet's method of route tables. + // See also [AKS doc]. + // + // [AKS doc]: https://aka.ms/aks/azure-cni-overlay + NetworkPluginModeOverlay NetworkPluginMode = "overlay" +) + // AzureManagedControlPlaneSpec defines the desired state of AzureManagedControlPlane. type AzureManagedControlPlaneSpec struct { // Version defines the desired Kubernetes version. @@ -110,6 +122,12 @@ type AzureManagedControlPlaneSpec struct { // +optional NetworkPlugin *string `json:"networkPlugin,omitempty"` + // NetworkPluginMode is the mode the network plugin should use. + // Allowed value is "overlay". + // +kubebuilder:validation:Enum=overlay + // +optional + NetworkPluginMode *NetworkPluginMode `json:"networkPluginMode,omitempty"` + // NetworkPolicy used for building Kubernetes network. // Allowed values are "azure", "calico". // Immutable. diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook.go b/api/v1beta1/azuremanagedcontrolplane_webhook.go index 91aad48286c..df5f6b26456 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -258,6 +258,10 @@ func (mw *azureManagedControlPlaneWebhook) ValidateUpdate(ctx context.Context, o allErrs = append(allErrs, errs...) } + if errs := m.validateNetworkPluginModeUpdate(old); len(errs) > 0 { + allErrs = append(allErrs, errs...) + } + if errs := m.validateOIDCIssuerProfileUpdate(old); len(errs) > 0 { allErrs = append(allErrs, errs...) } @@ -285,6 +289,7 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { m.validateManagedClusterNetwork, m.validateAutoScalerProfile, m.validateIdentity, + m.validateNetworkPluginMode, } var errs []error @@ -543,6 +548,17 @@ func (m *AzureManagedControlPlane) validateVirtualNetworkUpdate(old *AzureManage return allErrs } +// validateNetworkPluginModeUpdate validates update to NetworkPluginMode. +func (m *AzureManagedControlPlane) validateNetworkPluginModeUpdate(old *AzureManagedControlPlane) field.ErrorList { + var allErrs field.ErrorList + + if ptr.Deref(m.Spec.NetworkPluginMode, "") == NetworkPluginModeOverlay && old.Spec.NetworkPolicy != nil { + allErrs = append(allErrs, field.Forbidden(field.NewPath("Spec", "NetworkPluginMode"), fmt.Sprintf("%q NetworkPolicyMode cannot be enabled when NetworkPolicy is set", NetworkPluginModeOverlay))) + } + + return allErrs +} + // validateOIDCIssuerProfile validates an OIDCIssuerProfile. func (m *AzureManagedControlPlane) validateOIDCIssuerProfileUpdate(old *AzureManagedControlPlane) field.ErrorList { var allErrs field.ErrorList @@ -737,3 +753,20 @@ func (m *AzureManagedControlPlane) validateIdentity(_ client.Client) error { return nil } + +// validateNetworkPluginMode validates a NetworkPluginMode. +func (m *AzureManagedControlPlane) validateNetworkPluginMode(_ client.Client) error { + var allErrs field.ErrorList + + const kubenet = "kubenet" + if ptr.Deref(m.Spec.NetworkPluginMode, "") == NetworkPluginModeOverlay && + ptr.Deref(m.Spec.NetworkPlugin, "") == kubenet { + allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "NetworkPluginMode"), m.Spec.NetworkPluginMode, fmt.Sprintf("cannot be set to %q when NetworkPlugin is %q", NetworkPluginModeOverlay, kubenet))) + } + + if len(allErrs) > 0 { + return kerrors.NewAggregate(allErrs.ToAggregate().Errors()) + } + + return nil +} diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index 5518135d846..438bcbf0266 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -53,7 +53,9 @@ func TestDefaultingWebhook(t *testing.T) { g.Expect(*amcp.Spec.SSHPublicKey).NotTo(BeEmpty()) g.Expect(amcp.Spec.NodeResourceGroupName).To(Equal("MC_fooRg_fooName_fooLocation")) g.Expect(amcp.Spec.VirtualNetwork.Name).To(Equal("fooName")) + g.Expect(amcp.Spec.VirtualNetwork.CIDRBlock).To(Equal(defaultAKSVnetCIDR)) g.Expect(amcp.Spec.VirtualNetwork.Subnet.Name).To(Equal("fooName")) + g.Expect(amcp.Spec.VirtualNetwork.Subnet.CIDRBlock).To(Equal(defaultAKSNodeSubnetCIDR)) g.Expect(amcp.Spec.SKU.Tier).To(Equal(FreeManagedControlPlaneTier)) g.Expect(amcp.Spec.Identity.Type).To(Equal(ManagedControlPlaneIdentityTypeSystemAssigned)) g.Expect(*amcp.Spec.OIDCIssuerProfile.Enabled).To(BeFalse()) @@ -87,6 +89,24 @@ func TestDefaultingWebhook(t *testing.T) { g.Expect(amcp.Spec.VirtualNetwork.Subnet.Name).To(Equal("fooSubnetName")) g.Expect(amcp.Spec.SKU.Tier).To(Equal(PaidManagedControlPlaneTier)) g.Expect(*amcp.Spec.OIDCIssuerProfile.Enabled).To(BeTrue()) + + t.Logf("Testing amcp defaulting webhook with overlay") + amcp = &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fooName", + }, + Spec: AzureManagedControlPlaneSpec{ + ResourceGroupName: "fooRg", + Location: "fooLocation", + Version: "1.17.5", + SSHPublicKey: ptr.To(""), + NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + }, + } + err = mcpw.Default(context.Background(), amcp) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(amcp.Spec.VirtualNetwork.CIDRBlock).To(Equal(defaultAKSVnetCIDRForOverlay)) + g.Expect(amcp.Spec.VirtualNetwork.Subnet.CIDRBlock).To(Equal(defaultAKSNodeSubnetCIDRForOverlay)) } func TestValidatingWebhook(t *testing.T) { @@ -667,6 +687,30 @@ func TestValidatingWebhook(t *testing.T) { }, expectErr: true, }, + { + name: "overlay cannot be used with kubenet", + amcp: AzureManagedControlPlane{ + ObjectMeta: getAMCPMetaData(), + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.24.1", + NetworkPlugin: ptr.To("kubenet"), + NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + }, + }, + expectErr: true, + }, + { + name: "overlay can be used with azure", + amcp: AzureManagedControlPlane{ + ObjectMeta: getAMCPMetaData(), + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.24.1", + NetworkPlugin: ptr.To("azure"), + NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + }, + }, + expectErr: false, + }, } for _, tt := range tests { @@ -1422,6 +1466,49 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: true, }, + { + name: "NetworkPluginMode cannot change to \"overlay\" when NetworkPolicy is set", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + NetworkPolicy: ptr.To("anything"), + NetworkPluginMode: nil, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + }, + }, + wantErr: true, + }, + { + name: "NetworkPluginMode can change to \"overlay\" when NetworkPolicy is not set", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + NetworkPolicy: nil, + NetworkPluginMode: nil, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + NetworkPluginMode: ptr.To(NetworkPluginModeOverlay), + Version: "v0.0.0", + }, + }, + wantErr: false, + }, { name: "AzureManagedControlPlane OIDCIssuerProfile.Enabled false -> false OK", oldAMCP: &AzureManagedControlPlane{ diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 49507c6d1b7..d9270d475ba 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1165,6 +1165,11 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = new(string) **out = **in } + if in.NetworkPluginMode != nil { + in, out := &in.NetworkPluginMode, &out.NetworkPluginMode + *out = new(NetworkPluginMode) + **out = **in + } if in.NetworkPolicy != nil { in, out := &in.NetworkPolicy, &out.NetworkPolicy *out = new(string) diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 022dcc9d85a..9be7e235e45 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -484,6 +484,7 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec() azure.ResourceSpecGetter OutboundType: s.ControlPlane.Spec.OutboundType, Identity: s.ControlPlane.Spec.Identity, KubeletUserAssignedIdentity: s.ControlPlane.Spec.KubeletUserAssignedIdentity, + NetworkPluginMode: s.ControlPlane.Spec.NetworkPluginMode, } if s.ControlPlane.Spec.SSHPublicKey != nil { diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index cf999b06d74..b75f0335a0d 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -66,6 +66,9 @@ type ManagedClusterSpec struct { // NetworkPlugin used for building Kubernetes network. Possible values include: 'azure', 'kubenet'. Defaults to azure. NetworkPlugin string + // NetworkPluginMode is the mode the network plugin should use. + NetworkPluginMode *infrav1.NetworkPluginMode + // NetworkPolicy used for building Kubernetes network. Possible values include: 'calico', 'azure'. NetworkPolicy string @@ -351,6 +354,10 @@ func (s *ManagedClusterSpec) Parameters(ctx context.Context, existing interface{ } } + if s.NetworkPluginMode != nil { + managedCluster.Properties.NetworkProfile.NetworkPluginMode = ptr.To(armcontainerservice.NetworkPluginMode(*s.NetworkPluginMode)) + } + if s.PodCIDR != "" { managedCluster.Properties.NetworkProfile.PodCidr = &s.PodCIDR } @@ -590,12 +597,18 @@ func computeDiffOfNormalizedClusters(managedCluster armcontainerservice.ManagedC } } + if existingMC.Properties.NetworkProfile != nil { + existingMCPropertiesNormalized.NetworkProfile.LoadBalancerProfile = existingMC.Properties.NetworkProfile.LoadBalancerProfile + + existingMCPropertiesNormalized.NetworkProfile.NetworkPluginMode = existingMC.Properties.NetworkProfile.NetworkPluginMode + } if managedCluster.Properties.NetworkProfile != nil { propertiesNormalized.NetworkProfile.LoadBalancerProfile = managedCluster.Properties.NetworkProfile.LoadBalancerProfile - } - if existingMC.Properties.NetworkProfile != nil { - existingMCPropertiesNormalized.NetworkProfile.LoadBalancerProfile = existingMC.Properties.NetworkProfile.LoadBalancerProfile + propertiesNormalized.NetworkProfile.NetworkPluginMode = managedCluster.Properties.NetworkProfile.NetworkPluginMode + if propertiesNormalized.NetworkProfile.NetworkPluginMode == nil { + propertiesNormalized.NetworkProfile.NetworkPluginMode = existingMCPropertiesNormalized.NetworkProfile.NetworkPluginMode + } } if managedCluster.Properties.APIServerAccessProfile != nil { diff --git a/azure/services/managedclusters/spec_test.go b/azure/services/managedclusters/spec_test.go index f316122ae96..9b3e69d9d11 100644 --- a/azure/services/managedclusters/spec_test.go +++ b/azure/services/managedclusters/spec_test.go @@ -68,9 +68,10 @@ func TestParameters(t *testing.T) { Tags: map[string]string{ "test-tag": "test-value", }, - Version: "v1.22.0", - LoadBalancerSKU: "standard", - SSHPublicKey: base64.StdEncoding.EncodeToString([]byte("test-ssh-key")), + Version: "v1.22.0", + LoadBalancerSKU: "standard", + SSHPublicKey: base64.StdEncoding.EncodeToString([]byte("test-ssh-key")), + NetworkPluginMode: ptr.To(infrav1.NetworkPluginModeOverlay), OIDCIssuerProfile: &OIDCIssuerProfile{ Enabled: ptr.To(true), }, @@ -415,6 +416,54 @@ func TestParameters(t *testing.T) { g.Expect(result).To(BeNil()) }, }, + { + name: "setting networkPluginMode from nil to \"overlay\" will update", + existing: func() armcontainerservice.ManagedCluster { + c := getExistingCluster() + c.Properties.NetworkProfile.NetworkPluginMode = nil + return c + }(), + spec: &ManagedClusterSpec{ + Name: "test-managedcluster", + ResourceGroup: "test-rg", + Location: "test-location", + Tags: map[string]string{ + "test-tag": "test-value", + }, + Version: "v1.22.0", + LoadBalancerSKU: "standard", + OIDCIssuerProfile: &OIDCIssuerProfile{ + Enabled: ptr.To(true), + }, + NetworkPluginMode: ptr.To(infrav1.NetworkPluginModeOverlay), + }, + expect: func(g *WithT, result interface{}) { + g.Expect(result).To(BeAssignableToTypeOf(armcontainerservice.ManagedCluster{})) + g.Expect(result.(armcontainerservice.ManagedCluster).Properties.NetworkProfile.NetworkPluginMode).NotTo(BeNil()) + g.Expect(*result.(armcontainerservice.ManagedCluster).Properties.NetworkProfile.NetworkPluginMode).To(Equal(armcontainerservice.NetworkPluginModeOverlay)) + }, + }, + { + name: "setting networkPluginMode from \"overlay\" to nil doesn't require update", + existing: getExistingCluster(), + spec: &ManagedClusterSpec{ + Name: "test-managedcluster", + ResourceGroup: "test-rg", + Location: "test-location", + Tags: map[string]string{ + "test-tag": "test-value", + }, + Version: "v1.22.0", + LoadBalancerSKU: "standard", + OIDCIssuerProfile: &OIDCIssuerProfile{ + Enabled: ptr.To(true), + }, + NetworkPluginMode: nil, + }, + expect: func(g *WithT, result interface{}) { + g.Expect(result).To(BeNil()) + }, + }, { name: "update needed when oidc issuer profile enabled changes", existing: getExistingCluster(), @@ -584,7 +633,8 @@ func getSampleManagedCluster() armcontainerservice.ManagedCluster { NodeResourceGroup: ptr.To("test-node-rg"), EnableRBAC: ptr.To(true), NetworkProfile: &armcontainerservice.NetworkProfile{ - LoadBalancerSKU: ptr.To(armcontainerservice.LoadBalancerSKUStandard), + LoadBalancerSKU: ptr.To(armcontainerservice.LoadBalancerSKUStandard), + NetworkPluginMode: ptr.To(armcontainerservice.NetworkPluginModeOverlay), }, OidcIssuerProfile: &armcontainerservice.ManagedClusterOIDCIssuerProfile{ Enabled: ptr.To(true), diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index 04cba8bbd24..4d1bac26596 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -375,6 +375,12 @@ spec: - azure - kubenet type: string + networkPluginMode: + description: NetworkPluginMode is the mode the network plugin should + use. Allowed value is "overlay". + enum: + - overlay + type: string networkPolicy: description: NetworkPolicy used for building Kubernetes network. Allowed values are "azure", "calico". Immutable.