diff --git a/azure/converters/managedagentpool.go b/azure/converters/managedagentpool.go index 46720635ed7..c921464779d 100644 --- a/azure/converters/managedagentpool.go +++ b/azure/converters/managedagentpool.go @@ -24,25 +24,26 @@ import ( func AgentPoolToManagedClusterAgentPoolProfile(pool containerservice.AgentPool) containerservice.ManagedClusterAgentPoolProfile { properties := pool.ManagedClusterAgentPoolProfileProperties return containerservice.ManagedClusterAgentPoolProfile{ - Name: pool.Name, // Note: if converting from agentPoolSpec.Parameters(), this field will not be set - VMSize: properties.VMSize, - OsType: properties.OsType, - OsDiskSizeGB: properties.OsDiskSizeGB, - Count: properties.Count, - Type: properties.Type, - OrchestratorVersion: properties.OrchestratorVersion, - VnetSubnetID: properties.VnetSubnetID, - Mode: properties.Mode, - EnableAutoScaling: properties.EnableAutoScaling, - MaxCount: properties.MaxCount, - MinCount: properties.MinCount, - NodeTaints: properties.NodeTaints, - AvailabilityZones: properties.AvailabilityZones, - MaxPods: properties.MaxPods, - OsDiskType: properties.OsDiskType, - NodeLabels: properties.NodeLabels, - EnableUltraSSD: properties.EnableUltraSSD, - EnableNodePublicIP: properties.EnableNodePublicIP, - ScaleSetPriority: properties.ScaleSetPriority, + Name: pool.Name, // Note: if converting from agentPoolSpec.Parameters(), this field will not be set + VMSize: properties.VMSize, + OsType: properties.OsType, + OsDiskSizeGB: properties.OsDiskSizeGB, + Count: properties.Count, + Type: properties.Type, + OrchestratorVersion: properties.OrchestratorVersion, + VnetSubnetID: properties.VnetSubnetID, + Mode: properties.Mode, + EnableAutoScaling: properties.EnableAutoScaling, + MaxCount: properties.MaxCount, + MinCount: properties.MinCount, + NodeTaints: properties.NodeTaints, + AvailabilityZones: properties.AvailabilityZones, + MaxPods: properties.MaxPods, + OsDiskType: properties.OsDiskType, + NodeLabels: properties.NodeLabels, + EnableUltraSSD: properties.EnableUltraSSD, + EnableNodePublicIP: properties.EnableNodePublicIP, + NodePublicIPPrefixID: properties.NodePublicIPPrefixID, + ScaleSetPriority: properties.ScaleSetPriority, } } diff --git a/azure/scope/managedmachinepool.go b/azure/scope/managedmachinepool.go index 3fc15957ea9..c6958e8b504 100644 --- a/azure/scope/managedmachinepool.go +++ b/azure/scope/managedmachinepool.go @@ -169,14 +169,15 @@ func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane managedControlPlane.Spec.VirtualNetwork.Name, managedControlPlane.Spec.VirtualNetwork.Subnet.Name, ), - Mode: managedMachinePool.Spec.Mode, - MaxPods: managedMachinePool.Spec.MaxPods, - AvailabilityZones: managedMachinePool.Spec.AvailabilityZones, - OsDiskType: managedMachinePool.Spec.OsDiskType, - EnableUltraSSD: managedMachinePool.Spec.EnableUltraSSD, - Headers: maps.FilterByKeyPrefix(agentPoolAnnotations, azure.CustomHeaderPrefix), - EnableNodePublicIP: managedMachinePool.Spec.EnableNodePublicIP, - ScaleSetPriority: managedMachinePool.Spec.ScaleSetPriority, + Mode: managedMachinePool.Spec.Mode, + MaxPods: managedMachinePool.Spec.MaxPods, + AvailabilityZones: managedMachinePool.Spec.AvailabilityZones, + OsDiskType: managedMachinePool.Spec.OsDiskType, + EnableUltraSSD: managedMachinePool.Spec.EnableUltraSSD, + Headers: maps.FilterByKeyPrefix(agentPoolAnnotations, azure.CustomHeaderPrefix), + EnableNodePublicIP: managedMachinePool.Spec.EnableNodePublicIP, + NodePublicIPPrefixID: managedMachinePool.Spec.NodePublicIPPrefixID, + ScaleSetPriority: managedMachinePool.Spec.ScaleSetPriority, } if managedMachinePool.Spec.OSDiskSizeGB != nil { diff --git a/azure/services/agentpools/spec.go b/azure/services/agentpools/spec.go index a2235aeba1d..d65a8b88348 100644 --- a/azure/services/agentpools/spec.go +++ b/azure/services/agentpools/spec.go @@ -94,6 +94,9 @@ type AgentPoolSpec struct { // EnableNodePublicIP controls whether or not nodes in the agent pool each have a public IP address. EnableNodePublicIP *bool `json:"enableNodePublicIP,omitempty"` + // NodePublicIPPrefixID specifies the public IP prefix resource ID which VM nodes should use IPs from. + NodePublicIPPrefixID *string `json:"nodePublicIPPrefixID,omitempty"` + // ScaleSetPriority specifies the ScaleSetPriority for the node pool. Allowed values are 'Spot' and 'Regular' ScaleSetPriority *string `json:"scaleSetPriority,omitempty"` } @@ -205,25 +208,26 @@ func (s *AgentPoolSpec) Parameters(existing interface{}) (params interface{}, er return containerservice.AgentPool{ ManagedClusterAgentPoolProfileProperties: &containerservice.ManagedClusterAgentPoolProfileProperties{ - AvailabilityZones: availabilityZones, - Count: replicas, - EnableAutoScaling: s.EnableAutoScaling, - EnableUltraSSD: s.EnableUltraSSD, - MaxCount: s.MaxCount, - MaxPods: s.MaxPods, - MinCount: s.MinCount, - Mode: containerservice.AgentPoolMode(s.Mode), - NodeLabels: nodeLabels, - NodeTaints: nodeTaints, - OrchestratorVersion: s.Version, - OsDiskSizeGB: &s.OSDiskSizeGB, - OsDiskType: containerservice.OSDiskType(to.String(s.OsDiskType)), - OsType: containerservice.OSType(to.String(s.OSType)), - ScaleSetPriority: containerservice.ScaleSetPriority(to.String(s.ScaleSetPriority)), - Type: containerservice.AgentPoolTypeVirtualMachineScaleSets, - VMSize: sku, - VnetSubnetID: vnetSubnetID, - EnableNodePublicIP: s.EnableNodePublicIP, + AvailabilityZones: availabilityZones, + Count: replicas, + EnableAutoScaling: s.EnableAutoScaling, + EnableUltraSSD: s.EnableUltraSSD, + MaxCount: s.MaxCount, + MaxPods: s.MaxPods, + MinCount: s.MinCount, + Mode: containerservice.AgentPoolMode(s.Mode), + NodeLabels: nodeLabels, + NodeTaints: nodeTaints, + OrchestratorVersion: s.Version, + OsDiskSizeGB: &s.OSDiskSizeGB, + OsDiskType: containerservice.OSDiskType(to.String(s.OsDiskType)), + OsType: containerservice.OSType(to.String(s.OSType)), + ScaleSetPriority: containerservice.ScaleSetPriority(to.String(s.ScaleSetPriority)), + Type: containerservice.AgentPoolTypeVirtualMachineScaleSets, + VMSize: sku, + VnetSubnetID: vnetSubnetID, + EnableNodePublicIP: s.EnableNodePublicIP, + NodePublicIPPrefixID: s.NodePublicIPPrefixID, }, }, nil } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml index 4d69264c347..ca34e44c18a 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml @@ -235,6 +235,10 @@ spec: description: Node labels - labels for all of the nodes present in node pool type: object + nodePublicIPPrefixID: + description: NodePublicIPPrefixID specifies the public IP prefix resource + ID which VM nodes should use IPs from. + type: string osDiskSizeGB: description: OSDiskSizeGB is the disk size for every machine in this agent pool. If you specify 0, it will apply the default osDisk size diff --git a/docs/book/src/topics/managedcluster.md b/docs/book/src/topics/managedcluster.md index 87e19d9ddab..cf78a003548 100644 --- a/docs/book/src/topics/managedcluster.md +++ b/docs/book/src/topics/managedcluster.md @@ -481,6 +481,8 @@ Following is the list of immutable fields for managed clusters: | AzureManagedMachinePool | .spec.availabilityZones | | | AzureManagedMachinePool | .spec.maxPods | | | AzureManagedMachinePool | .spec.osType | | +| AzureManagedMachinePool | .spec.enableNodePublicIP | | +| AzureManagedMachinePool | .spec.nodePublicIPPrefixID | | ## Features diff --git a/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go b/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go index d7b30f45a2f..4e077ab1af0 100644 --- a/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go +++ b/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go @@ -46,6 +46,7 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.NodeLabels = restored.Spec.NodeLabels dst.Spec.EnableUltraSSD = restored.Spec.EnableUltraSSD dst.Spec.EnableNodePublicIP = restored.Spec.EnableNodePublicIP + dst.Spec.NodePublicIPPrefixID = restored.Spec.NodePublicIPPrefixID dst.Spec.ScaleSetPriority = restored.Spec.ScaleSetPriority dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index a0f5374e568..2f2439c10f1 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -918,6 +918,7 @@ func autoConvert_v1beta1_AzureManagedMachinePoolSpec_To_v1alpha3_AzureManagedMac // WARNING: in.EnableUltraSSD requires manual conversion: does not exist in peer-type // WARNING: in.OSType requires manual conversion: does not exist in peer-type // WARNING: in.EnableNodePublicIP requires manual conversion: does not exist in peer-type + // WARNING: in.NodePublicIPPrefixID requires manual conversion: does not exist in peer-type // WARNING: in.ScaleSetPriority requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go b/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go index ebeaadbbb91..a4e68ffdde9 100644 --- a/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go +++ b/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go @@ -46,6 +46,7 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.NodeLabels = restored.Spec.NodeLabels dst.Spec.EnableUltraSSD = restored.Spec.EnableUltraSSD dst.Spec.EnableNodePublicIP = restored.Spec.EnableNodePublicIP + dst.Spec.NodePublicIPPrefixID = restored.Spec.NodePublicIPPrefixID dst.Spec.ScaleSetPriority = restored.Spec.ScaleSetPriority dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index bd75f6f2798..17235b62259 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1218,6 +1218,7 @@ func autoConvert_v1beta1_AzureManagedMachinePoolSpec_To_v1alpha4_AzureManagedMac // WARNING: in.EnableUltraSSD requires manual conversion: does not exist in peer-type // WARNING: in.OSType requires manual conversion: does not exist in peer-type // WARNING: in.EnableNodePublicIP requires manual conversion: does not exist in peer-type + // WARNING: in.NodePublicIPPrefixID requires manual conversion: does not exist in peer-type // WARNING: in.ScaleSetPriority requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1beta1/azuremanagedmachinepool_types.go b/exp/api/v1beta1/azuremanagedmachinepool_types.go index 235f1d4dc84..1a423b57fb2 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_types.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_types.go @@ -103,6 +103,10 @@ type AzureManagedMachinePoolSpec struct { // +optional EnableNodePublicIP *bool `json:"enableNodePublicIP,omitempty"` + // NodePublicIPPrefixID specifies the public IP prefix resource ID which VM nodes should use IPs from. + // +optional + NodePublicIPPrefixID *string `json:"nodePublicIPPrefixID,omitempty"` + // ScaleSetPriority specifies the ScaleSetPriority value. Default to Regular. Possible values include: 'Regular', 'Spot' // +kubebuilder:validation:Enum=Regular;Spot // +optional diff --git a/exp/api/v1beta1/azuremanagedmachinepool_webhook.go b/exp/api/v1beta1/azuremanagedmachinepool_webhook.go index 38cca1a246a..7186f13a231 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_webhook.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_webhook.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "regexp" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" @@ -34,6 +35,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +var validNodePublicPrefixID = regexp.MustCompile(`(?i)^/?subscriptions/[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}/resourcegroups/[^/]+/providers/microsoft\.network/publicipprefixes/[^/]+$`) + //+kubebuilder:webhook:path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedmachinepool,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=infrastructure.cluster.x-k8s.io,resources=azuremanagedmachinepools,verbs=create;update,versions=v1beta1,name=default.azuremanagedmachinepools.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // Default implements webhook.Defaulter so a webhook will be registered for the type. @@ -61,6 +64,8 @@ func (m *AzureManagedMachinePool) ValidateCreate(client client.Client) error { m.validateOSType, m.validateName, m.validateNodeLabels, + m.validateNodePublicIPPrefixID, + m.validateEnableNodePublicIP, } var errs []error @@ -86,23 +91,11 @@ func (m *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client c err.Error())) } - if old.Spec.OSType != nil { - // Prevent OSType modification if it was already set to some value - if m.Spec.OSType == nil { - // unsetting the field is not allowed - allErrs = append(allErrs, - field.Invalid( - field.NewPath("Spec", "OSType"), - m.Spec.OSType, - "field is immutable, unsetting is not allowed")) - } else if *m.Spec.OSType != *old.Spec.OSType { - // changing the field is not allowed - allErrs = append(allErrs, - field.Invalid( - field.NewPath("Spec", "OSType"), - *m.Spec.OSType, - "field is immutable")) - } + if err := validateStringPtrImmutable( + field.NewPath("Spec", "OSType"), + old.Spec.OSType, + m.Spec.OSType); err != nil { + allErrs = append(allErrs, err) } if m.Spec.SKU != old.Spec.SKU { @@ -217,6 +210,12 @@ func (m *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client c m.Spec.EnableNodePublicIP); err != nil { allErrs = append(allErrs, err) } + if err := validateStringPtrImmutable( + field.NewPath("Spec", "NodePublicIPPrefixID"), + old.Spec.NodePublicIPPrefixID, + m.Spec.NodePublicIPPrefixID); err != nil { + allErrs = append(allErrs, err) + } if len(allErrs) != 0 { return apierrors.NewInvalid(GroupVersion.WithKind("AzureManagedMachinePool").GroupKind(), m.Name, allErrs) @@ -327,6 +326,27 @@ func (m *AzureManagedMachinePool) validateNodeLabels() error { return nil } +func (m *AzureManagedMachinePool) validateNodePublicIPPrefixID() error { + if m.Spec.NodePublicIPPrefixID != nil && !validNodePublicPrefixID.MatchString(*m.Spec.NodePublicIPPrefixID) { + return field.Invalid( + field.NewPath("Spec", "NodePublicIPPrefixID"), + m.Spec.NodePublicIPPrefixID, + fmt.Sprintf("resource ID must match %q", validNodePublicPrefixID.String())) + } + return nil +} + +func (m *AzureManagedMachinePool) validateEnableNodePublicIP() error { + if (m.Spec.EnableNodePublicIP == nil || !*m.Spec.EnableNodePublicIP) && + m.Spec.NodePublicIPPrefixID != nil { + return field.Invalid( + field.NewPath("Spec", "EnableNodePublicIP"), + m.Spec.EnableNodePublicIP, + "must be set to true when NodePublicIPPrefixID is set") + } + return nil +} + func ensureStringSlicesAreEqual(a []string, b []string) bool { if len(a) != len(b) { return false @@ -362,3 +382,21 @@ func validateBoolPtrImmutable(path *field.Path, oldVal, newVal *bool) *field.Err return nil } + +func validateStringPtrImmutable(path *field.Path, oldVal, newVal *string) *field.Error { + if oldVal != nil { + // Prevent modification if it was already set to some value + if newVal == nil { + // unsetting the field is not allowed + return field.Invalid(path, newVal, "field is immutable, unsetting is not allowed") + } + if *newVal != *oldVal { + // changing the field is not allowed + return field.Invalid(path, newVal, "field is immutable") + } + } else if newVal != nil { + return field.Invalid(path, newVal, "field is immutable, setting is not allowed") + } + + return nil +} diff --git a/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go b/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go index b449b84a6f6..17951fb969f 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go @@ -568,8 +568,6 @@ func TestValidateBoolPtrImmutable(t *testing.T) { } func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { - g := NewWithT(t) - tests := []struct { name string ammp *AzureManagedMachinePool @@ -698,10 +696,91 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { wantErr: true, errorLen: 1, }, + { + name: "pool with invalid public ip prefix", + ammp: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + EnableNodePublicIP: to.BoolPtr(true), + NodePublicIPPrefixID: to.StringPtr("not a valid resource ID"), + }, + }, + wantErr: true, + errorLen: 1, + }, + { + name: "pool with public ip prefix cannot omit node public IP", + ammp: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + EnableNodePublicIP: nil, + NodePublicIPPrefixID: to.StringPtr("subscriptions/11111111-2222-aaaa-bbbb-cccccccccccc/resourceGroups/public-ip-test/providers/Microsoft.Network/publicipprefixes/public-ip-prefix"), + }, + }, + wantErr: true, + errorLen: 1, + }, + { + name: "pool with public ip prefix cannot disable node public IP", + ammp: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + EnableNodePublicIP: to.BoolPtr(false), + NodePublicIPPrefixID: to.StringPtr("subscriptions/11111111-2222-aaaa-bbbb-cccccccccccc/resourceGroups/public-ip-test/providers/Microsoft.Network/publicipprefixes/public-ip-prefix"), + }, + }, + wantErr: true, + errorLen: 1, + }, + { + name: "pool with public ip prefix with node public IP enabled ok", + ammp: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + EnableNodePublicIP: to.BoolPtr(true), + NodePublicIPPrefixID: to.StringPtr("subscriptions/11111111-2222-aaaa-bbbb-cccccccccccc/resourceGroups/public-ip-test/providers/Microsoft.Network/publicipprefixes/public-ip-prefix"), + }, + }, + wantErr: false, + }, + { + name: "pool with public ip prefix with leading slash with node public IP enabled ok", + ammp: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + EnableNodePublicIP: to.BoolPtr(true), + NodePublicIPPrefixID: to.StringPtr("/subscriptions/11111111-2222-aaaa-bbbb-cccccccccccc/resourceGroups/public-ip-test/providers/Microsoft.Network/publicipprefixes/public-ip-prefix"), + }, + }, + wantErr: false, + }, + { + name: "pool without public ip prefix with node public IP unset ok", + ammp: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + EnableNodePublicIP: nil, + }, + }, + wantErr: false, + }, + { + name: "pool without public ip prefix with node public IP enabled ok", + ammp: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + EnableNodePublicIP: to.BoolPtr(true), + }, + }, + wantErr: false, + }, + { + name: "pool without public ip prefix with node public IP disabled ok", + ammp: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + EnableNodePublicIP: to.BoolPtr(false), + }, + }, + wantErr: false, + }, } var client client.Client for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) err := tc.ammp.ValidateCreate(client) if tc.wantErr { g.Expect(err).To(HaveOccurred()) diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index b5460bf39cd..e17c8960fcd 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -856,6 +856,11 @@ func (in *AzureManagedMachinePoolSpec) DeepCopyInto(out *AzureManagedMachinePool *out = new(bool) **out = **in } + if in.NodePublicIPPrefixID != nil { + in, out := &in.NodePublicIPPrefixID, &out.NodePublicIPPrefixID + *out = new(string) + **out = **in + } if in.ScaleSetPriority != nil { in, out := &in.ScaleSetPriority, &out.ScaleSetPriority *out = new(string) diff --git a/test/e2e/aks.go b/test/e2e/aks.go index 51ed8ec30bf..f2bc297e6c6 100644 --- a/test/e2e/aks.go +++ b/test/e2e/aks.go @@ -24,17 +24,23 @@ import ( "fmt" "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2020-02-01/containerservice" + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-08-01/network" "github.com/Azure/go-autorest/autorest/azure/auth" + "github.com/Azure/go-autorest/autorest/to" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/pkg/errors" "golang.org/x/mod/semver" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util/conditions" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -348,3 +354,119 @@ func GetLatestStableAKSKubernetesVersion(ctx context.Context, subscriptionID, lo } return maxVersion, nil } + +type AKSPublicIPPrefixSpecInput struct { + Cluster *clusterv1.Cluster + KubernetesVersion string + WaitIntervals []interface{} +} + +func AKSPublicIPPrefixSpec(ctx context.Context, inputGetter func() AKSPublicIPPrefixSpecInput) { + input := inputGetter() + + settings, err := auth.GetSettingsFromEnvironment() + Expect(err).NotTo(HaveOccurred()) + subscriptionID := settings.GetSubscriptionID() + auth, err := settings.GetAuthorizer() + Expect(err).NotTo(HaveOccurred()) + + mgmtClient := bootstrapClusterProxy.GetClient() + Expect(mgmtClient).NotTo(BeNil()) + + infraControlPlane := &infrav1exp.AzureManagedControlPlane{} + err = mgmtClient.Get(ctx, client.ObjectKey{Namespace: input.Cluster.Spec.ControlPlaneRef.Namespace, Name: input.Cluster.Spec.ControlPlaneRef.Name}, infraControlPlane) + Expect(err).NotTo(HaveOccurred()) + + resourceGroupName := infraControlPlane.Spec.ResourceGroupName + + publicIPPrefixClient := network.NewPublicIPPrefixesClient(subscriptionID) + publicIPPrefixClient.Authorizer = auth + + By("Creating public IP prefix with 2 addresses") + publicIPPrefixFuture, err := publicIPPrefixClient.CreateOrUpdate(ctx, resourceGroupName, input.Cluster.Name, network.PublicIPPrefix{ + Location: to.StringPtr(infraControlPlane.Spec.Location), + Sku: &network.PublicIPPrefixSku{ + Name: network.PublicIPPrefixSkuNameStandard, + }, + PublicIPPrefixPropertiesFormat: &network.PublicIPPrefixPropertiesFormat{ + PrefixLength: to.Int32Ptr(31), // In bits. This provides 2 addresses. + }, + }) + Expect(err).NotTo(HaveOccurred()) + var publicIPPrefix network.PublicIPPrefix + Eventually(func() error { + publicIPPrefix, err = publicIPPrefixFuture.Result(publicIPPrefixClient) + Logf("Got err %v", err) + return err + }, input.WaitIntervals...).Should(Succeed()) + + By("Creating node pool with 3 nodes") + infraMachinePool := &infrav1exp.AzureManagedMachinePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool3", + Namespace: input.Cluster.Namespace, + }, + Spec: infrav1exp.AzureManagedMachinePoolSpec{ + Mode: "User", + SKU: "Standard_D2s_v3", + EnableNodePublicIP: to.BoolPtr(true), + NodePublicIPPrefixID: to.StringPtr("/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroupName + "/providers/Microsoft.Network/publicipprefixes/" + *publicIPPrefix.Name), + }, + } + err = mgmtClient.Create(ctx, infraMachinePool) + Expect(err).NotTo(HaveOccurred()) + + machinePool := &expv1.MachinePool{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: infraMachinePool.Namespace, + Name: infraMachinePool.Name, + }, + Spec: expv1.MachinePoolSpec{ + ClusterName: input.Cluster.Name, + Replicas: to.Int32Ptr(3), + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Bootstrap: clusterv1.Bootstrap{ + DataSecretName: to.StringPtr(""), + }, + ClusterName: input.Cluster.Name, + InfrastructureRef: corev1.ObjectReference{ + APIVersion: infrav1exp.GroupVersion.String(), + Kind: "AzureManagedMachinePool", + Name: infraMachinePool.Name, + }, + Version: to.StringPtr(input.KubernetesVersion), + }, + }, + }, + } + err = mgmtClient.Create(ctx, machinePool) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying the AzureManagedMachinePool converges to a failed ready status") + Eventually(func(g Gomega) { + infraMachinePool := &infrav1exp.AzureManagedMachinePool{} + err := mgmtClient.Get(ctx, client.ObjectKeyFromObject(machinePool), infraMachinePool) + g.Expect(err).NotTo(HaveOccurred()) + cond := conditions.Get(infraMachinePool, infrav1.AgentPoolsReadyCondition) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(Equal(corev1.ConditionFalse)) + g.Expect(cond.Reason).To(Equal(infrav1.FailedReason)) + g.Expect(cond.Message).To(HavePrefix("failed to find vm scale set")) + }, input.WaitIntervals...).Should(Succeed()) + + By("Scaling the MachinePool to 2 nodes") + err = mgmtClient.Get(ctx, client.ObjectKeyFromObject(machinePool), machinePool) + Expect(err).NotTo(HaveOccurred()) + machinePool.Spec.Replicas = to.Int32Ptr(2) + err = mgmtClient.Update(ctx, machinePool) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying the AzureManagedMachinePool becomes ready") + Eventually(func(g Gomega) { + infraMachinePool := &infrav1exp.AzureManagedMachinePool{} + err := mgmtClient.Get(ctx, client.ObjectKeyFromObject(machinePool), infraMachinePool) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(conditions.IsTrue(infraMachinePool, infrav1.AgentPoolsReadyCondition)).To(BeTrue()) + }, input.WaitIntervals...).Should(Succeed()) +} diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 04fc29e1ef5..5c79e805d41 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -594,6 +594,16 @@ var _ = Describe("Workload cluster creation", func() { WaitForControlPlaneMachinesReady: WaitForAKSControlPlaneReady, }, }, result) + + By("creating a machine pool with public IP addresses from a prefix", func() { + AKSPublicIPPrefixSpec(ctx, func() AKSPublicIPPrefixSpecInput { + return AKSPublicIPPrefixSpecInput{ + Cluster: result.Cluster, + KubernetesVersion: kubernetesVersion, + WaitIntervals: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + } + }) + }) }) })