From 61ffb6d69b4f668572c51b2441c8246c64b7d5ea Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 12 Oct 2023 10:41:52 -0700 Subject: [PATCH] test: validate AKS node pool provisioned stability Signed-off-by: Jack Francis --- test/e2e/aks.go | 38 ++++++++++++++++++++++++++++++++-- test/e2e/config/azure-dev.yaml | 1 + 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/test/e2e/aks.go b/test/e2e/aks.go index 1c61eda53e7..ca84b21254c 100644 --- a/test/e2e/aks.go +++ b/test/e2e/aks.go @@ -101,13 +101,16 @@ func DiscoverAndWaitForAKSControlPlaneReady(ctx context.Context, input DiscoverA Expect(controlPlane).NotTo(BeNil()) Logf("Waiting for all AKS machines in the %s/%s 'system' node pool to exist", controlPlane.Namespace, controlPlane.Name) - WaitForAllControlPlaneAndMachinesToExist(ctx, WaitForControlPlaneAndMachinesReadyInput{ + waitForControlPlaneAndMachinesReadyInput := WaitForControlPlaneAndMachinesReadyInput{ Lister: input.Lister, Getter: input.Getter, ControlPlane: controlPlane, ClusterName: input.Cluster.Name, Namespace: input.Cluster.Namespace, - }, intervals...) + } + WaitForAllControlPlaneAndMachinesToExist(ctx, waitForControlPlaneAndMachinesReadyInput, intervals...) + Logf("Waiting to verify that all AzureManagedMachinePools are in a stable state") + WaitForAKSMachinePoolsToStabilize(ctx, waitForControlPlaneAndMachinesReadyInput, intervals...) } // GetAzureManagedControlPlaneByClusterInput contains the fields the required for fetching the azure managed control plane. @@ -208,3 +211,34 @@ func WaitForAKSSystemNodePoolMachinesToExist(ctx context.Context, input WaitForC return false }, intervals...).Should(Equal(true), "System machine pools not detected") } + +// WaitForAKSMachinePoolsToStabilize waits for a certain number of machines in the "system" node pool to exist. +func WaitForAKSMachinePoolsToStabilize(ctx context.Context, input WaitForControlPlaneAndMachinesReadyInput, intervals ...interface{}) { + var specName = "azuremanagedmachinepools-stabilize" + // First we wait for all AzureManagedMachinePools to be in a Ready state + Eventually(func(g Gomega) { + listOptions := client.InNamespace(input.Namespace) + ammpList := &infrav1.AzureManagedMachinePoolList{} + + err := input.Lister.List(ctx, ammpList, listOptions) + g.Expect(err).NotTo(HaveOccurred()) + + for _, pool := range ammpList.Items { + g.Expect(pool.Status.Ready).To(BeTrue()) + } + + }, intervals...).Should(Succeed()) + // Then we wait for a time to verify that no pool re-enters a non-Ready state + Eventually(func(g Gomega) { + listOptions := client.InNamespace(input.Namespace) + ammpList := &infrav1.AzureManagedMachinePoolList{} + + err := input.Lister.List(ctx, ammpList, listOptions) + g.Expect(err).NotTo(HaveOccurred()) + + for _, pool := range ammpList.Items { + g.Expect(pool.Status.Ready).To(BeFalse()) + } + + }, e2eConfig.GetIntervals(specName, "wait-machines-provisioned-stability")...).Should(Not(Succeed())) +} diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 6a6fa1ec4b3..5833b4b3a11 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -222,6 +222,7 @@ intervals: default/wait-service: ["15m", "10s"] default/wait-machine-pool-nodes: ["30m", "10s"] default/wait-nsg-update: ["20m", "10s"] + default/wait-machines-provisioned-stability: ["5m", "10s"] csi-migration/wait-controlplane-upgrade: ["60m", "10s"] csi-migration/wait-worker-nodes: ["60m", "10s"] csi-migration/wait-control-plane: ["60m", "10s"]