test: validate AKS node pool provisioned stability

Signed-off-by: Jack Francis <[email protected]>

test/e2e/aks.go

@@ -101,13 +101,16 @@ func DiscoverAndWaitForAKSControlPlaneReady(ctx context.Context, input DiscoverA
 	Expect(controlPlane).NotTo(BeNil())
 
 	Logf("Waiting for all AKS machines in the %s/%s 'system' node pool to exist", controlPlane.Namespace, controlPlane.Name)
-	WaitForAllControlPlaneAndMachinesToExist(ctx, WaitForControlPlaneAndMachinesReadyInput{
+	waitForControlPlaneAndMachinesReadyInput := WaitForControlPlaneAndMachinesReadyInput{
 		Lister:       input.Lister,
 		Getter:       input.Getter,
 		ControlPlane: controlPlane,
 		ClusterName:  input.Cluster.Name,
 		Namespace:    input.Cluster.Namespace,
-	}, intervals...)
+	}
+	WaitForAllControlPlaneAndMachinesToExist(ctx, waitForControlPlaneAndMachinesReadyInput, intervals...)
+	Logf("Waiting to verify that all AzureManagedMachinePools are in a stable state")
+	WaitForAKSMachinePoolsToStabilize(ctx, waitForControlPlaneAndMachinesReadyInput, intervals...)
 }
 
 // GetAzureManagedControlPlaneByClusterInput contains the fields the required for fetching the azure managed control plane.
@@ -208,3 +211,34 @@ func WaitForAKSSystemNodePoolMachinesToExist(ctx context.Context, input WaitForC
 		return false
 	}, intervals...).Should(Equal(true), "System machine pools not detected")
 }
+
+// WaitForAKSMachinePoolsToStabilize waits for a certain number of machines in the "system" node pool to exist.
+func WaitForAKSMachinePoolsToStabilize(ctx context.Context, input WaitForControlPlaneAndMachinesReadyInput, intervals ...interface{}) {
+	var specName = "azuremanagedmachinepools-stabilize"
+	// First we wait for all AzureManagedMachinePools to be in a Ready state
+	Eventually(func(g Gomega) {
+		listOptions := client.InNamespace(input.Namespace)
+		ammpList := &infrav1.AzureManagedMachinePoolList{}
+
+		err := input.Lister.List(ctx, ammpList, listOptions)
+		g.Expect(err).NotTo(HaveOccurred())
+
+		for _, pool := range ammpList.Items {
+			g.Expect(pool.Status.Ready).To(BeTrue())
+		}
+
+	}, intervals...).Should(Succeed())
+	// Then we wait for a time to verify that no pool re-enters a non-Ready state
+	Eventually(func(g Gomega) {
+		listOptions := client.InNamespace(input.Namespace)
+		ammpList := &infrav1.AzureManagedMachinePoolList{}
+
+		err := input.Lister.List(ctx, ammpList, listOptions)
+		g.Expect(err).NotTo(HaveOccurred())
+
+		for _, pool := range ammpList.Items {
+			g.Expect(pool.Status.Ready).To(BeFalse())
+		}
+
+	}, e2eConfig.GetIntervals(specName, "wait-machines-provisioned-stability")...).Should(Not(Succeed()))
+}

test/e2e/config/azure-dev.yaml

@@ -222,6 +222,7 @@ intervals:
   default/wait-service: ["15m", "10s"]
   default/wait-machine-pool-nodes: ["30m", "10s"]
   default/wait-nsg-update: ["20m", "10s"]
+  default/wait-machines-provisioned-stability: ["5m", "10s"]
   csi-migration/wait-controlplane-upgrade: ["60m", "10s"]
   csi-migration/wait-worker-nodes: ["60m", "10s"]
   csi-migration/wait-control-plane: ["60m", "10s"]