MHC: control plane init / cluster infra ready

Change the MachineHealthCheck logic to require the Cluster's "control
plane initialized" and "infrastructure ready" conditions to be true
before proceeding to determine if a target is unhealhty.

We can't look just at a Machine's last updated time when determining if
the Machine has exceeded the node startup timeout.  A node can't
bootstrap until after the cluster's infrastructure is ready and the
control plane is initialized, and we need to base node startup timeout
on the latter of machine creation time, control plane initialized time,
and cluster infrastructure ready time.

Signed-off-by: Andy Goldstein <[email protected]>

controllers/cluster_controller.go

@@ -462,19 +462,24 @@ func splitMachineList(list *clusterv1.MachineList) (*clusterv1.MachineList, *clu
 }
 
 func (r *ClusterReconciler) reconcileControlPlaneInitialized(ctx context.Context, cluster *clusterv1.Cluster) (ctrl.Result, error) {
+	log := ctrl.LoggerFrom(ctx)
+
 	// Skip checking if the control plane is initialized when using a Control Plane Provider (this is reconciled in
 	// reconcileControlPlane instead).
 	if cluster.Spec.ControlPlaneRef != nil {
+		log.V(4).Info("Skipping reconcileControlPlaneInitialized because cluster has a controlPlaneRef")
 		return ctrl.Result{}, nil
 	}
 
 	if conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) {
+		log.V(4).Info("Skipping reconcileControlPlaneInitialized because control plane already initialized")
 		return ctrl.Result{}, nil
 	}
 
+	log.V(4).Info("Checking for control plane initialization")
+
 	machines, err := getActiveMachinesInCluster(ctx, r.Client, cluster.Namespace, cluster.Name)
 	if err != nil {
-		log := ctrl.LoggerFrom(ctx)
 		log.Error(err, "unable to determine ControlPlaneInitialized")
 		return ctrl.Result{}, err
 	}

controllers/machine_controller_test.go

@@ -109,14 +109,8 @@ func TestWatches(t *testing.T) {
 		g.Expect(testEnv.Cleanup(ctx, do...)).To(Succeed())
 	}(ns, testCluster, defaultBootstrap)
 
-	// Patch cluster control plane initialized (this is required to start node watch)
-	patchHelper, err := patch.NewHelper(testCluster, testEnv)
-	g.Expect(err).ShouldNot(HaveOccurred())
-	conditions.MarkTrue(testCluster, clusterv1.ControlPlaneInitializedCondition)
-	g.Expect(patchHelper.Patch(ctx, testCluster, patch.WithStatusObservedGeneration{})).To(Succeed())
-
 	// Patch infra machine ready
-	patchHelper, err = patch.NewHelper(infraMachine, testEnv)
+	patchHelper, err := patch.NewHelper(infraMachine, testEnv)
 	g.Expect(err).ShouldNot(HaveOccurred())
 	g.Expect(unstructured.SetNestedField(infraMachine.Object, true, "status", "ready")).To(Succeed())
 	g.Expect(patchHelper.Patch(ctx, infraMachine, patch.WithStatusObservedGeneration{})).To(Succeed())
@@ -132,6 +126,9 @@ func TestWatches(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: "machine-created-",
 			Namespace:    ns.Name,
+			Labels: map[string]string{
+				clusterv1.MachineControlPlaneLabelName: "",
+			},
 		},
 		Spec: clusterv1.MachineSpec{
 			ClusterName: testCluster.Name,